Add a changelog entry for 10.2.8

(cherry picked from commit 93bcd39a83)

.circleci/config.yml

@@ -158,6 +158,14 @@ jobs:
                 cp core.* /tmp/core_dumps
             fi
           when: on_fail
+      - run:
+          name: 'Copy pg_upgrade logs for newData dir'
+          command: |
+            mkdir -p /tmp/pg_upgrade_newData_logs
+            if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
+                cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
+            fi
+          when: on_fail
       - store_artifacts:
           name: 'Save regressions'
           path: src/test/regress/regression.diffs
@@ -166,6 +174,9 @@ jobs:
           name: 'Save core dumps'
           path: /tmp/core_dumps
           when: on_fail
+      - store_artifacts:
+          name: 'Save pg_upgrade logs for newData dir'
+          path: /tmp/pg_upgrade_newData_logs
       - codecov/upload:
           flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade'
 
@@ -379,7 +390,7 @@ jobs:
           when: on_fail
       - store_artifacts:
           name: 'Save tap logs'
-          path: /home/circleci/project/src/test/recovery/tmp_check/log
+          path: /home/circleci/project/src/test/<< parameters.suite >>/tmp_check/log
           when: on_fail
       - store_artifacts:
           name: 'Save core dumps'
@@ -451,7 +462,7 @@ workflows:
       - build:
           name: build-14
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
 
       - check-style
       - check-sql-snapshots
@@ -522,6 +533,12 @@ workflows:
           image_tag: '12.8'
           suite: recovery
           requires: [build-12]
+      - tap-test-citus:
+          name: 'test-12_tap-columnar-freezing'
+          pg_major: 12
+          image_tag: '12.8'
+          suite: columnar_freezing
+          requires: [build-12]
       - test-citus:
           name: 'test-12_check-failure'
           pg_major: 12
@@ -596,6 +613,12 @@ workflows:
           image_tag: '13.4'
           suite: recovery
           requires: [build-13]
+      - tap-test-citus:
+          name: 'test-13_tap-columnar-freezing'
+          pg_major: 13
+          image_tag: '13.4'
+          suite: columnar_freezing
+          requires: [build-13]
       - test-citus:
           name: 'test-13_check-failure'
           pg_major: 13
@@ -607,74 +630,80 @@ workflows:
       - test-citus:
           name: 'test-14_check-multi'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-multi
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-multi-1'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-multi-1
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-mx'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-multi-mx
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-vanilla'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-vanilla
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-isolation'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-isolation
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-worker'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-worker
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-operations'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-operations
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-follower-cluster'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-follower-cluster
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-columnar'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-columnar
           requires: [build-14]
       - test-citus:
           name: 'test-14_check-columnar-isolation'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-columnar-isolation
           requires: [build-14]
       - tap-test-citus:
           name: 'test_14_tap-recovery'
           pg_major: 14
-          image_tag: '14beta3'
+          image_tag: '14.0'
           suite: recovery
           requires: [build-14]
+      - tap-test-citus:
+          name: 'test-14_tap-columnar-freezing'
+          pg_major: 14
+          image_tag: '14.0'
+          suite: columnar_freezing
+          requires: [build-14]
       - test-citus:
           name: 'test-14_check-failure'
           pg_major: 14
           image: citus/failtester
-          image_tag: '14beta3'
+          image_tag: '14.0'
           make: check-failure
           requires: [build-14]
 
@@ -689,14 +718,14 @@ workflows:
           name: 'test-12-14_check-pg-upgrade'
           old_pg_major: 12
           new_pg_major: 14
-          image_tag: '12-13-14'
+          image_tag: '12.8-13.4-14.0'
           requires: [build-12,build-14]
 
       - test-pg-upgrade:
           name: 'test-13-14_check-pg-upgrade'
           old_pg_major: 13
           new_pg_major: 14
-          image_tag: '12-13-14'
+          image_tag: '12.8-13.4-14.0'
           requires: [build-13,build-14]
 
       - test-citus-upgrade:

CHANGELOG.md

@@ -1,3 +1,105 @@
+### citus v10.2.8 (August 19, 2022) ###
+
+* Fixes compilation warning caused by latest upgrade script changes
+
+* Fixes compilation warning on PG13 + OpenSSL 3.0
+
+### citus v10.2.7 (August 19, 2022) ###
+
+* Fixes a bug that could cause failures in `INSERT INTO .. SELECT`
+
+* Fixes a bug that could cause leaking files when materialized views are
+  refreshed
+
+* Fixes an unexpected error for foreign tables when upgrading Postgres
+
+* Fixes columnar freezing/wraparound bug
+
+* Fixes reference table lock contention
+
+* Prevents alter table functions from dropping extensions
+
+### citus v10.2.5 (March 15, 2022) ###
+
+* Fixes a bug that could cause `worker_save_query_explain_analyze` to fail on
+  custom types
+
+* Fixes a bug that limits usage of sequences in non-integer columns
+
+* Fixes a crash that occurs when the aggregate that cannot be pushed-down
+  returns empty result from a worker
+
+* Improves concurrent metadata syncing and metadata changing DDL operations
+
+### citus v10.2.4 (February 1, 2022) ###
+
+* Adds support for operator class parameters in indexes
+
+* Fixes a bug with distributed functions that have `OUT` parameters or
+  return `TABLE`
+
+* Fixes a build error that happens when `lz4` is not installed
+
+* Improves self-deadlock prevention for `CREATE INDEX` &
+  `REINDEX CONCURRENTLY` commands for builds using PG14 or higher
+
+* Fixes a bug that causes commands to fail when `application_name` is set
+
+### citus v10.2.3 (November 29, 2021) ###
+
+* Adds `fix_partition_shard_index_names` udf to fix currently broken
+  partition index names
+
+* Fixes a bug that could break `DROP SCHEMA/EXTENSION` commands when there is
+  a columnar table
+
+* Fixes a bug that could break pg upgrades due to missing `pg_depend` records
+  for columnar table access method
+
+* Fixes a missing `FROM` clause entry error
+
+* Fixes an unexpected error that occurs when writing to a columnar table
+  created in older versions
+
+* Fixes issue when compiling Citus from source with some compilers
+
+* Reinstates optimisation for uniform shard interval ranges
+
+* Relaxes table ownership check to privileges check while acquiring lock
+
+### citus v10.2.2 (October 14, 2021) ###
+
+* Fixes a bug that causes reading columnar metapage as all-zeros when
+  writing to a columnar table
+
+* Fixes a bug that could cause prerequisite columnar table access method
+  objects being not created during pg upgrades
+
+* Fixes a bug that could cause `CREATE INDEX` to fail for expressions when
+  using custom `search_path`
+
+* Fixes an unexpected error that occurs due to aborted writes to a columnar
+  table with an index
+
+### citus v10.2.1 (September 24, 2021) ###
+
+* Adds missing version-mismatch checks for columnar tables
+
+* Adds missing version-mismatch checks for internal functions
+
+* Fixes a bug that could cause partition shards being not co-located with
+  parent shards
+
+* Fixes a bug that prevents pushing down boolean expressions when using
+  columnar custom scan
+
+* Fixes a clog lookup failure that could occur when writing to a columnar table
+
+* Fixes an issue that could cause unexpected errors when there is an
+  in-progress write to a columnar table
+
+* Revokes read access to `columnar.chunk` from unprivileged user
+
 ### citus v10.2.0 (September 14, 2021) ###
 
 * Adds PostgreSQL 14 support

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Citus 10.2devel.
+# Generated by GNU Autoconf 2.69 for Citus 10.2.8.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Citus'
 PACKAGE_TARNAME='citus'
-PACKAGE_VERSION='10.2devel'
+PACKAGE_VERSION='10.2.8'
-PACKAGE_STRING='Citus 10.2devel'
+PACKAGE_STRING='Citus 10.2.8'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1260,7 +1260,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures Citus 10.2devel to adapt to many kinds of systems.
+\`configure' configures Citus 10.2.8 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1322,7 +1322,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of Citus 10.2devel:";;
+     short | recursive ) echo "Configuration of Citus 10.2.8:";;
    esac
   cat <<\_ACEOF
 
@@ -1425,7 +1425,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-Citus configure 10.2devel
+Citus configure 10.2.8
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1908,7 +1908,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by Citus $as_me 10.2devel, which was
+It was created by Citus $as_me 10.2.8, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -4543,7 +4543,9 @@ if test "${with_lz4+set}" = set; then :
   withval=$with_lz4;
   case $withval in
     yes)
-      :
+
+$as_echo "#define HAVE_CITUS_LIBLZ4 1" >>confdefs.h
+
       ;;
     no)
       :
@@ -4556,6 +4558,8 @@ if test "${with_lz4+set}" = set; then :
 else
   with_lz4=yes
 
+$as_echo "#define HAVE_CITUS_LIBLZ4 1" >>confdefs.h
+
 fi
 
 
@@ -5356,7 +5360,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by Citus $as_me 10.2devel, which was
+This file was extended by Citus $as_me 10.2.8, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5418,7 +5422,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-Citus config.status 10.2devel
+Citus config.status 10.2.8
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -5,7 +5,7 @@
 # everyone needing autoconf installed, the resulting files are checked
 # into the SCM.
 
-AC_INIT([Citus], [10.2devel])
+AC_INIT([Citus], [10.2.8])
 AC_COPYRIGHT([Copyright (c) Citus Data, Inc.])
 
 # we'll need sed and awk for some of the version commands
@@ -220,7 +220,8 @@ AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL",
 # LZ4
 #
 PGAC_ARG_BOOL(with, lz4, yes,
-              [do not use lz4])
+              [do not use lz4],
+              [AC_DEFINE([HAVE_CITUS_LIBLZ4], 1, [Define to 1 to build with lz4 support. (--with-lz4)])])
 AC_SUBST(with_lz4)
 
 if test "$with_lz4" = yes; then

src/backend/columnar/columnar.c

@@ -29,7 +29,7 @@
 
 #if HAVE_LIBZSTD
 #define DEFAULT_COMPRESSION_TYPE COMPRESSION_ZSTD
-#elif HAVE_LIBLZ4
+#elif HAVE_CITUS_LIBLZ4
 #define DEFAULT_COMPRESSION_TYPE COMPRESSION_LZ4
 #else
 #define DEFAULT_COMPRESSION_TYPE COMPRESSION_PG_LZ
@@ -44,7 +44,7 @@ static const struct config_enum_entry columnar_compression_options[] =
 {
 	{ "none", COMPRESSION_NONE, false },
 	{ "pglz", COMPRESSION_PG_LZ, false },
-#if HAVE_LIBLZ4
+#if HAVE_CITUS_LIBLZ4
 	{ "lz4", COMPRESSION_LZ4, false },
 #endif
 #if HAVE_LIBZSTD

src/backend/columnar/columnar_compression.c

@@ -19,7 +19,7 @@
 
 #include "columnar/columnar_compression.h"
 
-#if HAVE_LIBLZ4
+#if HAVE_CITUS_LIBLZ4
 #include <lz4.h>
 #endif
 
@@ -63,7 +63,7 @@ CompressBuffer(StringInfo inputBuffer,
 {
 	switch (compressionType)
 	{
-#if HAVE_LIBLZ4
+#if HAVE_CITUS_LIBLZ4
 		case COMPRESSION_LZ4:
 		{
 			int maximumLength = LZ4_compressBound(inputBuffer->len);
@@ -170,7 +170,7 @@ DecompressBuffer(StringInfo buffer,
 			return buffer;
 		}
 
-#if HAVE_LIBLZ4
+#if HAVE_CITUS_LIBLZ4
 		case COMPRESSION_LZ4:
 		{
 			StringInfo decompressedBuffer = makeStringInfo();

src/backend/columnar/columnar_customscan.c

@@ -605,10 +605,11 @@ RelationIdGetNumberOfAttributes(Oid relationId)
 /*
  * CheckVarStats() checks whether a qual involving this Var is likely to be
  * useful based on the correlation stats. If so, or if stats are unavailable,
- * return true; otherwise return false.
+ * return true; otherwise return false and sets absVarCorrelation in case
+ * caller wants to use for logging purposes.
  */
 static bool
-CheckVarStats(PlannerInfo *root, Var *var, Oid sortop)
+CheckVarStats(PlannerInfo *root, Var *var, Oid sortop, float4 *absVarCorrelation)
 {
 	/*
 	 * Collect isunique, ndistinct, and varCorrelation.
@@ -642,6 +643,14 @@ CheckVarStats(PlannerInfo *root, Var *var, Oid sortop)
 	 */
 	if (Abs(varCorrelation) < ColumnarQualPushdownCorrelationThreshold)
 	{
+		if (absVarCorrelation)
+		{
+			/*
+			 * Report absVarCorrelation if caller wants to know why given
+			 * var is rejected.
+			 */
+			*absVarCorrelation = Abs(varCorrelation);
+		}
 		return false;
 	}
 
@@ -674,7 +683,7 @@ ExprReferencesRelid(Expr *expr, Index relid)
 
 
 /*
- * CheckPushdownClause tests to see if clause is a candidate for pushing down
+ * ExtractPushdownClause extracts an Expr node from given clause for pushing down
  * into the given rel (including join clauses). This test may not be exact in
  * all cases; it's used to reduce the search space for parameterization.
  *
@@ -683,19 +692,134 @@ ExprReferencesRelid(Expr *expr, Index relid)
  * and that doesn't seem worth the effort. Here we just look for "Var op Expr"
  * or "Expr op Var", where Var references rel and Expr references other rels
  * (or no rels at all).
+ *
+ * Moreover, this function also looks into BoolExpr's to recursively extract
+ * pushdownable OpExpr's of them:
+ * i)   AND_EXPR:
+ *      Take pushdownable args of AND expressions by ignoring the other args.
+ * ii)  OR_EXPR:
+ *      Ignore the whole OR expression if we cannot exract a pushdownable Expr
+ *      from one of its args.
+ * iii) NOT_EXPR:
+ *      Simply ignore NOT expressions since we don't expect to see them before
+ *      an expression that we can pushdown, see the comment in function.
+ *
+ * The reasoning for those three rules could also be summarized as such;
+ * for any expression that we cannot push-down, we must assume that it
+ * evaluates to true.
+ *
+ * For example, given following WHERE clause:
+ * (
+ *     (a > random() OR a < 30)
+ *     AND
+ *     a < 200
+ * ) OR
+ * (
+ *     a = 300
+ *     OR
+ *     a > 400
+ * );
+ * Even if we can pushdown (a < 30), we cannot pushdown (a > random() OR a < 30)
+ * due to (a > random()). However, we can pushdown (a < 200), so we extract
+ * (a < 200) from the lhs of the top level OR expression.
+ *
+ * For the rhs of the top level OR expression, since we can pushdown both (a = 300)
+ * and (a > 400), we take this part as is.
+ *
+ * Finally, since both sides of the top level OR expression yielded pushdownable
+ * expressions, we will pushdown the following:
+ *  (a < 200) OR ((a = 300) OR (a > 400))
  */
-static bool
+static Expr *
-CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
+ExtractPushdownClause(PlannerInfo *root, RelOptInfo *rel, Node *node)
 {
-	if (!IsA(clause, OpExpr) || list_length(((OpExpr *) clause)->args) != 2)
+	CHECK_FOR_INTERRUPTS();
+	check_stack_depth();
+
+	if (node == NULL)
+	{
+		return NULL;
+	}
+
+	if (IsA(node, BoolExpr))
+	{
+		BoolExpr *boolExpr = castNode(BoolExpr, node);
+		if (boolExpr->boolop == NOT_EXPR)
+		{
+			/*
+			 * Standard planner should have already applied de-morgan rule to
+			 * simple NOT expressions. If we encounter with such an expression
+			 * here, then it can't be a pushdownable one, such as:
+			 *   WHERE id NOT IN (SELECT id FROM something).
+			 */
+			ereport(ColumnarPlannerDebugLevel,
+					(errmsg("columnar planner: cannot push down clause: "
+							"must not contain a subplan")));
+			return NULL;
+		}
+
+		List *pushdownableArgs = NIL;
+
+		Node *boolExprArg = NULL;
+		foreach_ptr(boolExprArg, boolExpr->args)
+		{
+			Expr *pushdownableArg = ExtractPushdownClause(root, rel,
+														  (Node *) boolExprArg);
+			if (pushdownableArg)
+			{
+				pushdownableArgs = lappend(pushdownableArgs, pushdownableArg);
+			}
+			else if (boolExpr->boolop == OR_EXPR)
+			{
+				ereport(ColumnarPlannerDebugLevel,
+						(errmsg("columnar planner: cannot push down clause: "
+								"all arguments of an OR expression must be "
+								"pushdownable but one of them was not, due "
+								"to the reason given above")));
+				return NULL;
+			}
+
+			/* simply skip AND args that we cannot pushdown */
+		}
+
+		int npushdownableArgs = list_length(pushdownableArgs);
+		if (npushdownableArgs == 0)
+		{
+			ereport(ColumnarPlannerDebugLevel,
+					(errmsg("columnar planner: cannot push down clause: "
+							"none of the arguments were pushdownable, "
+							"due to the reason(s) given above ")));
+			return NULL;
+		}
+		else if (npushdownableArgs == 1)
+		{
+			return (Expr *) linitial(pushdownableArgs);
+		}
+
+		if (boolExpr->boolop == AND_EXPR)
+		{
+			return make_andclause(pushdownableArgs);
+		}
+		else if (boolExpr->boolop == OR_EXPR)
+		{
+			return make_orclause(pushdownableArgs);
+		}
+		else
+		{
+			/* already discarded NOT expr, so should not be reachable */
+			return NULL;
+		}
+	}
+
+	if (!IsA(node, OpExpr) || list_length(((OpExpr *) node)->args) != 2)
 	{
 		ereport(ColumnarPlannerDebugLevel,
 				(errmsg("columnar planner: cannot push down clause: "
 						"must be binary operator expression")));
-		return false;
+		return NULL;
 	}
 
-	OpExpr *opExpr = castNode(OpExpr, clause);
+	OpExpr *opExpr = castNode(OpExpr, node);
 	Expr *lhs = list_nth(opExpr->args, 0);
 	Expr *rhs = list_nth(opExpr->args, 1);
 
@@ -721,15 +845,15 @@ CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
 						"must match 'Var <op> Expr' or 'Expr <op> Var'"),
 				 errhint("Var must only reference this rel, "
 						 "and Expr must not reference this rel")));
-		return false;
+		return NULL;
 	}
 
 	if (varSide->varattno <= 0)
 	{
 		ereport(ColumnarPlannerDebugLevel,
 				(errmsg("columnar planner: cannot push down clause: "
-						"var is whole-row reference")));
+						"var is whole-row reference or system column")));
-		return false;
+		return NULL;
 	}
 
 	if (contain_volatile_functions((Node *) exprSide))
@@ -737,7 +861,7 @@ CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
 		ereport(ColumnarPlannerDebugLevel,
 				(errmsg("columnar planner: cannot push down clause: "
 						"expr contains volatile functions")));
-		return false;
+		return NULL;
 	}
 
 	/* only the default opclass is used for qual pushdown. */
@@ -753,7 +877,7 @@ CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
 				(errmsg("columnar planner: cannot push down clause: "
 						"cannot find default btree opclass and opfamily for type: %s",
 						format_type_be(varSide->vartype))));
-		return false;
+		return NULL;
 	}
 
 	if (!op_in_opfamily(opExpr->opno, varOpFamily))
@@ -762,7 +886,7 @@ CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
 				(errmsg("columnar planner: cannot push down clause: "
 						"operator %d not a member of opfamily %d",
 						opExpr->opno, varOpFamily)));
-		return false;
+		return NULL;
 	}
 
 	Oid sortop = get_opfamily_member(varOpFamily, varOpcInType,
@@ -773,15 +897,20 @@ CheckPushdownClause(PlannerInfo *root, RelOptInfo *rel, Expr *clause)
 	 * Check that statistics on the Var support the utility of this
 	 * clause.
 	 */
-	if (!CheckVarStats(root, varSide, sortop))
+	float4 absVarCorrelation = 0;
+	if (!CheckVarStats(root, varSide, sortop, &absVarCorrelation))
 	{
 		ereport(ColumnarPlannerDebugLevel,
 				(errmsg("columnar planner: cannot push down clause: "
-						"var attribute %d is uncorrelated", varSide->varattno)));
+						"absolute correlation (%.3f) of var attribute %d is "
-		return false;
+						"smaller than the value configured in "
+						"\"columnar.qual_pushdown_correlation_threshold\" "
+						"(%.3f)", absVarCorrelation, varSide->varattno,
+						ColumnarQualPushdownCorrelationThreshold)));
+		return NULL;
 	}
 
-	return true;
+	return (Expr *) node;
 }
 
 
@@ -806,12 +935,19 @@ FilterPushdownClauses(PlannerInfo *root, RelOptInfo *rel, List *inputClauses)
 		 * there's something we should do with pseudoconstants here.
 		 */
 		if (rinfo->pseudoconstant ||
-			!bms_is_member(rel->relid, rinfo->required_relids) ||
+			!bms_is_member(rel->relid, rinfo->required_relids))
-			!CheckPushdownClause(root, rel, rinfo->clause))
 		{
 			continue;
 		}
 
+		Expr *pushdownableExpr = ExtractPushdownClause(root, rel, (Node *) rinfo->clause);
+		if (!pushdownableExpr)
+		{
+			continue;
+		}
+
+		rinfo = copyObject(rinfo);
+		rinfo->clause = pushdownableExpr;
 		filteredClauses = lappend(filteredClauses, rinfo);
 	}
 

src/backend/columnar/columnar_metadata.c

@@ -335,8 +335,13 @@ DeleteColumnarTableOptions(Oid regclass, bool missingOk)
 	 */
 	Assert(!IsBinaryUpgrade);
 
-	Relation columnarOptions = relation_open(ColumnarOptionsRelationId(),
+	Relation columnarOptions = try_relation_open(ColumnarOptionsRelationId(),
-											 RowExclusiveLock);
+												 RowExclusiveLock);
+	if (columnarOptions == NULL)
+	{
+		/* extension has been dropped */
+		return false;
+	}
 
 	/* find existing item to remove */
 	ScanKeyData scanKey[1] = { 0 };
@@ -1178,8 +1183,18 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 
 	heap_inplace_update(columnarStripes, modifiedTuple);
 
+	/*
+	 * Existing tuple now contains modifications, because we used
+	 * heap_inplace_update().
+	 */
+	HeapTuple newTuple = oldTuple;
+
+	/*
+	 * Must not pass modifiedTuple, because BuildStripeMetadata expects a real
+	 * heap tuple with MVCC fields.
+	 */
 	StripeMetadata *modifiedStripeMetadata = BuildStripeMetadata(columnarStripes,
-																 modifiedTuple);
+																 newTuple);
 
 	CommandCounterIncrement();
 
@@ -1233,6 +1248,8 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
 
 /*
  * BuildStripeMetadata builds a StripeMetadata object from given heap tuple.
+ *
+ * NB: heapTuple must be a proper heap tuple with MVCC fields.
  */
 static StripeMetadata *
 BuildStripeMetadata(Relation columnarStripes, HeapTuple heapTuple)
@@ -1269,7 +1286,8 @@ BuildStripeMetadata(Relation columnarStripes, HeapTuple heapTuple)
 	 * subtransaction id here.
 	 */
 	TransactionId entryXmin = HeapTupleHeaderGetXmin(heapTuple->t_data);
-	stripeMetadata->aborted = TransactionIdDidAbort(entryXmin);
+	stripeMetadata->aborted = !TransactionIdIsInProgress(entryXmin) &&
+							  TransactionIdDidAbort(entryXmin);
 	stripeMetadata->insertedByCurrentXact =
 		TransactionIdIsCurrentTransactionId(entryXmin);
 

src/backend/columnar/columnar_storage.c

@@ -104,6 +104,10 @@ typedef struct PhysicalAddr
 								  "version or run \"ALTER EXTENSION citus UPDATE\"."
 
 
+/* only for testing purposes */
+PG_FUNCTION_INFO_V1(test_columnar_storage_write_new_page);
+
+
 /*
  * Map logical offsets to a physical page and offset where the data is kept.
  */
@@ -667,6 +671,7 @@ ReadFromBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf,
 			  uint32 len, bool force)
 {
 	Buffer buffer = ReadBuffer(rel, blockno);
+	LockBuffer(buffer, BUFFER_LOCK_SHARE);
 	Page page = BufferGetPage(buffer);
 	PageHeader phdr = (PageHeader) page;
 
@@ -678,7 +683,7 @@ ReadFromBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf,
 	}
 
 	memcpy_s(buf, len, page + offset, len);
-	ReleaseBuffer(buffer);
+	UnlockReleaseBuffer(buffer);
 }
 
 
@@ -703,13 +708,32 @@ WriteToBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf,
 		PageInit(page, BLCKSZ, 0);
 	}
 
-	if (phdr->pd_lower != offset || phdr->pd_upper - offset < len)
+	if (phdr->pd_lower < offset || phdr->pd_upper - offset < len)
 	{
 		elog(ERROR,
 			 "attempt to write columnar data of length %d to offset %d of block %d of relation %d",
 			 len, offset, blockno, rel->rd_id);
 	}
 
+	/*
+	 * After a transaction has been rolled-back, we might be
+	 * over-writing the rolledback write, so phdr->pd_lower can be
+	 * different from addr.offset.
+	 *
+	 * We reset pd_lower to reset the rolledback write.
+	 *
+	 * Given that we always align page reservation to the next page as of
+	 * 10.2, having such a disk page is only possible if write operaion
+	 * failed in an older version of columnar, but now user attempts writing
+	 * to that table in version >= 10.2.
+	 */
+	if (phdr->pd_lower > offset)
+	{
+		ereport(DEBUG4, (errmsg("overwriting page %u", blockno),
+						 errdetail("This can happen after a roll-back.")));
+		phdr->pd_lower = offset;
+	}
+
 	START_CRIT_SECTION();
 
 	memcpy_s(page + phdr->pd_lower, phdr->pd_upper - phdr->pd_lower, buf, len);
@@ -819,3 +843,36 @@ ColumnarMetapageCheckVersion(Relation rel, ColumnarMetapage *metapage)
 						errhint(OLD_METAPAGE_VERSION_HINT)));
 	}
 }
+
+
+/*
+ * test_columnar_storage_write_new_page is a UDF only used for testing
+ * purposes. It could make more sense to define this in columnar_debug.c,
+ * but the storage layer doesn't expose ColumnarMetapage to any other files,
+ * so we define it here.
+ */
+Datum
+test_columnar_storage_write_new_page(PG_FUNCTION_ARGS)
+{
+	Oid relationId = PG_GETARG_OID(0);
+
+	Relation relation = relation_open(relationId, AccessShareLock);
+
+	/*
+	 * Allocate a new page, write some data to there, and set reserved offset
+	 * to the start of that page. That way, for a subsequent write operation,
+	 * storage layer would try to overwrite the page that we allocated here.
+	 */
+	uint64 newPageOffset = ColumnarStorageGetReservedOffset(relation, false);
+
+	ColumnarStorageReserveData(relation, 100);
+	ColumnarStorageWrite(relation, newPageOffset, "foo_bar", 8);
+
+	ColumnarMetapage metapage = ColumnarMetapageRead(relation, false);
+	metapage.reservedOffset = newPageOffset;
+	ColumnarOverwriteMetapage(relation, metapage);
+
+	relation_close(relation, AccessShareLock);
+
+	PG_RETURN_VOID();
+}

src/backend/columnar/columnar_tableam.c

@@ -85,7 +85,6 @@ typedef struct ColumnarScanDescData
 	List *scanQual;
 } ColumnarScanDescData;
 
-typedef struct ColumnarScanDescData *ColumnarScanDesc;
 
 /*
  * IndexFetchColumnarData is the scan state passed between index_fetch_begin,
@@ -173,6 +172,8 @@ columnar_beginscan(Relation relation, Snapshot snapshot,
 				   ParallelTableScanDesc parallel_scan,
 				   uint32 flags)
 {
+	CheckCitusVersion(ERROR);
+
 	int natts = relation->rd_att->natts;
 
 	/* attr_needed represents 0-indexed attribute numbers */
@@ -418,6 +419,8 @@ columnar_parallelscan_reinitialize(Relation rel, ParallelTableScanDesc pscan)
 static IndexFetchTableData *
 columnar_index_fetch_begin(Relation rel)
 {
+	CheckCitusVersion(ERROR);
+
 	Oid relfilenode = rel->rd_node.relNode;
 	if (PendingWritesInUpperTransactions(relfilenode, GetCurrentSubTransactionId()))
 	{
@@ -472,8 +475,11 @@ columnar_index_fetch_tuple(struct IndexFetchTableData *sscan,
 	*call_again = false;
 
 	/*
-	 * No dead tuples are possible in columnar, set it to false if it's
+	 * Initialize all_dead to false if passed to be non-NULL.
-	 * passed to be non-NULL.
+	 *
+	 * XXX: For aborted writes, we should set all_dead to true but this would
+	 * require implementing columnar_index_delete_tuples for simple deletion
+	 * of dead tuples (TM_IndexDeleteOp.bottomup = false).
 	 */
 	if (all_dead)
 	{
@@ -638,6 +644,8 @@ static bool
 columnar_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot,
 								  Snapshot snapshot)
 {
+	CheckCitusVersion(ERROR);
+
 	uint64 rowNumber = tid_to_row_number(slot->tts_tid);
 	StripeMetadata *stripeMetadata = FindStripeByRowNumber(rel, rowNumber, snapshot);
 	return stripeMetadata != NULL;
@@ -649,7 +657,47 @@ static TransactionId
 columnar_index_delete_tuples(Relation rel,
 							 TM_IndexDeleteOp *delstate)
 {
-	elog(ERROR, "columnar_index_delete_tuples not implemented");
+	CheckCitusVersion(ERROR);
+
+	/*
+	 * XXX: We didn't bother implementing index_delete_tuple for neither of
+	 * simple deletion and bottom-up deletion cases. There is no particular
+	 * reason for that, just to keep things simple.
+	 *
+	 * See the rest of this function to see how we deal with
+	 * index_delete_tuples requests made to columnarAM.
+	 */
+
+	if (delstate->bottomup)
+	{
+		/*
+		 * Ignore any bottom-up deletion requests.
+		 *
+		 * Currently only caller in postgres that does bottom-up deletion is
+		 * _bt_bottomupdel_pass, which in turn calls _bt_delitems_delete_check.
+		 * And this function is okay with ndeltids being set to 0 by tableAM
+		 * for bottom-up deletion.
+		 */
+		delstate->ndeltids = 0;
+		return InvalidTransactionId;
+	}
+	else
+	{
+		/*
+		 * TableAM is not expected to set ndeltids to 0 for simple deletion
+		 * case, so here we cannot do the same trick that we do for
+		 * bottom-up deletion.
+		 * See the assertion around table_index_delete_tuples call in pg
+		 * function index_compute_xid_horizon_for_tuples.
+		 *
+		 * For this reason, to avoid receiving simple deletion requests for
+		 * columnar tables (bottomup = false), columnar_index_fetch_tuple
+		 * doesn't ever set all_dead to true in order to prevent triggering
+		 * simple deletion of index tuples. But let's throw an error to be on
+		 * the safe side.
+		 */
+		elog(ERROR, "columnar_index_delete_tuples not implemented for simple deletion");
+	}
 }
 
 
@@ -670,6 +718,8 @@ static void
 columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 					  int options, BulkInsertState bistate)
 {
+	CheckCitusVersion(ERROR);
+
 	/*
 	 * columnar_init_write_state allocates the write state in a longer
 	 * lasting context, so no need to worry about it.
@@ -716,6 +766,8 @@ static void
 columnar_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 					  CommandId cid, int options, BulkInsertState bistate)
 {
+	CheckCitusVersion(ERROR);
+
 	ColumnarWriteState *writeState = columnar_init_write_state(relation,
 															   RelationGetDescr(relation),
 															   GetCurrentSubTransactionId());
@@ -790,6 +842,8 @@ columnar_relation_set_new_filenode(Relation rel,
 								   TransactionId *freezeXid,
 								   MultiXactId *minmulti)
 {
+	CheckCitusVersion(ERROR);
+
 	if (persistence == RELPERSISTENCE_UNLOGGED)
 	{
 		ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -825,6 +879,8 @@ columnar_relation_set_new_filenode(Relation rel,
 static void
 columnar_relation_nontransactional_truncate(Relation rel)
 {
+	CheckCitusVersion(ERROR);
+
 	RelFileNode relfilenode = rel->rd_node;
 
 	NonTransactionDropWriteState(relfilenode.relNode);
@@ -871,6 +927,8 @@ columnar_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
 								   double *tups_vacuumed,
 								   double *tups_recently_dead)
 {
+	CheckCitusVersion(ERROR);
+
 	TupleDesc sourceDesc = RelationGetDescr(OldHeap);
 	TupleDesc targetDesc = RelationGetDescr(NewHeap);
 
@@ -960,6 +1018,27 @@ NeededColumnsList(TupleDesc tupdesc, Bitmapset *attr_needed)
 }
 
 
+/*
+ * ColumnarTableTupleCount returns the number of tuples that columnar
+ * table with relationId has by using stripe metadata.
+ */
+static uint64
+ColumnarTableTupleCount(Relation relation)
+{
+	List *stripeList = StripesForRelfilenode(relation->rd_node);
+	uint64 tupleCount = 0;
+
+	ListCell *lc = NULL;
+	foreach(lc, stripeList)
+	{
+		StripeMetadata *stripe = lfirst(lc);
+		tupleCount += stripe->rowCount;
+	}
+
+	return tupleCount;
+}
+
+
 /*
  * columnar_vacuum_rel implements VACUUM without FULL option.
  */
@@ -967,6 +1046,18 @@ static void
 columnar_vacuum_rel(Relation rel, VacuumParams *params,
 					BufferAccessStrategy bstrategy)
 {
+	if (!CheckCitusVersion(WARNING))
+	{
+		/*
+		 * Skip if the extension catalogs are not up-to-date, but avoid
+		 * erroring during auto-vacuum.
+		 */
+		return;
+	}
+
+	pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM,
+								  RelationGetRelid(rel));
+
 	/*
 	 * If metapage version of relation is older, then we hint users to VACUUM
 	 * the relation in ColumnarMetapageCheckVersion. So if needed, upgrade
@@ -990,6 +1081,79 @@ columnar_vacuum_rel(Relation rel, VacuumParams *params,
 	{
 		TruncateColumnar(rel, elevel);
 	}
+
+	RelationOpenSmgr(rel);
+	BlockNumber new_rel_pages = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
+
+	/* get the number of indexes */
+	List *indexList = RelationGetIndexList(rel);
+	int nindexes = list_length(indexList);
+
+	TransactionId oldestXmin;
+	TransactionId freezeLimit;
+	MultiXactId multiXactCutoff;
+
+	/* initialize xids */
+#if PG_VERSION_NUM >= PG_VERSION_15
+	MultiXactId oldestMxact;
+	vacuum_set_xid_limits(rel,
+						  params->freeze_min_age,
+						  params->freeze_table_age,
+						  params->multixact_freeze_min_age,
+						  params->multixact_freeze_table_age,
+						  &oldestXmin, &oldestMxact,
+						  &freezeLimit, &multiXactCutoff);
+
+	Assert(MultiXactIdPrecedesOrEquals(multiXactCutoff, oldestMxact));
+#else
+	TransactionId xidFullScanLimit;
+	MultiXactId mxactFullScanLimit;
+	vacuum_set_xid_limits(rel,
+						  params->freeze_min_age,
+						  params->freeze_table_age,
+						  params->multixact_freeze_min_age,
+						  params->multixact_freeze_table_age,
+						  &oldestXmin, &freezeLimit, &xidFullScanLimit,
+						  &multiXactCutoff, &mxactFullScanLimit);
+#endif
+
+	Assert(TransactionIdPrecedesOrEquals(freezeLimit, oldestXmin));
+
+	/*
+	 * Columnar storage doesn't hold any transaction IDs, so we can always
+	 * just advance to the most aggressive value.
+	 */
+	TransactionId newRelFrozenXid = oldestXmin;
+#if PG_VERSION_NUM >= PG_VERSION_15
+	MultiXactId newRelminMxid = oldestMxact;
+#else
+	MultiXactId newRelminMxid = multiXactCutoff;
+#endif
+
+	double new_live_tuples = ColumnarTableTupleCount(rel);
+
+	/* all visible pages are always 0 */
+	BlockNumber new_rel_allvisible = 0;
+
+#if PG_VERSION_NUM >= PG_VERSION_15
+	bool frozenxid_updated;
+	bool minmulti_updated;
+
+	vac_update_relstats(rel, new_rel_pages, new_live_tuples,
+						new_rel_allvisible, nindexes > 0,
+						newRelFrozenXid, newRelminMxid,
+						&frozenxid_updated, &minmulti_updated, false);
+#else
+	vac_update_relstats(rel, new_rel_pages, new_live_tuples,
+						new_rel_allvisible, nindexes > 0,
+						newRelFrozenXid, newRelminMxid, false);
+#endif
+
+	pgstat_report_vacuum(RelationGetRelid(rel),
+						 rel->rd_rel->relisshared,
+						 Max(new_live_tuples, 0),
+						 0);
+	pgstat_progress_end_command();
 }
 
 
@@ -1276,6 +1440,8 @@ columnar_index_build_range_scan(Relation columnarRelation,
 								void *callback_state,
 								TableScanDesc scan)
 {
+	CheckCitusVersion(ERROR);
+
 	if (start_blockno != 0 || numblocks != InvalidBlockNumber)
 	{
 		/*
@@ -1524,6 +1690,8 @@ columnar_index_validate_scan(Relation columnarRelation,
 							 ValidateIndexState *
 							 validateIndexState)
 {
+	CheckCitusVersion(ERROR);
+
 	ColumnarReportTotalVirtualBlocks(columnarRelation, snapshot,
 									 PROGRESS_SCAN_BLOCKS_TOTAL);
 
@@ -1694,6 +1862,8 @@ TupleSortSkipSmallerItemPointers(Tuplesortstate *tupleSort, ItemPointer targetIt
 static uint64
 columnar_relation_size(Relation rel, ForkNumber forkNumber)
 {
+	CheckCitusVersion(ERROR);
+
 	uint64 nblocks = 0;
 
 	/* Open it at the smgr level if not already done */
@@ -1719,6 +1889,8 @@ columnar_relation_size(Relation rel, ForkNumber forkNumber)
 static bool
 columnar_relation_needs_toast_table(Relation rel)
 {
+	CheckCitusVersion(ERROR);
+
 	return false;
 }
 
@@ -1728,6 +1900,8 @@ columnar_estimate_rel_size(Relation rel, int32 *attr_widths,
 						   BlockNumber *pages, double *tuples,
 						   double *allvisfrac)
 {
+	CheckCitusVersion(ERROR);
+
 	RelationOpenSmgr(rel);
 	*pages = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
 	*tuples = ColumnarTableRowCount(rel);
@@ -1899,6 +2073,8 @@ ColumnarTableDropHook(Oid relid)
 
 	if (IsColumnarTableAmTable(relid))
 	{
+		CheckCitusVersion(ERROR);
+
 		/*
 		 * Drop metadata. No need to drop storage here since for
 		 * tableam tables storage is managed by postgres.
@@ -2020,6 +2196,8 @@ ColumnarProcessUtility(PlannedStmt *pstmt,
 									   GetCreateIndexRelationLockMode(indexStmt));
 		if (rel->rd_tableam == GetColumnarTableAmRoutine())
 		{
+			CheckCitusVersion(ERROR);
+
 			if (!ColumnarSupportsIndexAM(indexStmt->accessMethod))
 			{
 				ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -2356,6 +2534,8 @@ PG_FUNCTION_INFO_V1(alter_columnar_table_set);
 Datum
 alter_columnar_table_set(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	Oid relationId = PG_GETARG_OID(0);
 
 	Relation rel = table_open(relationId, AccessExclusiveLock); /* ALTER TABLE LOCK */
@@ -2483,6 +2663,8 @@ PG_FUNCTION_INFO_V1(alter_columnar_table_reset);
 Datum
 alter_columnar_table_reset(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	Oid relationId = PG_GETARG_OID(0);
 
 	Relation rel = table_open(relationId, AccessExclusiveLock); /* ALTER TABLE LOCK */

src/backend/columnar/sql/columnar--10.2-1--10.2-2.sql

@@ -0,0 +1,5 @@
+-- columnar--10.2-1--10.2-2.sql
+
+-- revoke read access for columnar.chunk from unprivileged
+-- user as it contains chunk min/max values
+REVOKE SELECT ON columnar.chunk FROM PUBLIC;

src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql

@@ -0,0 +1,15 @@
+-- columnar--10.2-2--10.2-3.sql
+
+-- Since stripe_first_row_number_idx is required to scan a columnar table, we
+-- need to make sure that it is created before doing anything with columnar
+-- tables during pg upgrades.
+--
+-- However, a plain btree index is not a dependency of a table, so pg_upgrade
+-- cannot guarantee that stripe_first_row_number_idx gets created when
+-- creating columnar.stripe, unless we make it a unique "constraint".
+--
+-- To do that, drop stripe_first_row_number_idx and create a unique
+-- constraint with the same name to keep the code change at minimum.
+DROP INDEX columnar.stripe_first_row_number_idx;
+ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx
+UNIQUE (storage_id, first_row_number);

src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql

@@ -0,0 +1,5 @@
+-- columnar--10.2-3--10.2-4.sql
+
+#include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql"
+
+SELECT citus_internal.columnar_ensure_am_depends_catalog();

src/backend/columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql

@@ -0,0 +1,4 @@
+-- columnar--10.2-2--10.2-1.sql
+
+-- grant read access for columnar.chunk to unprivileged user
+GRANT SELECT ON columnar.chunk TO PUBLIC;

src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql

@@ -0,0 +1,4 @@
+-- columnar--10.2-3--10.2-2.sql
+
+ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx;
+CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number);

src/backend/columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql

@@ -0,0 +1,6 @@
+-- columnar--10.2-4--10.2-3.sql
+
+DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog();
+
+-- Note that we intentionally do not delete pg_depend records that we inserted
+-- via columnar--10.2-3--10.2-4.sql (by using columnar_ensure_am_depends_catalog).

src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/10.2-4.sql

@@ -0,0 +1,40 @@
+CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
+  RETURNS void
+  LANGUAGE plpgsql
+  SET search_path = pg_catalog
+AS $func$
+BEGIN
+  INSERT INTO pg_depend
+  SELECT -- Define a dependency edge from "columnar table access method" ..
+         'pg_am'::regclass::oid as classid,
+         (select oid from pg_am where amname = 'columnar') as objid,
+         0 as objsubid,
+         -- ... to each object that is registered to pg_class and that lives
+         -- in "columnar" schema. That contains catalog tables, indexes
+         -- created on them and the sequences created in "columnar" schema.
+         --
+         -- Given the possibility of user might have created their own objects
+         -- in columnar schema, we explicitly specify list of objects that we
+         -- are interested in.
+         'pg_class'::regclass::oid as refclassid,
+         columnar_schema_members.relname::regclass::oid as refobjid,
+         0 as refobjsubid,
+         'n' as deptype
+  FROM (VALUES ('columnar.chunk'),
+               ('columnar.chunk_group'),
+               ('columnar.chunk_group_pkey'),
+               ('columnar.chunk_pkey'),
+               ('columnar.options'),
+               ('columnar.options_pkey'),
+               ('columnar.storageid_seq'),
+               ('columnar.stripe'),
+               ('columnar.stripe_first_row_number_idx'),
+               ('columnar.stripe_pkey')
+       ) columnar_schema_members(relname)
+  -- Avoid inserting duplicate entries into pg_depend.
+  EXCEPT TABLE pg_depend;
+END;
+$func$;
+COMMENT ON FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
+  IS 'internal function responsible for creating dependencies from columnar '
+     'table access method to the rel objects in columnar schema';

src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/latest.sql

@@ -0,0 +1,40 @@
+CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
+  RETURNS void
+  LANGUAGE plpgsql
+  SET search_path = pg_catalog
+AS $func$
+BEGIN
+  INSERT INTO pg_depend
+  SELECT -- Define a dependency edge from "columnar table access method" ..
+         'pg_am'::regclass::oid as classid,
+         (select oid from pg_am where amname = 'columnar') as objid,
+         0 as objsubid,
+         -- ... to each object that is registered to pg_class and that lives
+         -- in "columnar" schema. That contains catalog tables, indexes
+         -- created on them and the sequences created in "columnar" schema.
+         --
+         -- Given the possibility of user might have created their own objects
+         -- in columnar schema, we explicitly specify list of objects that we
+         -- are interested in.
+         'pg_class'::regclass::oid as refclassid,
+         columnar_schema_members.relname::regclass::oid as refobjid,
+         0 as refobjsubid,
+         'n' as deptype
+  FROM (VALUES ('columnar.chunk'),
+               ('columnar.chunk_group'),
+               ('columnar.chunk_group_pkey'),
+               ('columnar.chunk_pkey'),
+               ('columnar.options'),
+               ('columnar.options_pkey'),
+               ('columnar.storageid_seq'),
+               ('columnar.stripe'),
+               ('columnar.stripe_first_row_number_idx'),
+               ('columnar.stripe_pkey')
+       ) columnar_schema_members(relname)
+  -- Avoid inserting duplicate entries into pg_depend.
+  EXCEPT TABLE pg_depend;
+END;
+$func$;
+COMMENT ON FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
+  IS 'internal function responsible for creating dependencies from columnar '
+     'table access method to the rel objects in columnar schema';

src/backend/distributed/citus.control

@@ -1,6 +1,6 @@
 # Citus extension
 comment = 'Citus distributed database'
-default_version = '10.2-1'
+default_version = '10.2-5'
 module_pathname = '$libdir/citus'
 relocatable = false
 schema = pg_catalog

src/backend/distributed/commands/alter_table.c

@@ -32,6 +32,8 @@
 #include "access/xact.h"
 #include "catalog/dependency.h"
 #include "catalog/pg_am.h"
+#include "catalog/pg_depend.h"
+#include "catalog/pg_rewrite_d.h"
 #include "columnar/columnar.h"
 #include "columnar/columnar_tableam.h"
 #include "commands/defrem.h"
@@ -209,6 +211,8 @@ static char * GetAccessMethodForMatViewIfExists(Oid viewOid);
 static bool WillRecreateForeignKeyToReferenceTable(Oid relationId,
 												   CascadeToColocatedOption cascadeOption);
 static void WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId);
+static void ErrorIfUnsupportedCascadeObjects(Oid relationId);
+static bool DoesCascadeDropUnsupportedObject(Oid classId, Oid id, HTAB *nodeMap);
 
 PG_FUNCTION_INFO_V1(undistribute_table);
 PG_FUNCTION_INFO_V1(alter_distributed_table);
@@ -385,6 +389,8 @@ UndistributeTable(TableConversionParameters *params)
 		ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(partitionList);
 	}
 
+	ErrorIfUnsupportedCascadeObjects(params->relationId);
+
 	params->conversionType = UNDISTRIBUTE_TABLE;
 	params->shardCountIsNull = true;
 	TableConversionState *con = CreateTableConversion(params);
@@ -416,6 +422,8 @@ AlterDistributedTable(TableConversionParameters *params)
 	EnsureTableNotPartition(params->relationId);
 	EnsureHashDistributedTable(params->relationId);
 
+	ErrorIfUnsupportedCascadeObjects(params->relationId);
+
 	params->conversionType = ALTER_DISTRIBUTED_TABLE;
 	TableConversionState *con = CreateTableConversion(params);
 	CheckAlterDistributedTableConversionParameters(con);
@@ -472,6 +480,8 @@ AlterTableSetAccessMethod(TableConversionParameters *params)
 		}
 	}
 
+	ErrorIfUnsupportedCascadeObjects(params->relationId);
+
 	params->conversionType = ALTER_TABLE_SET_ACCESS_METHOD;
 	params->shardCountIsNull = true;
 	TableConversionState *con = CreateTableConversion(params);
@@ -1234,6 +1244,94 @@ CreateCitusTableLike(TableConversionState *con)
 }
 
 
+/*
+ * ErrorIfUnsupportedCascadeObjects gets oid of a relation, finds the objects
+ * that dropping this relation cascades into and errors if there are any extensions
+ * that would be dropped.
+ */
+static void
+ErrorIfUnsupportedCascadeObjects(Oid relationId)
+{
+	HASHCTL info;
+	memset(&info, 0, sizeof(info));
+	info.keysize = sizeof(Oid);
+	info.entrysize = sizeof(Oid);
+	info.hash = oid_hash;
+	uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION);
+	HTAB *nodeMap = hash_create("object dependency map (oid)", 64, &info, hashFlags);
+
+	bool unsupportedObjectInDepGraph =
+		DoesCascadeDropUnsupportedObject(RelationRelationId, relationId, nodeMap);
+
+	if (unsupportedObjectInDepGraph)
+	{
+		ereport(ERROR, (errmsg("cannot alter table because an extension depends on it")));
+	}
+}
+
+
+/*
+ * DoesCascadeDropUnsupportedObject walks through the objects that depend on the
+ * object with object id and returns true if it finds any unsupported objects.
+ *
+ * This function only checks extensions as unsupported objects.
+ *
+ * Extension dependency is different than the rest. If an object depends on an extension
+ * dropping the object would drop the extension too.
+ * So we check with IsObjectAddressOwnedByExtension function.
+ */
+static bool
+DoesCascadeDropUnsupportedObject(Oid classId, Oid objectId, HTAB *nodeMap)
+{
+	bool found = false;
+	hash_search(nodeMap, &objectId, HASH_ENTER, &found);
+
+	if (found)
+	{
+		return false;
+	}
+
+	ObjectAddress objectAddress = { 0 };
+	ObjectAddressSet(objectAddress, classId, objectId);
+
+	if (IsObjectAddressOwnedByExtension(&objectAddress, NULL))
+	{
+		return true;
+	}
+
+	Oid targetObjectClassId = classId;
+	Oid targetObjectId = objectId;
+	List *dependencyTupleList = GetPgDependTuplesForDependingObjects(targetObjectClassId,
+																	 targetObjectId);
+
+	HeapTuple depTup = NULL;
+	foreach_ptr(depTup, dependencyTupleList)
+	{
+		Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup);
+
+		Oid dependingOid = InvalidOid;
+		Oid dependingClassId = InvalidOid;
+
+		if (pg_depend->classid == RewriteRelationId)
+		{
+			dependingOid = GetDependingView(pg_depend);
+			dependingClassId = RelationRelationId;
+		}
+		else
+		{
+			dependingOid = pg_depend->objid;
+			dependingClassId = pg_depend->classid;
+		}
+
+		if (DoesCascadeDropUnsupportedObject(dependingClassId, dependingOid, nodeMap))
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+
 /*
  * GetViewCreationCommandsOfTable takes a table oid generates the CREATE VIEW
  * commands for views that depend to the given table. This includes the views

src/backend/distributed/commands/create_distributed_table.c

@@ -31,6 +31,7 @@
 #include "catalog/pg_opclass.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_trigger.h"
+#include "catalog/pg_type.h"
 #include "commands/defrem.h"
 #include "commands/extension.h"
 #include "commands/sequence.h"
@@ -58,6 +59,7 @@
 #include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
 #include "distributed/remote_commands.h"
+#include "distributed/resource_lock.h"
 #include "distributed/shared_library_init.h"
 #include "distributed/worker_protocol.h"
 #include "distributed/worker_transaction.h"
@@ -474,9 +476,22 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	/*
 	 * Make sure that existing reference tables have been replicated to all the nodes
 	 * such that we can create foreign keys and joins work immediately after creation.
+	 *
+	 * This will take a lock on the nodes to make sure no nodes are added after we have
+	 * verified and ensured the reference tables are copied everywhere.
+	 * Although copying reference tables here for anything but creating a new colocation
+	 * group, it requires significant refactoring which we don't want to perform now.
 	 */
 	EnsureReferenceTablesExistOnAllNodes();
 
+	/*
+	 * While adding tables to a colocation group we need to make sure no concurrent
+	 * mutations happen on the colocation group with regards to its placements. It is
+	 * important that we have already copied any reference tables before acquiring this
+	 * lock as these are competing operations.
+	 */
+	LockColocationId(colocationId, ShareLock);
+
 	/* we need to calculate these variables before creating distributed metadata */
 	bool localTableEmpty = TableEmpty(relationId);
 	Oid colocatedTableId = ColocatedTableId(colocationId);
@@ -554,11 +569,16 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	{
 		List *partitionList = PartitionList(relationId);
 		Oid partitionRelationId = InvalidOid;
+		Oid namespaceId = get_rel_namespace(relationId);
+		char *schemaName = get_namespace_name(namespaceId);
+		char *relationName = get_rel_name(relationId);
+		char *parentRelationName = quote_qualified_identifier(schemaName, relationName);
+
 		foreach_oid(partitionRelationId, partitionList)
 		{
 			CreateDistributedTable(partitionRelationId, distributionColumn,
 								   distributionMethod, shardCount, false,
-								   colocateWithTableName, viaDeprecatedAPI);
+								   parentRelationName, viaDeprecatedAPI);
 		}
 	}
 
@@ -592,7 +612,7 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
  * Otherwise, the condition is ensured.
  */
 void
-EnsureSequenceTypeSupported(Oid seqOid, Oid seqTypId)
+EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId)
 {
 	List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE);
 	Oid citusTableId = InvalidOid;
@@ -617,9 +637,9 @@ EnsureSequenceTypeSupported(Oid seqOid, Oid seqTypId)
 			 */
 			if (currentSeqOid == seqOid)
 			{
-				Oid currentSeqTypId = GetAttributeTypeOid(citusTableId,
+				Oid currentAttributeTypId = GetAttributeTypeOid(citusTableId,
-														  currentAttnum);
+																currentAttnum);
-				if (seqTypId != currentSeqTypId)
+				if (attributeTypeId != currentAttributeTypId)
 				{
 					char *sequenceName = generate_qualified_relation_name(
 						seqOid);
@@ -711,17 +731,29 @@ EnsureDistributedSequencesHaveOneType(Oid relationId, List *dependentSequenceLis
 		 * We should make sure that the type of the column that uses
 		 * that sequence is supported
 		 */
-		Oid seqTypId = GetAttributeTypeOid(relationId, attnum);
+		Oid attributeTypeId = GetAttributeTypeOid(relationId, attnum);
-		EnsureSequenceTypeSupported(sequenceOid, seqTypId);
+		EnsureSequenceTypeSupported(sequenceOid, attributeTypeId);
 
 		/*
 		 * Alter the sequence's data type in the coordinator if needed.
+		 *
+		 * First, we should only change the sequence type if the column
+		 * is a supported sequence type. For example, if a sequence is used
+		 * in an expression which then becomes a text, we should not try to
+		 * alter the sequence type to text. Postgres only supports int2, int4
+		 * and int8 as the sequence type.
+		 *
 		 * A sequence's type is bigint by default and it doesn't change even if
 		 * it's used in an int column. We should change the type if needed,
 		 * and not allow future ALTER SEQUENCE ... TYPE ... commands for
-		 * sequences used as defaults in distributed tables
+		 * sequences used as defaults in distributed tables.
 		 */
-		AlterSequenceType(sequenceOid, seqTypId);
+		if (attributeTypeId == INT2OID ||
+			attributeTypeId == INT4OID ||
+			attributeTypeId == INT8OID)
+		{
+			AlterSequenceType(sequenceOid, attributeTypeId);
+		}
 	}
 }
 

src/backend/distributed/commands/extension.c

@@ -149,16 +149,6 @@ PostprocessCreateExtensionStmt(Node *node, const char *queryString)
 	/* extension management can only be done via coordinator node */
 	EnsureCoordinator();
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 * This guarantees that all active nodes will have the extension, because they will
-	 * either get it now, or get it in citus_add_node after this transaction finishes and
-	 * the pg_dist_object record becomes visible.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	/*
 	 * Make sure that the current transaction is already in sequential mode,
 	 * or can still safely be put in sequential mode
@@ -262,16 +252,6 @@ PreprocessDropExtensionStmt(Node *node, const char *queryString,
 	/* extension management can only be done via coordinator node */
 	EnsureCoordinator();
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 * This guarantees that all active nodes will drop the extension, because they will
-	 * either get it now, or get it in citus_add_node after this transaction finishes and
-	 * the pg_dist_object record becomes visible.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	/*
 	 * Make sure that the current transaction is already in sequential mode,
 	 * or can still safely be put in sequential mode
@@ -398,15 +378,6 @@ PreprocessAlterExtensionSchemaStmt(Node *node, const char *queryString,
 	/* extension management can only be done via coordinator node */
 	EnsureCoordinator();
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 * This guarantees that all active nodes will update the extension schema after
-	 * this transaction finishes and the pg_dist_object record becomes visible.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	/*
 	 * Make sure that the current transaction is already in sequential mode,
 	 * or can still safely be put in sequential mode
@@ -466,16 +437,6 @@ PreprocessAlterExtensionUpdateStmt(Node *node, const char *queryString,
 	/* extension management can only be done via coordinator node */
 	EnsureCoordinator();
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 * This guarantees that all active nodes will update the extension version, because
-	 * they will either get it now, or get it in citus_add_node after this transaction
-	 * finishes and the pg_dist_object record becomes visible.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	/*
 	 * Make sure that the current transaction is already in sequential mode,
 	 * or can still safely be put in sequential mode

src/backend/distributed/commands/function.c

@@ -83,6 +83,7 @@ static void EnsureSequentialModeForFunctionDDL(void);
 static void TriggerSyncMetadataToPrimaryNodes(void);
 static bool ShouldPropagateCreateFunction(CreateFunctionStmt *stmt);
 static bool ShouldPropagateAlterFunction(const ObjectAddress *address);
+static bool ShouldAddFunctionSignature(FunctionParameterMode mode);
 static ObjectAddress FunctionToObjectAddress(ObjectType objectType,
 											 ObjectWithArgs *objectWithArgs,
 											 bool missing_ok);
@@ -1298,7 +1299,11 @@ CreateFunctionStmtObjectAddress(Node *node, bool missing_ok)
 	FunctionParameter *funcParam = NULL;
 	foreach_ptr(funcParam, stmt->parameters)
 	{
-		objectWithArgs->objargs = lappend(objectWithArgs->objargs, funcParam->argType);
+		if (ShouldAddFunctionSignature(funcParam->mode))
+		{
+			objectWithArgs->objargs = lappend(objectWithArgs->objargs,
+											  funcParam->argType);
+		}
 	}
 
 	return FunctionToObjectAddress(objectType, objectWithArgs, missing_ok);
@@ -1855,8 +1860,7 @@ ObjectWithArgsFromOid(Oid funcOid)
 
 	for (int i = 0; i < numargs; i++)
 	{
-		if (argModes == NULL ||
+		if (argModes == NULL || ShouldAddFunctionSignature(argModes[i]))
-			argModes[i] != PROARGMODE_OUT || argModes[i] != PROARGMODE_TABLE)
 		{
 			objargs = lappend(objargs, makeTypeNameFromOid(argTypes[i], -1));
 		}
@@ -1869,6 +1873,35 @@ ObjectWithArgsFromOid(Oid funcOid)
 }
 
 
+/*
+ * ShouldAddFunctionSignature takes a FunctionParameterMode and returns true if it should
+ * be included in the function signature. Returns false otherwise.
+ */
+static bool
+ShouldAddFunctionSignature(FunctionParameterMode mode)
+{
+	/* only input parameters should be added to the generated signature */
+	switch (mode)
+	{
+		case FUNC_PARAM_IN:
+		case FUNC_PARAM_INOUT:
+		case FUNC_PARAM_VARIADIC:
+		{
+			return true;
+		}
+
+		case FUNC_PARAM_OUT:
+		case FUNC_PARAM_TABLE:
+		{
+			return false;
+		}
+
+		default:
+			return true;
+	}
+}
+
+
 /*
  * FunctionToObjectAddress returns the ObjectAddress of a Function or Procedure based on
  * its type and ObjectWithArgs describing the Function/Procedure. If missing_ok is set to

src/backend/distributed/commands/role.c

@@ -140,13 +140,6 @@ PostprocessAlterRoleStmt(Node *node, const char *queryString)
 
 	AlterRoleStmt *stmt = castNode(AlterRoleStmt, node);
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	DefElem *option = NULL;
 	foreach_ptr(option, stmt->options)
 	{

src/backend/distributed/commands/table.c

@@ -143,11 +143,14 @@ PreprocessDropTableStmt(Node *node, const char *queryString,
 			continue;
 		}
 
-		if (IsCitusTableType(relationId, REFERENCE_TABLE))
+		/*
+		 * While changing the tables that are part of a colocation group we need to
+		 * prevent concurrent mutations to the placements of the shard groups.
+		 */
+		CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId);
+		if (cacheEntry->colocationId != INVALID_COLOCATION_ID)
 		{
-			/* prevent concurrent EnsureReferenceTablesExistOnAllNodes */
+			LockColocationId(cacheEntry->colocationId, ShareLock);
-			int colocationId = CreateReferenceTableColocationId();
-			LockColocationId(colocationId, ExclusiveLock);
 		}
 
 		/* invalidate foreign key cache if the table involved in any foreign key */

src/backend/distributed/commands/type.c

@@ -130,16 +130,6 @@ PreprocessCompositeTypeStmt(Node *node, const char *queryString,
 	 */
 	EnsureCoordinator();
 
-	/*
-	 * Make sure that no new nodes are added after this point until the end of the
-	 * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the
-	 * ExclusiveLock taken by citus_add_node.
-	 * This guarantees that all active nodes will have the object, because they will
-	 * either get it now, or get it in citus_add_node after this transaction finishes and
-	 * the pg_dist_object record becomes visible.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	/* fully qualify before lookup and later deparsing */
 	QualifyTreeNode(node);
 

src/backend/distributed/commands/utility_hook.c

@@ -33,7 +33,9 @@
 #include "access/attnum.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
+#if PG_VERSION_NUM < 140000
 #include "access/xact.h"
+#endif
 #include "catalog/catalog.h"
 #include "catalog/dependency.h"
 #include "commands/dbcommands.h"
@@ -51,7 +53,9 @@
 #include "distributed/local_executor.h"
 #include "distributed/maintenanced.h"
 #include "distributed/coordinator_protocol.h"
+#if PG_VERSION_NUM < 140000
 #include "distributed/metadata_cache.h"
+#endif
 #include "distributed/metadata_sync.h"
 #include "distributed/multi_executor.h"
 #include "distributed/multi_explain.h"
@@ -67,6 +71,7 @@
 #include "tcop/utility.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
+#include "utils/snapmgr.h"
 #include "utils/syscache.h"
 
 bool EnableDDLPropagation = true; /* ddl propagation is enabled */
@@ -88,6 +93,9 @@ static void ProcessUtilityInternal(PlannedStmt *pstmt,
 								   struct QueryEnvironment *queryEnv,
 								   DestReceiver *dest,
 								   QueryCompletionCompat *completionTag);
+#if PG_VERSION_NUM >= 140000
+static void set_indexsafe_procflags(void);
+#endif
 static char * SetSearchPathToCurrentSearchPathCommand(void);
 static char * CurrentSearchPath(void);
 static void IncrementUtilityHookCountersIfNecessary(Node *parsetree);
@@ -906,9 +914,35 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 		/*
 		 * Start a new transaction to make sure CONCURRENTLY commands
 		 * on localhost do not block waiting for this transaction to finish.
+		 *
+		 * In addition to doing that, we also need to tell other backends
+		 * --including the ones spawned for connections opened to localhost to
+		 * build indexes on shards of this relation-- that concurrent index
+		 * builds can safely ignore us.
+		 *
+		 * Normally, DefineIndex() only does that if index doesn't have any
+		 * predicates (i.e.: where clause) and no index expressions at all.
+		 * However, now that we already called standard process utility,
+		 * index build on the shell table is finished anyway.
+		 *
+		 * The reason behind doing so is that we cannot guarantee not
+		 * grabbing any snapshots via adaptive executor, and the backends
+		 * creating indexes on local shards (if any) might block on waiting
+		 * for current xact of the current backend to finish, which would
+		 * cause self deadlocks that are not detectable.
 		 */
 		if (ddlJob->startNewTransaction)
 		{
+#if PG_VERSION_NUM < 140000
+
+			/*
+			 * Older versions of postgres doesn't have PROC_IN_SAFE_IC flag
+			 * so we cannot use set_indexsafe_procflags in those versions.
+			 *
+			 * For this reason, we do our best to ensure not grabbing any
+			 * snapshots later in the executor.
+			 */
+
 			/*
 			 * If cache is not populated, system catalog lookups will cause
 			 * the xmin of current backend to change. Then the last phase
@@ -929,8 +963,34 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 			 * will already be in the hash table, hence we won't be holding any snapshots.
 			 */
 			WarmUpConnParamsHash();
+#endif
+
+			/*
+			 * Since it is not certain whether the code-path that we followed
+			 * until reaching here caused grabbing any snapshots or not, we
+			 * need to pop the active snapshot if we had any, to ensure not
+			 * leaking any snapshots.
+			 *
+			 * For example, EnsureCoordinator might return without grabbing
+			 * any snapshots if we didn't receive any invalidation messages
+			 * but the otherwise is also possible.
+			 */
+			if (ActiveSnapshotSet())
+			{
+				PopActiveSnapshot();
+			}
+
 			CommitTransactionCommand();
 			StartTransactionCommand();
+
+#if PG_VERSION_NUM >= 140000
+
+			/*
+			 * Tell other backends to ignore us, even if we grab any
+			 * snapshots via adaptive executor.
+			 */
+			set_indexsafe_procflags();
+#endif
 		}
 
 		/* save old commit protocol to restore at xact end */
@@ -997,6 +1057,33 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob)
 }
 
 
+#if PG_VERSION_NUM >= 140000
+
+/*
+ * set_indexsafe_procflags sets PROC_IN_SAFE_IC flag in MyProc->statusFlags.
+ *
+ * The flag is reset automatically at transaction end, so it must be set
+ * for each transaction.
+ *
+ * Copied from pg/src/backend/commands/indexcmds.c
+ * Also see pg commit c98763bf51bf610b3ee7e209fc76c3ff9a6b3163.
+ */
+static void
+set_indexsafe_procflags(void)
+{
+	Assert(MyProc->xid == InvalidTransactionId &&
+		   MyProc->xmin == InvalidTransactionId);
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	MyProc->statusFlags |= PROC_IN_SAFE_IC;
+	ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags;
+	LWLockRelease(ProcArrayLock);
+}
+
+
+#endif
+
+
 /*
  * CreateCustomDDLTaskList creates a DDLJob which will apply a command to all placements
  * of shards of a distributed table. The command to be applied is generated by the
@@ -1260,7 +1347,8 @@ DDLTaskList(Oid relationId, const char *commandString)
 List *
 NodeDDLTaskList(TargetWorkerSet targets, List *commands)
 {
-	List *workerNodes = TargetWorkerSetNodeList(targets, NoLock);
+	/* don't allow concurrent node list changes that require an exclusive lock */
+	List *workerNodes = TargetWorkerSetNodeList(targets, RowShareLock);
 
 	if (list_length(workerNodes) <= 0)
 	{

src/backend/distributed/connection/connection_configuration.c

@@ -72,8 +72,8 @@ InitConnParams()
 /*
  * ResetConnParams frees all strings in the keywords and parameters arrays,
  * sets their elements to null, and resets the ConnParamsSize to zero before
- * adding back any hardcoded global connection settings (at present, only the
+ * adding back any hardcoded global connection settings (at present, there
- * fallback_application_name of 'citus').
+ * are no).
  */
 void
 ResetConnParams()
@@ -89,8 +89,6 @@ ResetConnParams()
 	ConnParams.size = 0;
 
 	InvalidateConnParamsHashEntries();
-
-	AddConnParam("fallback_application_name", CITUS_APPLICATION_NAME);
 }
 
 
@@ -253,14 +251,16 @@ GetConnParams(ConnectionHashKey *key, char ***keywords, char ***values,
 		"port",
 		"dbname",
 		"user",
-		"client_encoding"
+		"client_encoding",
+		"application_name"
 	};
 	const char *runtimeValues[] = {
 		key->hostname,
 		nodePortString,
 		key->database,
 		key->user,
-		GetDatabaseEncodingName()
+		GetDatabaseEncodingName(),
+		CITUS_APPLICATION_NAME
 	};
 
 	/*

src/backend/distributed/connection/connection_management.c

@@ -36,6 +36,7 @@
 #include "distributed/version_compat.h"
 #include "distributed/worker_log_messages.h"
 #include "mb/pg_wchar.h"
+#include "pg_config.h"
 #include "portability/instr_time.h"
 #include "storage/ipc.h"
 #include "utils/hsearch.h"
@@ -1155,6 +1156,8 @@ StartConnectionEstablishment(MultiConnection *connection, ConnectionHashKey *key
 }
 
 
+#if PG_VERSION_NUM < 140000
+
 /*
  * WarmUpConnParamsHash warms up the ConnParamsHash by loading all the
  * conn params for active primary nodes.
@@ -1176,6 +1179,9 @@ WarmUpConnParamsHash(void)
 }
 
 
+#endif
+
+
 /*
  * FindOrCreateConnParamsEntry searches ConnParamsHash for the given key,
  * if it is not found, it is created.

src/backend/distributed/deparser/citus_ruleutils.c

@@ -48,6 +48,7 @@
 #include "distributed/metadata_cache.h"
 #include "distributed/metadata_sync.h"
 #include "distributed/metadata_utility.h"
+#include "distributed/namespace_utils.h"
 #include "distributed/relay_utility.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_protocol.h"
@@ -739,6 +740,12 @@ deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid,
 												relationName),
 					 indexStmt->accessMethod);
 
+	/*
+	 * Switch to empty search_path to deparse_index_columns to produce fully-
+	 * qualified names in expressions.
+	 */
+	PushOverrideEmptySearchPath(CurrentMemoryContext);
+
 	/* index column or expression list begins here */
 	appendStringInfoChar(buffer, '(');
 	deparse_index_columns(buffer, indexStmt->indexParams, deparseContext);
@@ -749,10 +756,15 @@ deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid,
 	{
 		appendStringInfoString(buffer, "INCLUDE (");
 		deparse_index_columns(buffer, indexStmt->indexIncludingParams, deparseContext);
-		appendStringInfoChar(buffer, ')');
+		appendStringInfoString(buffer, ") ");
 	}
 
-	AppendStorageParametersToString(buffer, indexStmt->options);
+	if (indexStmt->options != NIL)
+	{
+		appendStringInfoString(buffer, "WITH (");
+		AppendStorageParametersToString(buffer, indexStmt->options);
+		appendStringInfoString(buffer, ") ");
+	}
 
 	if (indexStmt->whereClause != NULL)
 	{
@@ -760,6 +772,9 @@ deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid,
 																deparseContext, false,
 																false));
 	}
+
+	/* revert back to original search_path */
+	PopOverrideSearchPath();
 }
 
 
@@ -948,8 +963,9 @@ deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparse
 		/* Commit on postgres: 911e70207703799605f5a0e8aad9f06cff067c63*/
 		if (indexElement->opclassopts != NIL)
 		{
-			ereport(ERROR, errmsg(
+			appendStringInfoString(buffer, "(");
-						"citus currently doesn't support operator class parameters in indexes"));
+			AppendStorageParametersToString(buffer, indexElement->opclassopts);
+			appendStringInfoString(buffer, ") ");
 		}
 #endif
 
@@ -1081,13 +1097,6 @@ AppendStorageParametersToString(StringInfo stringBuffer, List *optionList)
 	ListCell *optionCell = NULL;
 	bool firstOptionPrinted = false;
 
-	if (optionList == NIL)
-	{
-		return;
-	}
-
-	appendStringInfo(stringBuffer, " WITH (");
-
 	foreach(optionCell, optionList)
 	{
 		DefElem *option = (DefElem *) lfirst(optionCell);
@@ -1104,8 +1113,6 @@ AppendStorageParametersToString(StringInfo stringBuffer, List *optionList)
 						 quote_identifier(optionName),
 						 quote_literal_cstr(optionValue));
 	}
-
-	appendStringInfo(stringBuffer, ")");
 }
 
 

src/backend/distributed/executor/insert_select_executor.c

@@ -55,7 +55,6 @@
 bool EnableRepartitionedInsertSelect = true;
 
 
-static Query * WrapSubquery(Query *subquery);
 static List * TwoPhaseInsertSelectTaskList(Oid targetRelationId, Query *insertSelectQuery,
 										   char *resultIdPrefix);
 static void ExecutePlanIntoRelation(Oid targetRelationId, List *insertTargetList,
@@ -299,100 +298,6 @@ NonPushableInsertSelectExecScan(CustomScanState *node)
 }
 
 
-/*
- * BuildSelectForInsertSelect extracts the SELECT part from an INSERT...SELECT query.
- * If the INSERT...SELECT has CTEs then these are added to the resulting SELECT instead.
- */
-Query *
-BuildSelectForInsertSelect(Query *insertSelectQuery)
-{
-	RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery);
-	Query *selectQuery = selectRte->subquery;
-
-	/*
-	 * Wrap the SELECT as a subquery if the INSERT...SELECT has CTEs or the SELECT
-	 * has top-level set operations.
-	 *
-	 * We could simply wrap all queries, but that might create a subquery that is
-	 * not supported by the logical planner. Since the logical planner also does
-	 * not support CTEs and top-level set operations, we can wrap queries containing
-	 * those without breaking anything.
-	 */
-	if (list_length(insertSelectQuery->cteList) > 0)
-	{
-		selectQuery = WrapSubquery(selectRte->subquery);
-
-		/* copy CTEs from the INSERT ... SELECT statement into outer SELECT */
-		selectQuery->cteList = copyObject(insertSelectQuery->cteList);
-		selectQuery->hasModifyingCTE = insertSelectQuery->hasModifyingCTE;
-	}
-	else if (selectQuery->setOperations != NULL)
-	{
-		/* top-level set operations confuse the ReorderInsertSelectTargetLists logic */
-		selectQuery = WrapSubquery(selectRte->subquery);
-	}
-
-	return selectQuery;
-}
-
-
-/*
- * WrapSubquery wraps the given query as a subquery in a newly constructed
- * "SELECT * FROM (...subquery...) citus_insert_select_subquery" query.
- */
-static Query *
-WrapSubquery(Query *subquery)
-{
-	ParseState *pstate = make_parsestate(NULL);
-	List *newTargetList = NIL;
-
-	Query *outerQuery = makeNode(Query);
-	outerQuery->commandType = CMD_SELECT;
-
-	/* create range table entries */
-	Alias *selectAlias = makeAlias("citus_insert_select_subquery", NIL);
-	RangeTblEntry *newRangeTableEntry = RangeTableEntryFromNSItem(
-		addRangeTableEntryForSubquery(
-			pstate, subquery,
-			selectAlias, false, true));
-	outerQuery->rtable = list_make1(newRangeTableEntry);
-
-	/* set the FROM expression to the subquery */
-	RangeTblRef *newRangeTableRef = makeNode(RangeTblRef);
-	newRangeTableRef->rtindex = 1;
-	outerQuery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL);
-
-	/* create a target list that matches the SELECT */
-	TargetEntry *selectTargetEntry = NULL;
-	foreach_ptr(selectTargetEntry, subquery->targetList)
-	{
-		/* exactly 1 entry in FROM */
-		int indexInRangeTable = 1;
-
-		if (selectTargetEntry->resjunk)
-		{
-			continue;
-		}
-
-		Var *newSelectVar = makeVar(indexInRangeTable, selectTargetEntry->resno,
-									exprType((Node *) selectTargetEntry->expr),
-									exprTypmod((Node *) selectTargetEntry->expr),
-									exprCollation((Node *) selectTargetEntry->expr), 0);
-
-		TargetEntry *newSelectTargetEntry = makeTargetEntry((Expr *) newSelectVar,
-															selectTargetEntry->resno,
-															selectTargetEntry->resname,
-															selectTargetEntry->resjunk);
-
-		newTargetList = lappend(newTargetList, newSelectTargetEntry);
-	}
-
-	outerQuery->targetList = newTargetList;
-
-	return outerQuery;
-}
-
-
 /*
  * TwoPhaseInsertSelectTaskList generates a list of tasks for a query that
  * inserts into a target relation and selects from a set of co-located

src/backend/distributed/executor/intermediate_results.c

@@ -44,7 +44,7 @@
 #include "utils/syscache.h"
 
 
-static bool CreatedResultsDirectory = false;
+static List *CreatedResultsDirectories = NIL;
 
 
 /* CopyDestReceiver can be used to stream results into a distributed table */
@@ -593,26 +593,28 @@ CreateIntermediateResultsDirectory(void)
 {
 	char *resultDirectory = IntermediateResultsDirectory();
 
-	if (!CreatedResultsDirectory)
+	int makeOK = mkdir(resultDirectory, S_IRWXU);
+	if (makeOK != 0)
 	{
-		int makeOK = mkdir(resultDirectory, S_IRWXU);
+		if (errno == EEXIST)
-		if (makeOK != 0)
 		{
-			if (errno == EEXIST)
+			/* someone else beat us to it, that's ok */
-			{
+			return resultDirectory;
-				/* someone else beat us to it, that's ok */
-				return resultDirectory;
-			}
-
-			ereport(ERROR, (errcode_for_file_access(),
-							errmsg("could not create intermediate results directory "
-								   "\"%s\": %m",
-								   resultDirectory)));
 		}
 
-		CreatedResultsDirectory = true;
+		ereport(ERROR, (errcode_for_file_access(),
+						errmsg("could not create intermediate results directory "
+							   "\"%s\": %m",
+							   resultDirectory)));
 	}
 
+	MemoryContext oldContext = MemoryContextSwitchTo(TopTransactionContext);
+
+	CreatedResultsDirectories =
+		lappend(CreatedResultsDirectories, pstrdup(resultDirectory));
+
+	MemoryContextSwitchTo(oldContext);
+
 	return resultDirectory;
 }
 
@@ -692,13 +694,14 @@ IntermediateResultsDirectory(void)
 
 
 /*
- * RemoveIntermediateResultsDirectory removes the intermediate result directory
+ * RemoveIntermediateResultsDirectories removes the intermediate result directory
  * for the current distributed transaction, if any was created.
  */
 void
-RemoveIntermediateResultsDirectory(void)
+RemoveIntermediateResultsDirectories(void)
 {
-	if (CreatedResultsDirectory)
+	char *directoryElement = NULL;
+	foreach_ptr(directoryElement, CreatedResultsDirectories)
 	{
 		/*
 		 * The shared directory is renamed before deleting it. Otherwise it
@@ -707,7 +710,7 @@ RemoveIntermediateResultsDirectory(void)
 		 * that's not possible. The current PID is included in the new
 		 * filename, so there can be no collisions with other backends.
 		 */
-		char *sharedName = IntermediateResultsDirectory();
+		char *sharedName = directoryElement;
 		StringInfo privateName = makeStringInfo();
 		appendStringInfo(privateName, "%s.removed-by-%d", sharedName, MyProcPid);
 		if (rename(sharedName, privateName->data))
@@ -727,9 +730,12 @@ RemoveIntermediateResultsDirectory(void)
 		{
 			PathNameDeleteTemporaryDir(privateName->data);
 		}
-
-		CreatedResultsDirectory = false;
 	}
+
+	/* cleanup */
+	list_free_deep(CreatedResultsDirectories);
+
+	CreatedResultsDirectories = NIL;
 }
 
 

src/backend/distributed/metadata/dependency.c

@@ -157,7 +157,6 @@ static void ApplyAddToDependencyList(ObjectAddressCollector *collector,
 static List * ExpandCitusSupportedTypes(ObjectAddressCollector *collector,
 										ObjectAddress target);
 static ViewDependencyNode * BuildViewDependencyGraph(Oid relationId, HTAB *nodeMap);
-static Oid GetDependingView(Form_pg_depend pg_depend);
 
 
 /*

src/backend/distributed/metadata/metadata_sync.c

@@ -2061,6 +2061,8 @@ ShouldInitiateMetadataSync(bool *lockFailure)
 Datum
 citus_internal_add_partition_metadata(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	PG_ENSURE_ARGNOTNULL(0, "relation");
 	Oid relationId = PG_GETARG_OID(0);
 
@@ -2211,6 +2213,8 @@ EnsurePartitionMetadataIsSane(Oid relationId, char distributionMethod, int coloc
 Datum
 citus_internal_add_shard_metadata(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	PG_ENSURE_ARGNOTNULL(0, "relation");
 	Oid relationId = PG_GETARG_OID(0);
 
@@ -2426,6 +2430,8 @@ EnsureShardMetadataIsSane(Oid relationId, int64 shardId, char storageType,
 Datum
 citus_internal_add_placement_metadata(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	int64 shardId = PG_GETARG_INT64(0);
 	int32 shardState = PG_GETARG_INT32(1);
 	int64 shardLength = PG_GETARG_INT64(2);
@@ -2537,6 +2543,8 @@ ShouldSkipMetadataChecks(void)
 Datum
 citus_internal_update_placement_metadata(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	int64 shardId = PG_GETARG_INT64(0);
 	int32 sourceGroupId = PG_GETARG_INT32(1);
 	int32 targetGroupId = PG_GETARG_INT32(2);
@@ -2602,6 +2610,8 @@ citus_internal_update_placement_metadata(PG_FUNCTION_ARGS)
 Datum
 citus_internal_delete_shard_metadata(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	int64 shardId = PG_GETARG_INT64(0);
 
 	if (!ShouldSkipMetadataChecks())
@@ -2640,6 +2650,8 @@ citus_internal_delete_shard_metadata(PG_FUNCTION_ARGS)
 Datum
 citus_internal_update_relation_colocation(PG_FUNCTION_ARGS)
 {
+	CheckCitusVersion(ERROR);
+
 	Oid relationId = PG_GETARG_OID(0);
 	uint32 tagetColocationId = PG_GETARG_UINT32(1);
 

src/backend/distributed/metadata/node_metadata.c

@@ -180,9 +180,6 @@ citus_set_coordinator_host(PG_FUNCTION_ARGS)
 	Name nodeClusterName = PG_GETARG_NAME(3);
 	nodeMetadata.nodeCluster = NameStr(*nodeClusterName);
 
-	/* prevent concurrent modification */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	bool isCoordinatorInMetadata = false;
 	WorkerNode *coordinatorNode = PrimaryNodeForGroup(COORDINATOR_GROUP_ID,
 													  &isCoordinatorInMetadata);
@@ -1410,12 +1407,6 @@ AddNodeMetadata(char *nodeName, int32 nodePort,
 
 	*nodeAlreadyExists = false;
 
-	/*
-	 * Prevent / wait for concurrent modification before checking whether
-	 * the worker already exists in pg_dist_node.
-	 */
-	LockRelationOid(DistNodeRelationId(), RowShareLock);
-
 	WorkerNode *workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort);
 	if (workerNode != NULL)
 	{

src/backend/distributed/operations/create_shards.c

@@ -365,7 +365,7 @@ CreateReferenceTableShard(Oid distributedTableId)
 	List *nodeList = ReferenceTablePlacementNodeList(ShareLock);
 	nodeList = SortList(nodeList, CompareWorkerNodes);
 
-	int replicationFactor = ReferenceTableReplicationFactor();
+	int replicationFactor = list_length(nodeList);
 
 	/* get the next shard id */
 	uint64 shardId = GetNextShardId();

src/backend/distributed/planner/deparse_shard_query.c

@@ -39,7 +39,6 @@
 #include "utils/syscache.h"
 
 
-static void AddInsertAliasIfNeeded(Query *query);
 static void UpdateTaskQueryString(Query *query, Task *task);
 static RelationShard * FindRelationShard(Oid inputRelationId, List *relationShardList);
 static void ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte);
@@ -159,7 +158,7 @@ RebuildQueryStrings(Job *workerJob)
  * deparsing issues (e.g. RETURNING might reference the original table name,
  * which has been replaced by a shard name).
  */
-static void
+void
 AddInsertAliasIfNeeded(Query *query)
 {
 	Assert(query->commandType == CMD_INSERT);

src/backend/distributed/planner/insert_select_planner.c

@@ -48,8 +48,10 @@
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 #include "utils/rel.h"
+#include <nodes/print.h>
 
 
+static void PrepareInsertSelectForCitusPlanner(Query *insertSelectQuery);
 static DistributedPlan * CreateInsertSelectPlanInternal(uint64 planId,
 														Query *originalQuery,
 														PlannerRestrictionContext *
@@ -83,6 +85,7 @@ static DeferredErrorMessage * InsertPartitionColumnMatchesSelect(Query *query,
 static DistributedPlan * CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse,
 														   ParamListInfo boundParams);
 static DeferredErrorMessage * NonPushableInsertSelectSupported(Query *insertSelectQuery);
+static Query * WrapSubquery(Query *subquery);
 static void RelabelTargetEntryList(List *selectTargetList, List *insertTargetList);
 static List * AddInsertSelectCasts(List *insertTargetList, List *selectTargetList,
 								   Oid targetRelationId);
@@ -370,14 +373,17 @@ CreateDistributedInsertSelectPlan(Query *originalQuery,
  * combineQuery, this function also creates a dummy combineQuery for that.
  */
 DistributedPlan *
-CreateInsertSelectIntoLocalTablePlan(uint64 planId, Query *originalQuery, ParamListInfo
+CreateInsertSelectIntoLocalTablePlan(uint64 planId, Query *insertSelectQuery,
-									 boundParams, bool hasUnresolvedParams,
+									 ParamListInfo boundParams, bool hasUnresolvedParams,
 									 PlannerRestrictionContext *plannerRestrictionContext)
 {
-	RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(originalQuery);
+	RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery);
+
+	PrepareInsertSelectForCitusPlanner(insertSelectQuery);
+
+	/* get the SELECT query (may have changed after PrepareInsertSelectForCitusPlanner) */
+	Query *selectQuery = selectRte->subquery;
 
-	Query *selectQuery = BuildSelectForInsertSelect(originalQuery);
-	originalQuery->cteList = NIL;
 	DistributedPlan *distPlan = CreateDistributedPlan(planId, selectQuery,
 													  copyObject(selectQuery),
 													  boundParams, hasUnresolvedParams,
@@ -417,12 +423,84 @@ CreateInsertSelectIntoLocalTablePlan(uint64 planId, Query *originalQuery, ParamL
 	 * distributed select instead of returning it.
 	 */
 	selectRte->subquery = distPlan->combineQuery;
-	distPlan->combineQuery = originalQuery;
+	distPlan->combineQuery = insertSelectQuery;
 
 	return distPlan;
 }
 
 
+/*
+ * PrepareInsertSelectForCitusPlanner prepares an INSERT..SELECT query tree
+ * that was passed to the planner for use by Citus.
+ *
+ * First, it rebuilds the target lists of the INSERT and the SELECT
+ * to be in the same order, which is not guaranteed in the parse tree.
+ *
+ * Second, some of the constants in the target list will have type
+ * "unknown", which would confuse the Citus planner. To address that,
+ * we add casts to SELECT target list entries whose type does not correspond
+ * to the destination. This also helps us feed the output directly into
+ * a COPY stream for INSERT..SELECT via coordinator.
+ *
+ * In case of UNION or other set operations, the SELECT does not have a
+ * clearly defined target list, so we first wrap the UNION in a subquery.
+ * UNION queries do not have the "unknown" type problem.
+ *
+ * Finally, if the INSERT has CTEs, we move those CTEs into the SELECT,
+ * such that we can plan the SELECT as an independent query. To ensure
+ * the ctelevelsup for CTE RTE's remain the same, we wrap the SELECT into
+ * a subquery, unless we already did so in case of a UNION.
+ */
+static void
+PrepareInsertSelectForCitusPlanner(Query *insertSelectQuery)
+{
+	RangeTblEntry *insertRte = ExtractResultRelationRTEOrError(insertSelectQuery);
+	RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery);
+	Oid targetRelationId = insertRte->relid;
+
+	bool isWrapped = false;
+
+	if (selectRte->subquery->setOperations != NULL)
+	{
+		/*
+		 * Prepare UNION query for reordering and adding casts by
+		 * wrapping it in a subquery to have a single target list.
+		 */
+		selectRte->subquery = WrapSubquery(selectRte->subquery);
+		isWrapped = true;
+	}
+
+	/* this is required for correct deparsing of the query */
+	ReorderInsertSelectTargetLists(insertSelectQuery, insertRte, selectRte);
+
+	/*
+	 * Cast types of insert target list and select projection list to
+	 * match the column types of the target relation.
+	 */
+	selectRte->subquery->targetList =
+		AddInsertSelectCasts(insertSelectQuery->targetList,
+							 copyObject(selectRte->subquery->targetList),
+							 targetRelationId);
+
+	if (list_length(insertSelectQuery->cteList) > 0)
+	{
+		if (!isWrapped)
+		{
+			/*
+			 * By wrapping the SELECT in a subquery, we can avoid adjusting
+			 * ctelevelsup in RTE's that point to the CTEs.
+			 */
+			selectRte->subquery = WrapSubquery(selectRte->subquery);
+		}
+
+		/* copy CTEs from the INSERT ... SELECT statement into outer SELECT */
+		selectRte->subquery->cteList = copyObject(insertSelectQuery->cteList);
+		selectRte->subquery->hasModifyingCTE = insertSelectQuery->hasModifyingCTE;
+		insertSelectQuery->cteList = NIL;
+	}
+}
+
+
 /*
  * CreateCombineQueryForRouterPlan is used for creating a dummy combineQuery
  * for a router plan, since router plans normally don't have one.
@@ -881,12 +959,11 @@ ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte,
 	ListCell *insertTargetEntryCell;
 	List *newSubqueryTargetlist = NIL;
 	List *newInsertTargetlist = NIL;
+	List *columnNameList = NIL;
 	int resno = 1;
-	Index insertTableId = 1;
+	Index selectTableId = 2;
 	int targetEntryIndex = 0;
 
-	AssertArg(InsertSelectIntoCitusTable(originalQuery));
-
 	Query *subquery = subqueryRte->subquery;
 
 	Oid insertRelationId = insertRte->relid;
@@ -954,6 +1031,9 @@ ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte,
 											newSubqueryTargetEntry);
 		}
 
+		Value *columnName = makeString(newSubqueryTargetEntry->resname);
+		columnNameList = lappend(columnNameList, columnName);
+
 		/*
 		 * The newly created select target entry cannot be a junk entry since junk
 		 * entries are not in the final target list and we're processing the
@@ -961,7 +1041,7 @@ ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte,
 		 */
 		Assert(!newSubqueryTargetEntry->resjunk);
 
-		Var *newInsertVar = makeVar(insertTableId, originalAttrNo,
+		Var *newInsertVar = makeVar(selectTableId, resno,
 									exprType((Node *) newSubqueryTargetEntry->expr),
 									exprTypmod((Node *) newSubqueryTargetEntry->expr),
 									exprCollation((Node *) newSubqueryTargetEntry->expr),
@@ -1005,6 +1085,7 @@ ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte,
 
 	originalQuery->targetList = newInsertTargetlist;
 	subquery->targetList = newSubqueryTargetlist;
+	subqueryRte->eref->colnames = columnNameList;
 
 	return NULL;
 }
@@ -1412,19 +1493,10 @@ CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse, ParamListInfo bou
 		return distributedPlan;
 	}
 
-	Query *selectQuery = BuildSelectForInsertSelect(insertSelectQuery);
+	PrepareInsertSelectForCitusPlanner(insertSelectQuery);
 
-	selectRte->subquery = selectQuery;
+	/* get the SELECT query (may have changed after PrepareInsertSelectForCitusPlanner) */
-	ReorderInsertSelectTargetLists(insertSelectQuery, insertRte, selectRte);
+	Query *selectQuery = selectRte->subquery;
-
-	/*
-	 * Cast types of insert target list and select projection list to
-	 * match the column types of the target relation.
-	 */
-	selectQuery->targetList =
-		AddInsertSelectCasts(insertSelectQuery->targetList,
-							 selectQuery->targetList,
-							 targetRelationId);
 
 	/*
 	 * Later we might need to call WrapTaskListForProjection(), which requires
@@ -1506,6 +1578,63 @@ InsertSelectResultIdPrefix(uint64 planId)
 }
 
 
+/*
+ * WrapSubquery wraps the given query as a subquery in a newly constructed
+ * "SELECT * FROM (...subquery...) citus_insert_select_subquery" query.
+ */
+static Query *
+WrapSubquery(Query *subquery)
+{
+	ParseState *pstate = make_parsestate(NULL);
+	List *newTargetList = NIL;
+
+	Query *outerQuery = makeNode(Query);
+	outerQuery->commandType = CMD_SELECT;
+
+	/* create range table entries */
+	Alias *selectAlias = makeAlias("citus_insert_select_subquery", NIL);
+	RangeTblEntry *newRangeTableEntry = RangeTableEntryFromNSItem(
+		addRangeTableEntryForSubquery(
+			pstate, subquery,
+			selectAlias, false, true));
+	outerQuery->rtable = list_make1(newRangeTableEntry);
+
+	/* set the FROM expression to the subquery */
+	RangeTblRef *newRangeTableRef = makeNode(RangeTblRef);
+	newRangeTableRef->rtindex = 1;
+	outerQuery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL);
+
+	/* create a target list that matches the SELECT */
+	TargetEntry *selectTargetEntry = NULL;
+	foreach_ptr(selectTargetEntry, subquery->targetList)
+	{
+		/* exactly 1 entry in FROM */
+		int indexInRangeTable = 1;
+
+		if (selectTargetEntry->resjunk)
+		{
+			continue;
+		}
+
+		Var *newSelectVar = makeVar(indexInRangeTable, selectTargetEntry->resno,
+									exprType((Node *) selectTargetEntry->expr),
+									exprTypmod((Node *) selectTargetEntry->expr),
+									exprCollation((Node *) selectTargetEntry->expr), 0);
+
+		TargetEntry *newSelectTargetEntry = makeTargetEntry((Expr *) newSelectVar,
+															selectTargetEntry->resno,
+															selectTargetEntry->resname,
+															selectTargetEntry->resjunk);
+
+		newTargetList = lappend(newTargetList, newSelectTargetEntry);
+	}
+
+	outerQuery->targetList = newTargetList;
+
+	return outerQuery;
+}
+
+
 /*
  * RelabelTargetEntryList relabels select target list to have matching names with
  * insert target list.
@@ -1557,16 +1686,22 @@ AddInsertSelectCasts(List *insertTargetList, List *selectTargetList,
 	{
 		TargetEntry *insertEntry = (TargetEntry *) lfirst(insertEntryCell);
 		TargetEntry *selectEntry = (TargetEntry *) lfirst(selectEntryCell);
-		Var *insertColumn = (Var *) insertEntry->expr;
+
 		Form_pg_attribute attr = TupleDescAttr(destTupleDescriptor,
 											   insertEntry->resno - 1);
 
-		Oid sourceType = insertColumn->vartype;
+		Oid sourceType = exprType((Node *) selectEntry->expr);
 		Oid targetType = attr->atttypid;
 		if (sourceType != targetType)
 		{
-			insertEntry->expr = CastExpr((Expr *) insertColumn, sourceType, targetType,
+			/* ReorderInsertSelectTargetLists ensures we only have Vars */
-										 attr->attcollation, attr->atttypmod);
+			Assert(IsA(insertEntry->expr, Var));
+
+			/* we will cast the SELECT expression, so the type changes */
+			Var *insertVar = (Var *) insertEntry->expr;
+			insertVar->vartype = targetType;
+			insertVar->vartypmod = attr->atttypmod;
+			insertVar->varcollid = attr->attcollation;
 
 			/*
 			 * We cannot modify the selectEntry in-place, because ORDER BY or

src/backend/distributed/planner/local_plan_cache.c

@@ -174,6 +174,8 @@ DeparseLocalShardQuery(Query *jobQuery, List *relationShardList, Oid
 		 */
 		Assert(!CheckInsertSelectQuery(jobQuery));
 
+		AddInsertAliasIfNeeded(jobQuery);
+
 		/*
 		 * For INSERT queries we cannot use pg_get_query_def. Mainly because we
 		 * cannot run UpdateRelationToShardNames on an INSERT query. This is

src/backend/distributed/planner/multi_explain.c

@@ -1487,7 +1487,9 @@ WrapQueryForExplainAnalyze(const char *queryString, TupleDesc tupleDesc)
 		}
 
 		Form_pg_attribute attr = &tupleDesc->attrs[columnIndex];
-		char *attrType = format_type_with_typemod(attr->atttypid, attr->atttypmod);
+		char *attrType = format_type_extended(attr->atttypid, attr->atttypmod,
+											  FORMAT_TYPE_TYPEMOD_GIVEN |
+											  FORMAT_TYPE_FORCE_QUALIFY);
 
 		appendStringInfo(columnDef, "field_%d %s", columnIndex, attrType);
 	}

src/backend/distributed/planner/multi_logical_optimizer.c

@@ -1616,7 +1616,19 @@ MasterAggregateExpression(Aggref *originalAggregate,
 		Expr *directarg;
 		foreach_ptr(directarg, originalAggregate->aggdirectargs)
 		{
-			if (!IsA(directarg, Const) && !IsA(directarg, Param))
+			/*
+			 * Need to replace nodes that contain any Vars with Vars referring
+			 * to the related column of the result set returned for the worker
+			 * aggregation.
+			 *
+			 * When there are no Vars, then the expression can be fully evaluated
+			 * on the coordinator, so we skip it here. This is not just an
+			 * optimization, but the result of the expression might require
+			 * calling the final function of the aggregate, and doing so when
+			 * there are no input rows (i.e.: with an empty tuple slot) is not
+			 * desirable for the node-executor methods.
+			 */
+			if (pull_var_clause_default((Node *) directarg) != NIL)
 			{
 				Var *var = makeVar(masterTableId, walkerContext->columnId,
 								   exprType((Node *) directarg),
@@ -3070,7 +3082,13 @@ WorkerAggregateExpressionList(Aggref *originalAggregate,
 		Expr *directarg;
 		foreach_ptr(directarg, originalAggregate->aggdirectargs)
 		{
-			if (!IsA(directarg, Const) && !IsA(directarg, Param))
+			/*
+			 * The worker aggregation should execute any node that contains any
+			 * Var nodes and return the result in the targetlist, so that the
+			 * combine query can then fetch the result via remote scan; see
+			 * MasterAggregateExpression.
+			 */
+			if (pull_var_clause_default((Node *) directarg) != NIL)
 			{
 				workerAggregateList = lappend(workerAggregateList, directarg);
 			}

src/backend/distributed/planner/multi_router_planner.c

@@ -3558,19 +3558,9 @@ DeferErrorIfUnsupportedRouterPlannableSelectQuery(Query *query)
 							 NULL, NULL);
 	}
 
-	if (contain_nextval_expression_walker((Node *) query->targetList, NULL))
-	{
-		/*
-		 * We let queries with nextval in the target list fall through to
-		 * the logical planner, which knows how to handle those queries.
-		 */
-		return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-							 "Sequences cannot be used in router queries",
-							 NULL, NULL);
-	}
-
 	bool hasPostgresOrCitusLocalTable = false;
 	bool hasDistributedTable = false;
+	bool hasReferenceTable = false;
 
 	ExtractRangeTableRelationWalker((Node *) query, &rangeTableRelationList);
 	foreach(rangeTableRelationCell, rangeTableRelationList)
@@ -3586,6 +3576,11 @@ DeferErrorIfUnsupportedRouterPlannableSelectQuery(Query *query)
 				hasPostgresOrCitusLocalTable = true;
 				continue;
 			}
+			else if (IsCitusTableType(distributedTableId, REFERENCE_TABLE))
+			{
+				hasReferenceTable = true;
+				continue;
+			}
 			else if (IsCitusTableType(distributedTableId, CITUS_LOCAL_TABLE))
 			{
 				hasPostgresOrCitusLocalTable = true;
@@ -3628,6 +3623,28 @@ DeferErrorIfUnsupportedRouterPlannableSelectQuery(Query *query)
 		}
 	}
 
+	/*
+	 * We want to make sure nextval happens on the coordinator / the current
+	 * node, since the user may have certain expectations around the values
+	 * produced by the sequence. We therefore cannot push down the nextval
+	 * call as part of a router query.
+	 *
+	 * We let queries with nextval in the target list fall through to
+	 * the logical planner, which will ensure that the nextval is called
+	 * in the combine query on the coordinator.
+	 *
+	 * If there are no distributed or reference tables in the query,
+	 * then the query will anyway happen on the coordinator, so we can
+	 * allow nextval.
+	 */
+	if (contain_nextval_expression_walker((Node *) query->targetList, NULL) &&
+		(hasDistributedTable || hasReferenceTable))
+	{
+		return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+							 "Sequences cannot be used in router queries",
+							 NULL, NULL);
+	}
+
 	/* local tables are not allowed if there are distributed tables */
 	if (hasPostgresOrCitusLocalTable && hasDistributedTable)
 	{

src/backend/distributed/sql/citus--10.2-1--10.2-2.sql

@@ -0,0 +1,5 @@
+-- citus--10.2-1--10.2-2
+
+-- bump version to 10.2-2
+
+#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"

src/backend/distributed/sql/citus--10.2-2--10.2-3.sql

@@ -0,0 +1,5 @@
+-- citus--10.2-2--10.2-3
+
+-- bump version to 10.2-3
+
+#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"

src/backend/distributed/sql/citus--10.2-3--10.2-4.sql

@@ -0,0 +1,10 @@
+-- citus--10.2-3--10.2-4
+
+-- bump version to 10.2-4
+
+#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
+
+#include "udfs/fix_partition_shard_index_names/10.2-4.sql"
+#include "udfs/fix_all_partition_shard_index_names/10.2-4.sql"
+#include "udfs/worker_fix_partition_shard_index_names/10.2-4.sql"
+#include "udfs/citus_finish_pg_upgrade/10.2-4.sql"

src/backend/distributed/sql/citus--10.2-4--10.2-5.sql

@@ -0,0 +1 @@
+#include "udfs/citus_finish_pg_upgrade/10.2-5.sql"

src/backend/distributed/sql/citus--10.2-5--10.2-4.sql

@@ -0,0 +1 @@
+#include "udfs/citus_finish_pg_upgrade/10.2-4.sql"

src/backend/distributed/sql/downgrades/citus--10.2-2--10.2-1.sql

@@ -0,0 +1,3 @@
+-- citus--10.2-2--10.2-1
+
+#include "../../../columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql"

src/backend/distributed/sql/downgrades/citus--10.2-3--10.2-2.sql

@@ -0,0 +1,3 @@
+-- citus--10.2-3--10.2-2
+
+#include "../../../columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql"

src/backend/distributed/sql/downgrades/citus--10.2-4--10.2-3.sql

@@ -0,0 +1,12 @@
+-- citus--10.2-4--10.2-3
+
+DROP FUNCTION pg_catalog.fix_all_partition_shard_index_names();
+DROP FUNCTION pg_catalog.fix_partition_shard_index_names(regclass);
+DROP FUNCTION pg_catalog.worker_fix_partition_shard_index_names(regclass, text, text);
+
+#include "../udfs/citus_finish_pg_upgrade/10.2-1.sql"
+
+-- This needs to be done after downgrading citus_finish_pg_upgrade. This is
+-- because citus_finish_pg_upgrade/10.2-4 depends on columnar_ensure_am_depends_catalog,
+-- which is dropped by columnar--10.2-4--10.2-3.sql
+#include "../../../columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql"

src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-4.sql

@@ -0,0 +1,144 @@
+CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade()
+    RETURNS void
+    LANGUAGE plpgsql
+    SET search_path = pg_catalog
+    AS $cppu$
+DECLARE
+    table_name regclass;
+    command text;
+    trigger_name text;
+BEGIN
+
+
+    IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN
+    EXECUTE $cmd$
+        CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray);
+        COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray)
+        IS 'concatenate input arrays into a single array';
+    $cmd$;
+    ELSE
+    EXECUTE $cmd$
+        CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray);
+        COMMENT ON AGGREGATE array_cat_agg(anyarray)
+        IS 'concatenate input arrays into a single array';
+    $cmd$;
+    END IF;
+
+    --
+    -- Citus creates the array_cat_agg but because of a compatibility
+    -- issue between pg13-pg14, we drop and create it during upgrade.
+    -- And as Citus creates it, there needs to be a dependency to the
+    -- Citus extension, so we create that dependency here.
+    -- We are not using:
+    --  ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg
+    -- because we don't have an easy way to check if the aggregate
+    -- exists with anyarray type or anycompatiblearray type.
+
+    INSERT INTO pg_depend
+    SELECT
+        'pg_proc'::regclass::oid as classid,
+        (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid,
+        0 as objsubid,
+        'pg_extension'::regclass::oid as refclassid,
+        (select oid from pg_extension where extname = 'citus') as refobjid,
+        0 as refobjsubid ,
+        'e' as deptype;
+
+    --
+    -- restore citus catalog tables
+    --
+    INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition;
+    INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard;
+    INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement;
+    INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata;
+    INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node;
+    INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group;
+    INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction;
+    INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation;
+    -- enterprise catalog tables
+    INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo;
+    INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo;
+
+    INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT
+        name,
+        default_strategy,
+        shard_cost_function::regprocedure::regproc,
+        node_capacity_function::regprocedure::regproc,
+        shard_allowed_on_node_function::regprocedure::regproc,
+        default_threshold,
+        minimum_threshold,
+        improvement_threshold
+    FROM public.pg_dist_rebalance_strategy;
+
+    --
+    -- drop backup tables
+    --
+    DROP TABLE public.pg_dist_authinfo;
+    DROP TABLE public.pg_dist_colocation;
+    DROP TABLE public.pg_dist_local_group;
+    DROP TABLE public.pg_dist_node;
+    DROP TABLE public.pg_dist_node_metadata;
+    DROP TABLE public.pg_dist_partition;
+    DROP TABLE public.pg_dist_placement;
+    DROP TABLE public.pg_dist_poolinfo;
+    DROP TABLE public.pg_dist_shard;
+    DROP TABLE public.pg_dist_transaction;
+    DROP TABLE public.pg_dist_rebalance_strategy;
+
+    --
+    -- reset sequences
+    --
+    PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false);
+    PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false);
+    PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false);
+    PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false);
+    PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false);
+
+    --
+    -- register triggers
+    --
+    FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition
+    LOOP
+        trigger_name := 'truncate_trigger_' || table_name::oid;
+        command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()';
+        EXECUTE command;
+        command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name);
+        EXECUTE command;
+    END LOOP;
+
+    --
+    -- set dependencies
+    --
+    INSERT INTO pg_depend
+    SELECT
+        'pg_class'::regclass::oid as classid,
+        p.logicalrelid::regclass::oid as objid,
+        0 as objsubid,
+        'pg_extension'::regclass::oid as refclassid,
+        (select oid from pg_extension where extname = 'citus') as refobjid,
+        0 as refobjsubid ,
+        'n' as deptype
+    FROM pg_catalog.pg_dist_partition p;
+
+    -- set dependencies for columnar table access method
+    PERFORM citus_internal.columnar_ensure_am_depends_catalog();
+
+    -- restore pg_dist_object from the stable identifiers
+    TRUNCATE citus.pg_dist_object;
+    INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid)
+    SELECT
+        address.classid,
+        address.objid,
+        address.objsubid,
+        naming.distribution_argument_index,
+        naming.colocationid
+    FROM
+        public.pg_dist_object naming,
+        pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address;
+
+    DROP TABLE public.pg_dist_object;
+END;
+$cppu$;
+
+COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade()
+    IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade';

src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-5.sql

@@ -0,0 +1,144 @@
+CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade()
+    RETURNS void
+    LANGUAGE plpgsql
+    SET search_path = pg_catalog
+    AS $cppu$
+DECLARE
+    table_name regclass;
+    command text;
+    trigger_name text;
+BEGIN
+
+
+    IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN
+    EXECUTE $cmd$
+        CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray);
+        COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray)
+        IS 'concatenate input arrays into a single array';
+    $cmd$;
+    ELSE
+    EXECUTE $cmd$
+        CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray);
+        COMMENT ON AGGREGATE array_cat_agg(anyarray)
+        IS 'concatenate input arrays into a single array';
+    $cmd$;
+    END IF;
+
+    --
+    -- Citus creates the array_cat_agg but because of a compatibility
+    -- issue between pg13-pg14, we drop and create it during upgrade.
+    -- And as Citus creates it, there needs to be a dependency to the
+    -- Citus extension, so we create that dependency here.
+    -- We are not using:
+    --  ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg
+    -- because we don't have an easy way to check if the aggregate
+    -- exists with anyarray type or anycompatiblearray type.
+
+    INSERT INTO pg_depend
+    SELECT
+        'pg_proc'::regclass::oid as classid,
+        (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid,
+        0 as objsubid,
+        'pg_extension'::regclass::oid as refclassid,
+        (select oid from pg_extension where extname = 'citus') as refobjid,
+        0 as refobjsubid ,
+        'e' as deptype;
+
+    --
+    -- restore citus catalog tables
+    --
+    INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition;
+    INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard;
+    INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement;
+    INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata;
+    INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node;
+    INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group;
+    INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction;
+    INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation;
+    -- enterprise catalog tables
+    INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo;
+    INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo;
+
+    INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT
+        name,
+        default_strategy,
+        shard_cost_function::regprocedure::regproc,
+        node_capacity_function::regprocedure::regproc,
+        shard_allowed_on_node_function::regprocedure::regproc,
+        default_threshold,
+        minimum_threshold,
+        improvement_threshold
+    FROM public.pg_dist_rebalance_strategy;
+
+    --
+    -- drop backup tables
+    --
+    DROP TABLE public.pg_dist_authinfo;
+    DROP TABLE public.pg_dist_colocation;
+    DROP TABLE public.pg_dist_local_group;
+    DROP TABLE public.pg_dist_node;
+    DROP TABLE public.pg_dist_node_metadata;
+    DROP TABLE public.pg_dist_partition;
+    DROP TABLE public.pg_dist_placement;
+    DROP TABLE public.pg_dist_poolinfo;
+    DROP TABLE public.pg_dist_shard;
+    DROP TABLE public.pg_dist_transaction;
+    DROP TABLE public.pg_dist_rebalance_strategy;
+
+    --
+    -- reset sequences
+    --
+    PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false);
+    PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false);
+    PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false);
+    PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false);
+    PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false);
+
+    --
+    -- register triggers
+    --
+    FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f'
+    LOOP
+        trigger_name := 'truncate_trigger_' || table_name::oid;
+        command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()';
+        EXECUTE command;
+        command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name);
+        EXECUTE command;
+    END LOOP;
+
+    --
+    -- set dependencies
+    --
+    INSERT INTO pg_depend
+    SELECT
+        'pg_class'::regclass::oid as classid,
+        p.logicalrelid::regclass::oid as objid,
+        0 as objsubid,
+        'pg_extension'::regclass::oid as refclassid,
+        (select oid from pg_extension where extname = 'citus') as refobjid,
+        0 as refobjsubid ,
+        'n' as deptype
+    FROM pg_catalog.pg_dist_partition p;
+
+    -- set dependencies for columnar table access method
+    PERFORM citus_internal.columnar_ensure_am_depends_catalog();
+
+    -- restore pg_dist_object from the stable identifiers
+    TRUNCATE citus.pg_dist_object;
+    INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid)
+    SELECT
+        address.classid,
+        address.objid,
+        address.objsubid,
+        naming.distribution_argument_index,
+        naming.colocationid
+    FROM
+        public.pg_dist_object naming,
+        pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address;
+
+    DROP TABLE public.pg_dist_object;
+END;
+$cppu$;
+
+COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade()
+    IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade';

src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql

@@ -97,7 +97,7 @@ BEGIN
     --
     -- register triggers
     --
-    FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition
+    FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f'
     LOOP
         trigger_name := 'truncate_trigger_' || table_name::oid;
         command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()';
@@ -120,6 +120,9 @@ BEGIN
         'n' as deptype
     FROM pg_catalog.pg_dist_partition p;
 
+    -- set dependencies for columnar table access method
+    PERFORM citus_internal.columnar_ensure_am_depends_catalog();
+
     -- restore pg_dist_object from the stable identifiers
     TRUNCATE citus.pg_dist_object;
     INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid)

src/backend/distributed/sql/udfs/fix_all_partition_shard_index_names/10.2-4.sql

@@ -0,0 +1,21 @@
+CREATE OR REPLACE FUNCTION pg_catalog.fix_all_partition_shard_index_names()
+  RETURNS SETOF regclass
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+	dist_partitioned_table_name regclass;
+BEGIN
+  FOR dist_partitioned_table_name IN SELECT p.logicalrelid
+                                     FROM pg_dist_partition p
+                                     JOIN pg_class c ON p.logicalrelid = c.oid
+                                     WHERE c.relkind = 'p'
+		                                 ORDER BY c.relname, c.oid
+    LOOP
+      EXECUTE 'SELECT fix_partition_shard_index_names( ' || quote_literal(dist_partitioned_table_name) || ' )';
+      RETURN NEXT dist_partitioned_table_name;
+    END LOOP;
+  RETURN;
+END;
+$$;
+COMMENT ON FUNCTION pg_catalog.fix_all_partition_shard_index_names()
+  IS 'fix index names on partition shards of all tables';

src/backend/distributed/sql/udfs/fix_all_partition_shard_index_names/latest.sql

@@ -0,0 +1,21 @@
+CREATE OR REPLACE FUNCTION pg_catalog.fix_all_partition_shard_index_names()
+  RETURNS SETOF regclass
+  LANGUAGE plpgsql
+  AS $$
+DECLARE
+	dist_partitioned_table_name regclass;
+BEGIN
+  FOR dist_partitioned_table_name IN SELECT p.logicalrelid
+                                     FROM pg_dist_partition p
+                                     JOIN pg_class c ON p.logicalrelid = c.oid
+                                     WHERE c.relkind = 'p'
+		                                 ORDER BY c.relname, c.oid
+    LOOP
+      EXECUTE 'SELECT fix_partition_shard_index_names( ' || quote_literal(dist_partitioned_table_name) || ' )';
+      RETURN NEXT dist_partitioned_table_name;
+    END LOOP;
+  RETURN;
+END;
+$$;
+COMMENT ON FUNCTION pg_catalog.fix_all_partition_shard_index_names()
+  IS 'fix index names on partition shards of all tables';

src/backend/distributed/sql/udfs/fix_partition_shard_index_names/10.2-4.sql

@@ -0,0 +1,6 @@
+CREATE FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass)
+  RETURNS void
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$fix_partition_shard_index_names$$;
+COMMENT ON FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass)
+  IS 'fix index names on partition shards of given table';

src/backend/distributed/sql/udfs/fix_partition_shard_index_names/latest.sql

@@ -0,0 +1,6 @@
+CREATE FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass)
+  RETURNS void
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$fix_partition_shard_index_names$$;
+COMMENT ON FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass)
+  IS 'fix index names on partition shards of given table';

src/backend/distributed/sql/udfs/worker_fix_partition_shard_index_names/10.2-4.sql

@@ -0,0 +1,10 @@
+CREATE FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass,
+                                                                  partition_shard text,
+                                                                  new_partition_shard_index_name text)
+  RETURNS void
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$worker_fix_partition_shard_index_names$$;
+COMMENT ON FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass,
+                                                                      partition_shard text,
+                                                                      new_partition_shard_index_name text)
+  IS 'fix the name of the index on given partition shard that is child of given parent_index';

src/backend/distributed/sql/udfs/worker_fix_partition_shard_index_names/latest.sql

@@ -0,0 +1,10 @@
+CREATE FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass,
+                                                                  partition_shard text,
+                                                                  new_partition_shard_index_name text)
+  RETURNS void
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$worker_fix_partition_shard_index_names$$;
+COMMENT ON FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass,
+                                                                      partition_shard text,
+                                                                      new_partition_shard_index_name text)
+  IS 'fix the name of the index on given partition shard that is child of given parent_index';

src/backend/distributed/transaction/transaction_management.c

@@ -320,7 +320,7 @@ CoordinatedTransactionCallback(XactEvent event, void *arg)
 			/* stop propagating notices from workers, we know the query is failed */
 			DisableWorkerMessagePropagation();
 
-			RemoveIntermediateResultsDirectory();
+			RemoveIntermediateResultsDirectories();
 
 			ResetShardPlacementTransactionState();
 
@@ -408,7 +408,7 @@ CoordinatedTransactionCallback(XactEvent event, void *arg)
 			 * existing folders that are associated with distributed transaction
 			 * ids on the worker nodes.
 			 */
-			RemoveIntermediateResultsDirectory();
+			RemoveIntermediateResultsDirectories();
 
 			UnSetDistributedTransactionId();
 			break;
@@ -420,10 +420,10 @@ CoordinatedTransactionCallback(XactEvent event, void *arg)
 			 * If the distributed query involves 2PC, we already removed
 			 * the intermediate result directory on XACT_EVENT_PREPARE. However,
 			 * if not, we should remove it here on the COMMIT. Since
-			 * RemoveIntermediateResultsDirectory() is idempotent, we're safe
+			 * RemoveIntermediateResultsDirectories() is idempotent, we're safe
 			 * to call it here again even if the transaction involves 2PC.
 			 */
-			RemoveIntermediateResultsDirectory();
+			RemoveIntermediateResultsDirectories();
 
 			/* nothing further to do if there's no managed remote xacts */
 			if (CurrentCoordinatedTransactionState == COORD_TRANS_NONE)

src/backend/distributed/utils/enable_ssl.c

@@ -8,6 +8,19 @@
  *-------------------------------------------------------------------------
  */
 
+
+/*
+ * Make sure that functions marked as deprecated in OpenSSL 3.0 don't trigger
+ * deprecation warnings by indicating that we're using the OpenSSL 1.0.1
+ * compatibile API. Postgres does this by already in PG14, so we should not do
+ * it otherwise we get warnings about redefining this value.
+ */
+#if PG_VERSION_NUM < PG_VERSION_14
+#ifndef OPENSSL_API_COMPAT
+#define OPENSSL_API_COMPAT 0x1000100L
+#endif
+#endif
+
 #include "postgres.h"
 
 #include "distributed/connection_management.h"

src/backend/distributed/utils/multi_partitioning_utils.c

@@ -11,11 +11,13 @@
 #include "access/genam.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
+#include "catalog/index.h"
 #include "catalog/indexing.h"
 #include "catalog/partition.h"
 #include "catalog/pg_class.h"
 #include "catalog/pg_constraint.h"
 #include "catalog/pg_inherits.h"
+#include "commands/tablecmds.h"
 #include "common/string.h"
 #include "distributed/citus_nodes.h"
 #include "distributed/adaptive_executor.h"
@@ -26,13 +28,16 @@
 #include "distributed/deparse_shard_query.h"
 #include "distributed/listutils.h"
 #include "distributed/metadata_utility.h"
+#include "distributed/multi_executor.h"
 #include "distributed/multi_partitioning_utils.h"
 #include "distributed/multi_physical_planner.h"
 #include "distributed/relay_utility.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shardinterval_utils.h"
 #include "distributed/version_compat.h"
+#include "distributed/worker_protocol.h"
 #include "lib/stringinfo.h"
+#include "nodes/makefuncs.h"
 #include "nodes/pg_list.h"
 #include "pgstat.h"
 #include "partitioning/partdesc.h"
@@ -41,12 +46,22 @@
 #include "utils/lsyscache.h"
 #include "utils/rel.h"
 #include "utils/syscache.h"
+#include "utils/varlena.h"
 
 static char * PartitionBound(Oid partitionId);
 static Relation try_relation_open_nolock(Oid relationId);
 static List * CreateFixPartitionConstraintsTaskList(Oid relationId);
 static List * WorkerFixPartitionConstraintCommandList(Oid relationId, uint64 shardId,
 													  List *checkConstraintList);
+static List * CreateFixPartitionShardIndexNamesTaskList(Oid parentRelationId);
+static List * WorkerFixPartitionShardIndexNamesCommandList(uint64 parentShardId,
+														   List *indexIdList);
+static List * WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex(
+	char *qualifiedParentShardIndexName, Oid parentIndexId);
+static List * WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(Oid
+																			partitionIndexId,
+																			char *
+																			qualifiedParentShardIndexName);
 static List * CheckConstraintNameListForRelation(Oid relationId);
 static bool RelationHasConstraint(Oid relationId, char *constraintName);
 static char * RenameConstraintCommand(Oid relationId, char *constraintName,
@@ -55,6 +70,8 @@ static char * RenameConstraintCommand(Oid relationId, char *constraintName,
 
 PG_FUNCTION_INFO_V1(fix_pre_citus10_partitioned_table_constraint_names);
 PG_FUNCTION_INFO_V1(worker_fix_pre_citus10_partitioned_table_constraint_names);
+PG_FUNCTION_INFO_V1(fix_partition_shard_index_names);
+PG_FUNCTION_INFO_V1(worker_fix_partition_shard_index_names);
 
 
 /*
@@ -130,6 +147,167 @@ worker_fix_pre_citus10_partitioned_table_constraint_names(PG_FUNCTION_ARGS)
 }
 
 
+/*
+ * fix_partition_shard_index_names fixes the index names of shards of partitions of
+ * partitioned tables on workers.
+ *
+ * When running CREATE INDEX on parent_table, we didn't explicitly create the index on
+ * each partition as well. Postgres created indexes for partitions in the coordinator,
+ * and also in the workers. Actually, Postgres auto-generates index names when auto-creating
+ * indexes on each partition shard of the parent shards. If index name is too long, it
+ * truncates the name and adds _idx postfix to it. However, when truncating the name, the
+ * shardId of the partition shard can be lost. This may result in the same index name used for
+ * the partition shell table and one of the partition shards.
+ * For more details, check issue #4962 https://github.com/citusdata/citus/issues/4962
+ *
+ * fix_partition_shard_index_names renames indexes of shards of partition tables to include
+ * the shardId at the end of the name, regardless of whether index name was long or short
+ * As a result there will be no index name ending in _idx, rather all will end in _{shardid}
+ * Algorithm is:
+ * foreach parentShard in shardListOfParentTableId:
+ *  foreach parentIndex on parent:
+ *      generate qualifiedParentShardIndexName -> parentShardIndex
+ *      foreach inheritedPartitionIndex on parentIndex:
+ *          get table relation of inheritedPartitionIndex -> partitionId
+ *          foreach partitionShard in shardListOfPartitionid:
+ *              generate qualifiedPartitionShardName -> partitionShard
+ *              generate newPartitionShardIndexName
+ *              (the following happens in the worker node)
+ *              foreach inheritedPartitionShardIndex on parentShardIndex:
+ *                  if table relation of inheritedPartitionShardIndex is partitionShard:
+ *                      if inheritedPartitionShardIndex does not have proper name:
+ *                          Rename(inheritedPartitionShardIndex, newPartitionShardIndexName)
+ *                      break
+ */
+Datum
+fix_partition_shard_index_names(PG_FUNCTION_ARGS)
+{
+	CheckCitusVersion(ERROR);
+	EnsureCoordinator();
+
+	Oid relationId = PG_GETARG_OID(0);
+
+	Relation relation = try_relation_open(relationId, AccessExclusiveLock);
+
+	if (relation == NULL)
+	{
+		ereport(NOTICE, (errmsg("relation with OID %u does not exist, skipping",
+								relationId)));
+		PG_RETURN_VOID();
+	}
+
+	if (relation->rd_rel->relkind != RELKIND_PARTITIONED_TABLE)
+	{
+		relation_close(relation, NoLock);
+		ereport(ERROR, (errmsg(
+							"Fixing shard index names is only applicable to partitioned"
+							" tables, and \"%s\" is not a partitioned table",
+							RelationGetRelationName(relation))));
+	}
+
+	if (!IsCitusTable(relationId))
+	{
+		relation_close(relation, NoLock);
+		ereport(ERROR, (errmsg("fix_partition_shard_index_names can "
+							   "only be called for distributed partitioned tables")));
+	}
+
+	EnsureTableOwner(relationId);
+
+	List *taskList = CreateFixPartitionShardIndexNamesTaskList(relationId);
+
+	/* do not do anything if there are no index names to fix */
+	if (taskList != NIL)
+	{
+		bool localExecutionSupported = true;
+		RowModifyLevel modLevel = ROW_MODIFY_NONE;
+		ExecutionParams *execParams = CreateBasicExecutionParams(modLevel, taskList,
+																 MaxAdaptiveExecutorPoolSize,
+																 localExecutionSupported);
+		ExecuteTaskListExtended(execParams);
+	}
+
+	relation_close(relation, NoLock);
+
+	PG_RETURN_VOID();
+}
+
+
+/*
+ * worker_fix_partition_shard_index_names fixes the index name of the index on given
+ * partition shard that has parent the given parent index.
+ * The parent index should be the index of a shard of a distributed partitioned table.
+ */
+Datum
+worker_fix_partition_shard_index_names(PG_FUNCTION_ARGS)
+{
+	Oid parentShardIndexId = PG_GETARG_OID(0);
+
+	text *partitionShardName = PG_GETARG_TEXT_P(1);
+
+	/* resolve partitionShardId from passed in schema and partition shard name */
+	List *partitionShardNameList = textToQualifiedNameList(partitionShardName);
+	RangeVar *partitionShard = makeRangeVarFromNameList(partitionShardNameList);
+
+	/* lock the relation with the lock mode */
+	bool missing_ok = true;
+	Oid partitionShardId = RangeVarGetRelid(partitionShard, NoLock, missing_ok);
+
+	if (!OidIsValid(partitionShardId))
+	{
+		PG_RETURN_VOID();
+	}
+
+	CheckCitusVersion(ERROR);
+	EnsureTableOwner(partitionShardId);
+
+	text *newPartitionShardIndexNameText = PG_GETARG_TEXT_P(2);
+	char *newPartitionShardIndexName = text_to_cstring(
+		newPartitionShardIndexNameText);
+
+	if (!has_subclass(parentShardIndexId))
+	{
+		ereport(ERROR, (errmsg("could not fix child index names: "
+							   "index is not partitioned")));
+	}
+
+	List *partitionShardIndexIds = find_inheritance_children(parentShardIndexId,
+															 ShareRowExclusiveLock);
+	Oid partitionShardIndexId = InvalidOid;
+	foreach_oid(partitionShardIndexId, partitionShardIndexIds)
+	{
+		if (IndexGetRelation(partitionShardIndexId, false) == partitionShardId)
+		{
+			char *partitionShardIndexName = get_rel_name(partitionShardIndexId);
+			if (ExtractShardIdFromTableName(partitionShardIndexName, missing_ok) ==
+				INVALID_SHARD_ID)
+			{
+				/*
+				 * ExtractShardIdFromTableName will return INVALID_SHARD_ID if
+				 * partitionShardIndexName doesn't end in _shardid. In that case,
+				 * we want to rename this partition shard index to newPartitionShardIndexName,
+				 * which ends in _shardid, hence we maintain naming consistency:
+				 * we can reach this partition shard index by conventional Citus naming
+				 */
+				RenameStmt *stmt = makeNode(RenameStmt);
+
+				stmt->renameType = OBJECT_INDEX;
+				stmt->missing_ok = false;
+				char *idxNamespace = get_namespace_name(get_rel_namespace(
+															partitionShardIndexId));
+				stmt->relation = makeRangeVar(idxNamespace, partitionShardIndexName, -1);
+				stmt->newname = newPartitionShardIndexName;
+
+				RenameRelation(stmt);
+			}
+			break;
+		}
+	}
+
+	PG_RETURN_VOID();
+}
+
+
 /*
  * CreateFixPartitionConstraintsTaskList goes over all the partitions of a distributed
  * partitioned table, and creates the list of tasks to execute
@@ -257,6 +435,199 @@ WorkerFixPartitionConstraintCommandList(Oid relationId, uint64 shardId,
 }
 
 
+/*
+ * CreateFixPartitionShardIndexNamesTaskList goes over all the indexes of a distributed
+ * partitioned table, and creates the list of tasks to execute
+ * worker_fix_partition_shard_index_names UDF on worker nodes.
+ *
+ * We create parent_table_shard_count tasks,
+ * each task with parent_indexes_count x parent_partitions_count query strings.
+ */
+static List *
+CreateFixPartitionShardIndexNamesTaskList(Oid parentRelationId)
+{
+	List *taskList = NIL;
+
+	/* enumerate the tasks when putting them to the taskList */
+	int taskId = 1;
+
+	Relation parentRelation = RelationIdGetRelation(parentRelationId);
+
+	List *parentIndexIdList = RelationGetIndexList(parentRelation);
+
+	/* early exit if the parent relation does not have any indexes */
+	if (parentIndexIdList == NIL)
+	{
+		RelationClose(parentRelation);
+		return NIL;
+	}
+
+	List *partitionList = PartitionList(parentRelationId);
+
+	/* early exit if the parent relation does not have any partitions */
+	if (partitionList == NIL)
+	{
+		RelationClose(parentRelation);
+		return NIL;
+	}
+
+	List *parentShardIntervalList = LoadShardIntervalList(parentRelationId);
+
+	/* lock metadata before getting placement lists */
+	LockShardListMetadata(parentShardIntervalList, ShareLock);
+	Oid partitionId = InvalidOid;
+	foreach_oid(partitionId, partitionList)
+	{
+		List *partitionShardIntervalList = LoadShardIntervalList(partitionId);
+		LockShardListMetadata(partitionShardIntervalList, ShareLock);
+	}
+
+	ShardInterval *parentShardInterval = NULL;
+	foreach_ptr(parentShardInterval, parentShardIntervalList)
+	{
+		uint64 parentShardId = parentShardInterval->shardId;
+
+		List *queryStringList = WorkerFixPartitionShardIndexNamesCommandList(
+			parentShardId, parentIndexIdList);
+
+		Task *task = CitusMakeNode(Task);
+		task->jobId = INVALID_JOB_ID;
+		task->taskId = taskId++;
+
+		task->taskType = DDL_TASK;
+		SetTaskQueryStringList(task, queryStringList);
+		task->dependentTaskList = NULL;
+		task->replicationModel = REPLICATION_MODEL_INVALID;
+		task->anchorShardId = parentShardId;
+		task->taskPlacementList = ActiveShardPlacementList(parentShardId);
+
+		taskList = lappend(taskList, task);
+	}
+
+	RelationClose(parentRelation);
+
+	return taskList;
+}
+
+
+/*
+ * WorkerFixPartitionShardIndexNamesCommandList creates a list of queries that will fix
+ * all child index names of parent indexes on given shard of parent partitioned table.
+ */
+static List *
+WorkerFixPartitionShardIndexNamesCommandList(uint64 parentShardId,
+											 List *parentIndexIdList)
+{
+	List *commandList = NIL;
+	Oid parentIndexId = InvalidOid;
+	foreach_oid(parentIndexId, parentIndexIdList)
+	{
+		if (!has_subclass(parentIndexId))
+		{
+			continue;
+		}
+
+		/*
+		 * Get the qualified name of the corresponding index of given parent index
+		 * in the parent shard with given parentShardId
+		 */
+		char *parentIndexName = get_rel_name(parentIndexId);
+		char *parentShardIndexName = pstrdup(parentIndexName);
+		AppendShardIdToName(&parentShardIndexName, parentShardId);
+		Oid schemaId = get_rel_namespace(parentIndexId);
+		char *schemaName = get_namespace_name(schemaId);
+		char *qualifiedParentShardIndexName = quote_qualified_identifier(schemaName,
+																		 parentShardIndexName);
+		List *commands = WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex(
+			qualifiedParentShardIndexName, parentIndexId);
+		commandList = list_concat(commandList, commands);
+	}
+
+	return commandList;
+}
+
+
+/*
+ * WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex creates a list of queries that will fix
+ * all child index names of given index on shard of parent partitioned table.
+ */
+static List *
+WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex(
+	char *qualifiedParentShardIndexName, Oid parentIndexId)
+{
+	List *commandList = NIL;
+
+	/*
+	 * Get the list of all partition indexes that are children of current
+	 * index on parent
+	 */
+	List *partitionIndexIds = find_inheritance_children(parentIndexId,
+														ShareRowExclusiveLock);
+	Oid partitionIndexId = InvalidOid;
+	foreach_oid(partitionIndexId, partitionIndexIds)
+	{
+		List *commands = WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(
+			partitionIndexId, qualifiedParentShardIndexName);
+		commandList = list_concat(commandList, commands);
+	}
+	return commandList;
+}
+
+
+/*
+ * WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex creates a list of queries that will fix
+ * all child index names of given index on shard of parent partitioned table, whose table relation is a shard
+ * of the partition that is the table relation of given partitionIndexId
+ */
+static List *
+WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(Oid partitionIndexId,
+															  char *
+															  qualifiedParentShardIndexName)
+{
+	List *commandList = NIL;
+
+	/* get info for this partition relation of this index*/
+	char *partitionIndexName = get_rel_name(partitionIndexId);
+	Oid partitionId = IndexGetRelation(partitionIndexId, false);
+	char *partitionName = get_rel_name(partitionId);
+	char *partitionSchemaName = get_namespace_name(get_rel_namespace(partitionId));
+	List *partitionShardIntervalList = LoadShardIntervalList(partitionId);
+
+	ShardInterval *partitionShardInterval = NULL;
+	foreach_ptr(partitionShardInterval, partitionShardIntervalList)
+	{
+		/*
+		 * Prepare commands for each shard of current partition
+		 * to fix the index name that corresponds to the
+		 * current parent index name
+		 */
+		uint64 partitionShardId = partitionShardInterval->shardId;
+
+		/* get qualified partition shard name */
+		char *partitionShardName = pstrdup(partitionName);
+		AppendShardIdToName(&partitionShardName, partitionShardId);
+		char *qualifiedPartitionShardName = quote_qualified_identifier(
+			partitionSchemaName,
+			partitionShardName);
+
+		/* generate the new correct index name */
+		char *newPartitionShardIndexName = pstrdup(partitionIndexName);
+		AppendShardIdToName(&newPartitionShardIndexName, partitionShardId);
+
+		/* create worker_fix_partition_shard_index_names command */
+		StringInfo shardQueryString = makeStringInfo();
+		appendStringInfo(shardQueryString,
+						 "SELECT worker_fix_partition_shard_index_names(%s::regclass, %s, %s)",
+						 quote_literal_cstr(qualifiedParentShardIndexName),
+						 quote_literal_cstr(qualifiedPartitionShardName),
+						 quote_literal_cstr(newPartitionShardIndexName));
+		commandList = lappend(commandList, shardQueryString->data);
+	}
+
+	return commandList;
+}
+
+
 /*
  * RelationHasConstraint checks if a relation has a constraint with a given name.
  */

src/backend/distributed/utils/reference_table_utils.c

@@ -60,8 +60,16 @@ PG_FUNCTION_INFO_V1(replicate_reference_tables);
 Datum
 replicate_reference_tables(PG_FUNCTION_ARGS)
 {
+	/* to prevent concurrent node additions while copying reference tables */
+	LockRelationOid(DistNodeRelationId(), ShareLock);
 	EnsureReferenceTablesExistOnAllNodes();
 
+	/*
+	 * Given the copying of reference tables and updating metadata have been done via a
+	 * loopback connection we do not have to retain the lock on pg_dist_node anymore.
+	 */
+	UnlockRelationOid(DistNodeRelationId(), ShareLock);
+
 	PG_RETURN_VOID();
 }
 
@@ -91,56 +99,85 @@ EnsureReferenceTablesExistOnAllNodes(void)
 void
 EnsureReferenceTablesExistOnAllNodesExtended(char transferMode)
 {
-	/*
+	List *referenceTableIdList = NIL;
-	 * Prevent this function from running concurrently with itself.
+	uint64 shardId = INVALID_SHARD_ID;
-	 *
+	List *newWorkersList = NIL;
-	 * It also prevents concurrent DROP TABLE or DROP SCHEMA. We need this
+	const char *referenceTableName = NULL;
-	 * because through-out this function we assume values in referenceTableIdList
-	 * are still valid.
-	 *
-	 * We don't need to handle other kinds of reference table DML/DDL here, since
-	 * master_copy_shard_placement gets enough locks for that.
-	 *
-	 * We also don't need special handling for concurrent create_refernece_table.
-	 * Since that will trigger a call to this function from another backend,
-	 * which will block until our call is finished.
-	 */
 	int colocationId = CreateReferenceTableColocationId();
-	LockColocationId(colocationId, ExclusiveLock);
-
-	List *referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE);
-	if (referenceTableIdList == NIL)
-	{
-		/* no reference tables exist */
-		UnlockColocationId(colocationId, ExclusiveLock);
-		return;
-	}
-
-	Oid referenceTableId = linitial_oid(referenceTableIdList);
-	const char *referenceTableName = get_rel_name(referenceTableId);
-	List *shardIntervalList = LoadShardIntervalList(referenceTableId);
-	if (list_length(shardIntervalList) == 0)
-	{
-		/* check for corrupt metadata */
-		ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard",
-							   referenceTableName)));
-	}
-
-	ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
-	uint64 shardId = shardInterval->shardId;
 
 	/*
-	 * We only take an access share lock, otherwise we'll hold up citus_add_node.
+	 * Most of the time this function should result in a conclusion where we do not need
-	 * In case of create_reference_table() where we don't want concurrent writes
+	 * to copy any reference tables. To prevent excessive locking the majority of the time
-	 * to pg_dist_node, we have already acquired ShareLock on pg_dist_node.
+	 * we run our precondition checks first with a lower lock. If, after checking with the
+	 * lower lock, that we might need to copy reference tables we check with a more
+	 * aggressive and self conflicting lock. It is important to be self conflicting in the
+	 * second run to make sure that two concurrent calls to this routine will actually not
+	 * run concurrently after the initial check.
+	 *
+	 * If after two iterations of precondition checks we still find the need for copying
+	 * reference tables we exit the loop with all locks held. This will prevent concurrent
+	 * DROP TABLE and create_reference_table calls so that the list of reference tables we
+	 * operate on are stable.
+	 *
+	 * Since the changes to the reference table placements are made via loopback
+	 * connections we release the locks held at the end of this function. Due to Citus
+	 * only running transactions in READ COMMITTED mode we can be sure that other
+	 * transactions correctly find the metadata entries.
 	 */
-	List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId,
+	LOCKMODE lockmodes[] = { AccessShareLock, ExclusiveLock };
-																 AccessShareLock);
+	for (int lockmodeIndex = 0; lockmodeIndex < lengthof(lockmodes); lockmodeIndex++)
-	if (list_length(newWorkersList) == 0)
 	{
-		/* nothing to do, no need for lock */
+		LockColocationId(colocationId, lockmodes[lockmodeIndex]);
-		UnlockColocationId(colocationId, ExclusiveLock);
+
-		return;
+		referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE);
+		if (referenceTableIdList == NIL)
+		{
+			/*
+			 * No reference tables exist, make sure that any locks obtained earlier are
+			 * released. It will probably not matter, but we release the locks in the
+			 * reverse order we obtained them in.
+			 */
+			for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0;
+				 releaseLockmodeIndex--)
+			{
+				UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]);
+			}
+			return;
+		}
+
+		Oid referenceTableId = linitial_oid(referenceTableIdList);
+		referenceTableName = get_rel_name(referenceTableId);
+		List *shardIntervalList = LoadShardIntervalList(referenceTableId);
+		if (list_length(shardIntervalList) == 0)
+		{
+			/* check for corrupt metadata */
+			ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard",
+								   referenceTableName)));
+		}
+
+		ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
+		shardId = shardInterval->shardId;
+
+		/*
+		 * We only take an access share lock, otherwise we'll hold up citus_add_node.
+		 * In case of create_reference_table() where we don't want concurrent writes
+		 * to pg_dist_node, we have already acquired ShareLock on pg_dist_node.
+		 */
+		newWorkersList = WorkersWithoutReferenceTablePlacement(shardId, AccessShareLock);
+		if (list_length(newWorkersList) == 0)
+		{
+			/*
+			 * All workers alreaddy have a copy of the reference tables, make sure that
+			 * any locks obtained earlier are released. It will probably not matter, but
+			 * we release the locks in the reverse order we obtained them in.
+			 */
+			for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0;
+				 releaseLockmodeIndex--)
+			{
+				UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]);
+			}
+			return;
+		}
 	}
 
 	/*
@@ -221,10 +258,17 @@ EnsureReferenceTablesExistOnAllNodesExtended(char transferMode)
 	}
 
 	/*
-	 * Unblock other backends, they will probably observe that there are no
+	 * Since reference tables have been copied via a loopback connection we do not have to
-	 * more worker nodes without placements, unless nodes were added concurrently
+	 * retain our locks. Since Citus only runs well in READ COMMITTED mode we can be sure
+	 * that other transactions will find the reference tables copied.
+	 * We have obtained and held multiple locks, here we unlock them all in the reverse
+	 * order we have obtained them in.
 	 */
-	UnlockColocationId(colocationId, ExclusiveLock);
+	for (int releaseLockmodeIndex = lengthof(lockmodes) - 1; releaseLockmodeIndex >= 0;
+		 releaseLockmodeIndex--)
+	{
+		UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]);
+	}
 }
 
 
@@ -425,6 +469,28 @@ CreateReferenceTableColocationId()
 }
 
 
+uint32
+GetReferenceTableColocationId()
+{
+	int shardCount = 1;
+	Oid distributionColumnType = InvalidOid;
+	Oid distributionColumnCollation = InvalidOid;
+
+	/*
+	 * We don't maintain replication factor of reference tables anymore and
+	 * just use -1 instead. We don't use this value in any places.
+	 */
+	int replicationFactor = -1;
+
+	/* check for existing colocations */
+	uint32 colocationId =
+		ColocationId(shardCount, replicationFactor, distributionColumnType,
+					 distributionColumnCollation);
+
+	return colocationId;
+}
+
+
 /*
  * DeleteAllReferenceTablePlacementsFromNodeGroup function iterates over list of reference
  * tables and deletes all reference table placements from pg_dist_placement table
@@ -504,19 +570,6 @@ CompareOids(const void *leftElement, const void *rightElement)
 }
 
 
-/*
- * ReferenceTableReplicationFactor returns the replication factor for
- * reference tables.
- */
-int
-ReferenceTableReplicationFactor(void)
-{
-	List *nodeList = ReferenceTablePlacementNodeList(NoLock);
-	int replicationFactor = list_length(nodeList);
-	return replicationFactor;
-}
-
-
 /*
  * ReplicateAllReferenceTablesToNode function finds all reference tables and
  * replicates them to the given worker node. It also modifies pg_dist_colocation
@@ -527,6 +580,16 @@ ReferenceTableReplicationFactor(void)
 void
 ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort)
 {
+	int colocationId = GetReferenceTableColocationId();
+	if (colocationId == INVALID_COLOCATION_ID)
+	{
+		/* no reference tables in system */
+		return;
+	}
+
+	/* prevent changes in table set while replicating reference tables */
+	LockColocationId(colocationId, RowExclusiveLock);
+
 	List *referenceTableList = CitusTableTypeIdList(REFERENCE_TABLE);
 
 	/* if there is no reference table, we do not need to replicate anything */

src/backend/distributed/utils/resource_lock.c

@@ -183,20 +183,49 @@ lock_shard_resources(PG_FUNCTION_ARGS)
 	int shardIdCount = ArrayObjectCount(shardIdArrayObject);
 	Datum *shardIdArrayDatum = DeconstructArrayObject(shardIdArrayObject);
 
+	/*
+	 * The executor calls this UDF for modification queries. So, any user
+	 * who has the the rights to modify this table are actually able
+	 * to call the UDF.
+	 *
+	 * So, at this point, we make sure that any malicious user who doesn't
+	 * have modification privileges to call this UDF.
+	 *
+	 * Update/Delete/Truncate commands already acquires ExclusiveLock
+	 * on the executor. However, for INSERTs, the user might have only
+	 * INSERTs granted, so add a special case for it.
+	 */
+	AclMode aclMask = ACL_UPDATE | ACL_DELETE | ACL_TRUNCATE;
+	if (lockMode == RowExclusiveLock)
+	{
+		aclMask |= ACL_INSERT;
+	}
+
 	for (int shardIdIndex = 0; shardIdIndex < shardIdCount; shardIdIndex++)
 	{
 		int64 shardId = DatumGetInt64(shardIdArrayDatum[shardIdIndex]);
 
 		/*
-		 * We don't want random users to block writes. The callers of this
+		 * We don't want random users to block writes. If the current user
-		 * function either operates on all the colocated placements, such
+		 * has privileges to modify the shard, then the user can already
-		 * as shard moves, or requires superuser such as adding node.
+		 * acquire the lock. So, we allow.
-		 * In other words, the coordinator initiated operations has already
-		 * ensured table owner, we are preventing any malicious attempt to
-		 * use this function.
 		 */
 		bool missingOk = true;
-		EnsureShardOwner(shardId, missingOk);
+		Oid relationId = LookupShardRelationFromCatalog(shardId, missingOk);
+
+		if (!OidIsValid(relationId) && missingOk)
+		{
+			/*
+			 * This could happen in two ways. First, a malicious user is trying
+			 * to acquire locks on non-existing shards. Second, the metadata has
+			 * not been synced (or not yet visible) to this node. In the second
+			 * case, there is no point in locking the shards because no other
+			 * transaction can be accessing the table.
+			 */
+			continue;
+		}
+
+		EnsureTablePermissions(relationId, aclMask);
 
 		LockShardResource(shardId, lockMode);
 	}

src/backend/distributed/utils/shardinterval_utils.c

@@ -297,7 +297,7 @@ FindShardIntervalIndex(Datum searchedValue, CitusTableCacheEntry *cacheEntry)
 	ShardInterval **shardIntervalCache = cacheEntry->sortedShardIntervalArray;
 	int shardCount = cacheEntry->shardIntervalArrayLength;
 	FmgrInfo *compareFunction = cacheEntry->shardIntervalCompareFunction;
-	bool useBinarySearch = (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) ||
+	bool useBinarySearch = (!IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) ||
 							!cacheEntry->hasUniformHashDistribution);
 	int shardIndex = INVALID_SHARD_INDEX;
 

src/backend/distributed/worker/worker_create_or_replace.c

@@ -61,7 +61,7 @@ WrapCreateOrReplace(const char *sql)
  * have this functionality or where their implementation is not sufficient.
  *
  * Besides checking if an object of said name exists it tries to compare the object to be
- * created with the one in the local catalog. If there is a difference the on in the local
+ * created with the one in the local catalog. If there is a difference the one in the local
  * catalog will be renamed after which the statement can be executed on this worker to
  * create the object.
  *

src/include/citus_config.h.in

@@ -38,7 +38,7 @@
 #undef HAVE_LIBCURL
 
 /* Define to 1 if you have the `lz4' library (-llz4). */
-#undef HAVE_LIBLZ4
+#undef HAVE_CITUS_LIBLZ4
 
 /* Define to 1 if you have the `zstd' library (-lzstd). */
 #undef HAVE_LIBZSTD

src/include/citus_version.h.in

@@ -25,7 +25,7 @@
 #undef HAVE_LIBCURL
 
 /* Define to 1 if you have the `liblz4' library (-llz4). */
-#undef HAVE_LIBLZ4
+#undef HAVE_CITUS_LIBLZ4
 
 /* Define to 1 if you have the `libzstd' library (-lzstd). */
 #undef HAVE_LIBZSTD

src/include/distributed/connection_management.h

@@ -16,6 +16,7 @@
 #include "distributed/transaction_management.h"
 #include "distributed/remote_transaction.h"
 #include "lib/ilist.h"
+#include "pg_config.h"
 #include "portability/instr_time.h"
 #include "utils/guc.h"
 #include "utils/hsearch.h"
@@ -264,5 +265,7 @@ extern void MarkConnectionConnected(MultiConnection *connection);
 extern double MillisecondsPassedSince(instr_time moment);
 extern long MillisecondsToTimeout(instr_time start, long msAfterStart);
 
+#if PG_VERSION_NUM < 140000
 extern void WarmUpConnParamsHash(void);
+#endif
 #endif /* CONNECTION_MANAGMENT_H */

src/include/distributed/deparse_shard_query.h

@@ -29,6 +29,7 @@ extern void SetTaskQueryStringList(Task *task, List *queryStringList);
 extern char * TaskQueryString(Task *task);
 extern char * TaskQueryStringAtIndex(Task *task, int index);
 extern int GetTaskQueryType(Task *task);
+extern void AddInsertAliasIfNeeded(Query *query);
 
 
 #endif /* DEPARSE_SHARD_QUERY_H */

src/include/distributed/insert_select_executor.h

@@ -19,7 +19,6 @@
 extern bool EnableRepartitionedInsertSelect;
 
 extern TupleTableSlot * NonPushableInsertSelectExecScan(CustomScanState *node);
-extern Query * BuildSelectForInsertSelect(Query *insertSelectQuery);
 extern bool IsSupportedRedistributionTarget(Oid targetRelationId);
 extern bool IsRedistributablePlan(Plan *selectPlan);
 

src/include/distributed/intermediate_results.h

@@ -57,7 +57,7 @@ extern void WriteToLocalFile(StringInfo copyData, FileCompat *fileCompat);
 extern uint64 RemoteFileDestReceiverBytesSent(DestReceiver *destReceiver);
 extern void SendQueryResultViaCopy(const char *resultId);
 extern void ReceiveQueryResultViaCopy(const char *resultId);
-extern void RemoveIntermediateResultsDirectory(void);
+extern void RemoveIntermediateResultsDirectories(void);
 extern int64 IntermediateResultSize(const char *resultId);
 extern char * QueryResultFileName(const char *resultId);
 extern char * CreateIntermediateResultsDirectory(void);

src/include/distributed/metadata/dependency.h

@@ -15,6 +15,8 @@
 #include "postgres.h"
 
 #include "catalog/objectaddress.h"
+#include "catalog/pg_depend.h"
+#include "distributed/errormessage.h"
 #include "nodes/pg_list.h"
 
 extern List * GetUniqueDependenciesList(List *objectAddressesList);
@@ -24,5 +26,6 @@ extern bool SupportedDependencyByCitus(const ObjectAddress *address);
 extern List * GetPgDependTuplesForDependingObjects(Oid targetObjectClassId,
 												   Oid targetObjectId);
 extern List * GetDependingViews(Oid relationId);
+extern Oid GetDependingView(Form_pg_depend pg_depend);
 
 #endif /* CITUS_DEPENDENCY_H */

src/include/distributed/metadata_utility.h

@@ -291,7 +291,7 @@ extern bool GetNodeDiskSpaceStatsForConnection(MultiConnection *connection,
 											   uint64 *availableBytes,
 											   uint64 *totalBytes);
 extern void ExecuteQueryViaSPI(char *query, int SPIOK);
-extern void EnsureSequenceTypeSupported(Oid seqOid, Oid seqTypId);
+extern void EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId);
 extern void AlterSequenceType(Oid seqOid, Oid typeOid);
 extern void MarkSequenceListDistributedAndPropagateDependencies(List *sequenceList);
 extern void MarkSequenceDistributedAndPropagateDependencies(Oid sequenceOid);

src/include/distributed/reference_table_utils.h

@@ -21,6 +21,7 @@
 extern void EnsureReferenceTablesExistOnAllNodes(void);
 extern void EnsureReferenceTablesExistOnAllNodesExtended(char transferMode);
 extern uint32 CreateReferenceTableColocationId(void);
+extern uint32 GetReferenceTableColocationId(void);
 extern void DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId);
 extern int CompareOids(const void *leftElement, const void *rightElement);
 extern int ReferenceTableReplicationFactor(void);

src/test/columnar_freezing/Makefile

@@ -0,0 +1,37 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/test/columnar_freezing
+#
+# Test that columnar freezing works.
+#
+#-------------------------------------------------------------------------
+
+subdir = src/test/columnar_freezing
+top_builddir = ../../..
+include $(top_builddir)/Makefile.global
+
+# copied from pgxs/Makefile.global to use postgres' abs build dir for pg_regress
+ifeq ($(enable_tap_tests),yes)
+
+define citus_prove_installcheck
+rm -rf '$(CURDIR)'/tmp_check
+$(MKDIR_P) '$(CURDIR)'/tmp_check
+cd $(srcdir) && \
+TESTDIR='$(CURDIR)' \
+PATH="$(bindir):$$PATH" \
+PGPORT='6$(DEF_PGPORT)' \
+top_builddir='$(CURDIR)/$(top_builddir)' \
+PG_REGRESS='$(pgxsdir)/src/test/regress/pg_regress' \
+TEMP_CONFIG='$(CURDIR)'/postgresql.conf \
+$(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl)
+endef
+
+else
+citus_prove_installcheck = @echo "TAP tests not enabled when postgres was compiled"
+endif
+
+installcheck:
+	$(citus_prove_installcheck)
+
+clean distclean maintainer-clean:
+	rm -rf tmp_check

src/test/columnar_freezing/postgresql.conf

@@ -0,0 +1,7 @@
+shared_preload_libraries=citus
+shared_preload_libraries='citus'
+vacuum_freeze_min_age = 50000
+vacuum_freeze_table_age = 50000
+synchronous_commit = off
+fsync = off
+

src/test/columnar_freezing/t/001_columnar_freezing.pl

@@ -0,0 +1,52 @@
+# Minimal test testing streaming replication
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 2;
+
+# Initialize single node
+my $node_one = get_new_node('node_one');
+$node_one->init();
+$node_one->start;
+
+# initialize the citus extension
+$node_one->safe_psql('postgres', "CREATE EXTENSION citus;");
+
+# create columnar table and insert simple data to verify the data survives a crash
+$node_one->safe_psql('postgres', "
+CREATE TABLE test_row(i int);
+INSERT INTO test_row VALUES (1);
+CREATE TABLE test_columnar_freeze(i int) USING columnar WITH(autovacuum_enabled=false);
+INSERT INTO test_columnar_freeze VALUES (1);
+");
+
+my $ten_thousand_updates = "";
+
+foreach (1..10000) {
+    $ten_thousand_updates .= "UPDATE test_row SET i = i + 1;\n";
+}
+
+# 70K updates
+foreach (1..7) {
+    $node_one->safe_psql('postgres', $ten_thousand_updates);
+}
+
+my $result = $node_one->safe_psql('postgres', "
+select age(relfrozenxid) < 70000 as was_frozen
+  from pg_class where relname='test_columnar_freeze';
+");
+print "node one count: $result\n";
+is($result, qq(f), 'columnar table was not frozen');
+
+$node_one->safe_psql('postgres', 'VACUUM FREEZE test_columnar_freeze;');
+
+$result = $node_one->safe_psql('postgres', "
+select age(relfrozenxid) < 70000 as was_frozen
+  from pg_class where relname='test_columnar_freeze';
+");
+print "node one count: $result\n";
+is($result, qq(t), 'columnar table was frozen');
+
+$node_one->stop('fast');
+

src/test/regress/after_pg_upgrade_schedule

@@ -1 +1,5 @@
-test: upgrade_basic_after upgrade_columnar_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects
+test: upgrade_basic_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects
+
+# This attempts dropping citus extension (and rollbacks), so please do
+# not run in parallel with any other tests.
+test: upgrade_columnar_after

src/test/regress/before_pg_upgrade_schedule

@@ -2,7 +2,10 @@
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views
 test: upgrade_basic_before
-test: upgrade_columnar_before
 test: upgrade_ref2ref_before
 test: upgrade_type_before
 test: upgrade_distributed_function_before upgrade_rebalance_strategy_before
+
+# upgrade_columnar_before renames public schema to citus_schema, so let's
+# run this test as the last one.
+test: upgrade_columnar_before

src/test/regress/bin/normalize.sed

@@ -246,3 +246,6 @@ s/TRIM\(BOTH FROM value\)/btrim\(value\)/g
 s/pg14\.idx.*/pg14\.xxxxx/g
 
 s/CREATE TABLESPACE test_tablespace LOCATION.*/CREATE TABLESPACE test_tablespace LOCATION XXXX/g
+
+# columnar log for var correlation
+s/(.*absolute correlation \()([0,1]\.[0-9]+)(\) of var attribute [0-9]+ is smaller than.*)/\1X\.YZ\3/g

src/test/regress/expected/aggregate_support.out

@@ -712,6 +712,19 @@ select array_agg(val order by valf) from aggdata;
  {0,NULL,2,3,5,2,4,NULL,NULL,8,NULL}
 (1 row)
 
+-- test by using some other node types as arguments to agg
+select key, percentile_cont((key - (key > 4)::int) / 10.0) within group(order by val) from aggdata group by key;
+ key | percentile_cont
+---------------------------------------------------------------------
+   1 |               2
+   2 |             2.4
+   3 |               4
+   5 |
+   6 |
+   7 |               8
+   9 |               0
+(7 rows)
+
 -- Test TransformSubqueryNode
 select * FROM (
     SELECT key, mode() within group (order by floor(agg1.val/2)) m from aggdata agg1
@@ -932,5 +945,100 @@ SELECT square_func(5), a, count(a) FROM t1 GROUP BY a;
 ERROR:  function aggregate_support.square_func(integer) does not exist
 HINT:  No function matches the given name and argument types. You might need to add explicit type casts.
 CONTEXT:  while executing command on localhost:xxxxx
+-- Test the cases where the worker agg exec. returns no tuples.
+CREATE TABLE dist_table (dist_col int, agg_col numeric);
+SELECT create_distributed_table('dist_table', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE ref_table (int_col int);
+SELECT create_reference_table('ref_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col)
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ percentile_disc
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col)
+FROM (SELECT *, random() FROM dist_table) a;
+ percentile_disc
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT PERCENTILE_DISC((2 > random())::int::numeric / 10) WITHIN GROUP (ORDER BY agg_col)
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ percentile_disc
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT SUM(COALESCE(agg_col, 3))
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ sum
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT AVG(COALESCE(agg_col, 10))
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ avg
+---------------------------------------------------------------------
+
+(1 row)
+
+insert into dist_table values (2, 11.2), (3, NULL), (6, 3.22), (3, 4.23), (5, 5.25), (4, 63.4), (75, NULL), (80, NULL), (96, NULL), (8, 1078), (0, 1.19);
+-- run the same queries after loading some data
+SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col)
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ percentile_disc
+---------------------------------------------------------------------
+            3.22
+(1 row)
+
+SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col)
+FROM (SELECT *, random() FROM dist_table) a;
+ percentile_disc
+---------------------------------------------------------------------
+            3.22
+(1 row)
+
+SELECT PERCENTILE_DISC((2 > random())::int::numeric / 10) WITHIN GROUP (ORDER BY agg_col)
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ percentile_disc
+---------------------------------------------------------------------
+            1.19
+(1 row)
+
+SELECT floor(SUM(COALESCE(agg_col, 3)))
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ floor
+---------------------------------------------------------------------
+  1178
+(1 row)
+
+SELECT floor(AVG(COALESCE(agg_col, 10)))
+FROM dist_table
+LEFT JOIN ref_table ON TRUE;
+ floor
+---------------------------------------------------------------------
+   109
+(1 row)
+
 set client_min_messages to error;
 drop schema aggregate_support cascade;

src/test/regress/expected/columnar_chunk_filtering.out

@@ -645,7 +645,7 @@ alter table coltest add column x5 int default (random()*20000)::int;
 analyze coltest;
 -- test that expressions on whole-row references are not pushed down
 select * from coltest where coltest = (1,1,1,1);
-NOTICE:  columnar planner: cannot push down clause: var is whole-row reference
+NOTICE:  columnar planner: cannot push down clause: var is whole-row reference or system column
 NOTICE:  columnar planner: adding CustomScan path for coltest
 DETAIL:  unparameterized; 0 clauses pushed down
  id | x1 | x2 | x3 | x5
@@ -655,7 +655,7 @@ DETAIL:  unparameterized; 0 clauses pushed down
 -- test that expressions on uncorrelated attributes are not pushed down
 set columnar.qual_pushdown_correlation to default;
 select * from coltest where x5 = 23484;
-NOTICE:  columnar planner: cannot push down clause: var attribute 5 is uncorrelated
+NOTICE:  columnar planner: cannot push down clause: absolute correlation (X.YZ) of var attribute 5 is smaller than the value configured in "columnar.qual_pushdown_correlation_threshold" (0.900)
 NOTICE:  columnar planner: adding CustomScan path for coltest
 DETAIL:  unparameterized; 0 clauses pushed down
  id | x1 | x2 | x3 | x5
@@ -819,3 +819,250 @@ select * from numrange_test natural join numrange_test2 order by nr;
 
 DROP TABLE atest1, atest2, t1, t2, t3, numrange_test, numrange_test2;
 set default_table_access_method to default;
+set columnar.planner_debug_level to notice;
+BEGIN;
+  SET LOCAL columnar.stripe_row_limit = 2000;
+  SET LOCAL columnar.chunk_group_row_limit = 1000;
+  create table pushdown_test (a int, b int) using columnar;
+  insert into pushdown_test values (generate_series(1, 200000));
+COMMIT;
+SET columnar.max_custom_scan_paths TO 50;
+SET columnar.qual_pushdown_correlation_threshold TO 0.0;
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                     QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=2 loops=1)
+         Filter: ((a = 204356) OR (a = 104356) OR (a = 76556))
+         Rows Removed by Filter: 1998
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a = 204356) OR (a = 104356) OR (a = 76556))
+         Columnar Chunk Groups Removed by Filter: 198
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+  sum
+---------------------------------------------------------------------
+ 180912
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                     QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1)
+         Filter: ((a = 194356) OR (a = 104356) OR (a = 76556))
+         Rows Removed by Filter: 2997
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a = 194356) OR (a = 104356) OR (a = 76556))
+         Columnar Chunk Groups Removed by Filter: 197
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+  sum
+---------------------------------------------------------------------
+ 375268
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a > a*-1 + b;
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+                                QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1)
+         Filter: ((a = 204356) OR (a > ((a * '-1'::integer) + b)))
+         Rows Removed by Filter: 200000
+         Columnar Projected Columns: a, b
+(5 rows)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                              QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=38998 loops=1)
+         Filter: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000)))
+         Rows Removed by Filter: 2
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000)))
+         Columnar Chunk Groups Removed by Filter: 161
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 1099459500
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+                                      QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=200000 loops=1)
+         Filter: ((((a)::double precision > random()) AND (a < (2 * a))) OR (a > 100))
+         Columnar Projected Columns: a
+(4 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+     sum
+---------------------------------------------------------------------
+ 20000100000
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                       QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3010 loops=1)
+         Filter: ((((a)::double precision > random()) AND (a <= 2000)) OR (a > 198990))
+         Rows Removed by Filter: 990
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a <= 2000) OR (a > 198990))
+         Columnar Chunk Groups Removed by Filter: 196
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 203491455
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where
+(
+  a > random()
+  and
+  (
+    (a < 200 and a not in (select a from pushdown_test)) or
+    (a > 1000 and a < 2000)
+  )
+)
+or
+(a > 200000-2010);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must not contain a subplan
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                                                  QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3009 loops=1)
+         Filter: ((((a)::double precision > random()) AND (((a < 200) AND (NOT (SubPlan 1))) OR ((a > 1000) AND (a < 2000)))) OR (a > 197990))
+         Rows Removed by Filter: 1991
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (((a < 200) OR ((a > 1000) AND (a < 2000))) OR (a > 197990))
+         Columnar Chunk Groups Removed by Filter: 195
+         SubPlan 1
+           ->  Materialize (actual rows=100 loops=199)
+                 ->  Custom Scan (ColumnarScan) on pushdown_test pushdown_test_1 (actual rows=199 loops=1)
+                       Columnar Projected Columns: a
+(11 rows)
+
+SELECT sum(a) FROM pushdown_test where
+(
+  a > random()
+  and
+  (
+    (a < 200 and a not in (select a from pushdown_test)) or
+    (a > 1000 and a < 2000)
+  )
+)
+or
+(a > 200000-2010);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must not contain a subplan
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 401479455
+(1 row)
+
+create function stable_1(arg int) returns int language plpgsql STRICT IMMUTABLE as
+$$ BEGIN RETURN 1+arg; END; $$;
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000));
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                        QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1)
+         Filter: ((a < 6001) AND ((a)::double precision = random()) AND (a < stable_1(a)))
+         Rows Removed by Filter: 6000
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (a < 6001)
+         Columnar Chunk Groups Removed by Filter: 194
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000));
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+ sum
+---------------------------------------------------------------------
+
+(1 row)
+
+RESET columnar.max_custom_scan_paths;
+RESET columnar.qual_pushdown_correlation_threshold;
+RESET columnar.planner_debug_level;
+DROP TABLE pushdown_test;

src/test/regress/expected/columnar_chunk_filtering_0.out

@@ -645,7 +645,7 @@ alter table coltest add column x5 int default (random()*20000)::int;
 analyze coltest;
 -- test that expressions on whole-row references are not pushed down
 select * from coltest where coltest = (1,1,1,1);
-NOTICE:  columnar planner: cannot push down clause: var is whole-row reference
+NOTICE:  columnar planner: cannot push down clause: var is whole-row reference or system column
 NOTICE:  columnar planner: adding CustomScan path for coltest
 DETAIL:  unparameterized; 0 clauses pushed down
  id | x1 | x2 | x3 | x5
@@ -655,7 +655,7 @@ DETAIL:  unparameterized; 0 clauses pushed down
 -- test that expressions on uncorrelated attributes are not pushed down
 set columnar.qual_pushdown_correlation to default;
 select * from coltest where x5 = 23484;
-NOTICE:  columnar planner: cannot push down clause: var attribute 5 is uncorrelated
+NOTICE:  columnar planner: cannot push down clause: absolute correlation (X.YZ) of var attribute 5 is smaller than the value configured in "columnar.qual_pushdown_correlation_threshold" (0.900)
 NOTICE:  columnar planner: adding CustomScan path for coltest
 DETAIL:  unparameterized; 0 clauses pushed down
  id | x1 | x2 | x3 | x5
@@ -819,3 +819,250 @@ select * from numrange_test natural join numrange_test2 order by nr;
 
 DROP TABLE atest1, atest2, t1, t2, t3, numrange_test, numrange_test2;
 set default_table_access_method to default;
+set columnar.planner_debug_level to notice;
+BEGIN;
+  SET LOCAL columnar.stripe_row_limit = 2000;
+  SET LOCAL columnar.chunk_group_row_limit = 1000;
+  create table pushdown_test (a int, b int) using columnar;
+  insert into pushdown_test values (generate_series(1, 200000));
+COMMIT;
+SET columnar.max_custom_scan_paths TO 50;
+SET columnar.qual_pushdown_correlation_threshold TO 0.0;
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                     QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=2 loops=1)
+         Filter: ((a = 204356) OR (a = 104356) OR (a = 76556))
+         Rows Removed by Filter: 1998
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a = 204356) OR (a = 104356) OR (a = 76556))
+         Columnar Chunk Groups Removed by Filter: 198
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+  sum
+---------------------------------------------------------------------
+ 180912
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                     QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1)
+         Filter: ((a = 194356) OR (a = 104356) OR (a = 76556))
+         Rows Removed by Filter: 2997
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a = 194356) OR (a = 104356) OR (a = 76556))
+         Columnar Chunk Groups Removed by Filter: 197
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556;
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+  sum
+---------------------------------------------------------------------
+ 375268
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a > a*-1 + b;
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+                                QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1)
+         Filter: ((a = 204356) OR (a > ((a * '-1'::integer) + b)))
+         Rows Removed by Filter: 200000
+         Columnar Projected Columns: a, b
+(5 rows)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                              QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=38998 loops=1)
+         Filter: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000)))
+         Rows Removed by Filter: 2
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000)))
+         Columnar Chunk Groups Removed by Filter: 161
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 1099459500
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+                                      QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=200000 loops=1)
+         Filter: ((((a)::double precision > random()) AND (a < (2 * a))) OR (a > 100))
+         Columnar Projected Columns: a
+(4 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above
+NOTICE:  columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+     sum
+---------------------------------------------------------------------
+ 20000100000
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                       QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3010 loops=1)
+         Filter: ((((a)::double precision > random()) AND (a <= 2000)) OR (a > 198990))
+         Rows Removed by Filter: 990
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: ((a <= 2000) OR (a > 198990))
+         Columnar Chunk Groups Removed by Filter: 196
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010);
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 203491455
+(1 row)
+
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where
+(
+  a > random()
+  and
+  (
+    (a < 200 and a not in (select a from pushdown_test)) or
+    (a > 1000 and a < 2000)
+  )
+)
+or
+(a > 200000-2010);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must not contain a subplan
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                                                  QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=3009 loops=1)
+         Filter: ((((a)::double precision > random()) AND (((a < 200) AND (NOT (SubPlan 1))) OR ((a > 1000) AND (a < 2000)))) OR (a > 197990))
+         Rows Removed by Filter: 1991
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (((a < 200) OR ((a > 1000) AND (a < 2000))) OR (a > 197990))
+         Columnar Chunk Groups Removed by Filter: 195
+         SubPlan 1
+           ->  Materialize (actual rows=100 loops=199)
+                 ->  Custom Scan (ColumnarScan) on pushdown_test pushdown_test_1 (actual rows=199 loops=1)
+                       Columnar Projected Columns: a
+(11 rows)
+
+SELECT sum(a) FROM pushdown_test where
+(
+  a > random()
+  and
+  (
+    (a < 200 and a not in (select a from pushdown_test)) or
+    (a > 1000 and a < 2000)
+  )
+)
+or
+(a > 200000-2010);
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 0 clauses pushed down
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must not contain a subplan
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+    sum
+---------------------------------------------------------------------
+ 401479455
+(1 row)
+
+create function stable_1(arg int) returns int language plpgsql STRICT IMMUTABLE as
+$$ BEGIN RETURN 1+arg; END; $$;
+EXPLAIN (analyze on, costs off, timing off, summary off)
+SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000));
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+                                        QUERY PLAN
+---------------------------------------------------------------------
+ Aggregate (actual rows=1 loops=1)
+   ->  Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1)
+         Filter: ((a < 6001) AND ((a)::double precision = random()) AND (a < stable_1(a)))
+         Rows Removed by Filter: 6000
+         Columnar Projected Columns: a
+         Columnar Chunk Group Filters: (a < 6001)
+         Columnar Chunk Groups Removed by Filter: 194
+(7 rows)
+
+SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000));
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: cannot push down clause: must match 'Var <op> Expr' or 'Expr <op> Var'
+HINT:  Var must only reference this rel, and Expr must not reference this rel
+NOTICE:  columnar planner: adding CustomScan path for pushdown_test
+DETAIL:  unparameterized; 1 clauses pushed down
+ sum
+---------------------------------------------------------------------
+
+(1 row)
+
+RESET columnar.max_custom_scan_paths;
+RESET columnar.qual_pushdown_correlation_threshold;
+RESET columnar.planner_debug_level;
+DROP TABLE pushdown_test;

src/test/regress/expected/columnar_indexes.out

@@ -704,5 +704,90 @@ begin;
   insert into uniq select generate_series(1,100);
 ERROR:  cannot read from index when there is unflushed data in upper transactions
 rollback;
+-- Show that we nicely ignore index deletion requests made to columnarAM.
+--
+-- An INSERT command might trigger index deletion if index already had dead
+-- entries for the key we are about to insert.
+-- There are two ways of index deletion:
+--   a) simple deletion
+--   b) bottom-up deletion (>= pg14)
+--
+-- Since columnar_index_fetch_tuple never sets all_dead to true, columnarAM
+-- doesn't expect to receive simple deletion as we don't mark any index
+-- entries as dead.
+-- Otherwise, columnarAM would throw an error for all of below six test cases.
+--
+-- However, since columnarAM doesn't delete any dead entries via simple
+-- deletion, postgres might ask for a more comprehensive deletion (bottom-up)
+-- at some point when pg >= 14.
+-- For this reason, all following six test cases would certainly trigger
+-- bottom-up deletion. Show that we gracefully ignore such requests.
+CREATE TABLE index_tuple_delete (a int UNIQUE) USING COLUMNAR;
+ALTER TABLE index_tuple_delete SET (autovacuum_enabled = false);
+BEGIN;
+  -- i) rollback before flushing
+	INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+ROLLBACK;
+-- index deletion test-1
+BEGIN;
+  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+ROLLBACK;
+COPY index_tuple_delete FROM PROGRAM 'seq 10000';
+TRUNCATE index_tuple_delete;
+BEGIN;
+  -- ii) rollback after flushing
+	INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+  SELECT SUM(a) > 0 FROM index_tuple_delete;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+ROLLBACK;
+-- index deletion test-2
+BEGIN;
+  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+ROLLBACK;
+COPY index_tuple_delete FROM PROGRAM 'seq 10000';
+TRUNCATE index_tuple_delete;
+BEGIN;
+  -- iii) rollback before flushing, use savepoint
+  SAVEPOINT sp1;
+	  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+  ROLLBACK TO sp1;
+  -- index deletion test-3
+  SAVEPOINT sp2;
+    INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+  ROLLBACK TO sp2;
+  COPY index_tuple_delete FROM PROGRAM 'seq 10000';
+ROLLBACK;
+-- index deletion test-4
+BEGIN;
+  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+ROLLBACK;
+COPY index_tuple_delete FROM PROGRAM 'seq 10000';
+TRUNCATE index_tuple_delete;
+BEGIN;
+  -- iv) rollback after flushing, use savepoint
+  SAVEPOINT sp1;
+	  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+    SELECT SUM(a) > 0 FROM index_tuple_delete;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+  ROLLBACK TO sp1;
+  -- index deletion test-5
+  SAVEPOINT sp2;
+    INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+  ROLLBACK TO sp2;
+  COPY index_tuple_delete FROM PROGRAM 'seq 10000';
+ROLLBACK;
+-- index deletion test-6
+BEGIN;
+  INSERT INTO index_tuple_delete SELECT i FROM generate_series(0,10000)i;
+ROLLBACK;
+COPY index_tuple_delete FROM PROGRAM 'seq 10000';
 SET client_min_messages TO WARNING;
 DROP SCHEMA columnar_indexes CASCADE;

src/test/regress/expected/columnar_insert.out

@@ -291,6 +291,20 @@ BEGIN;
 (1 row)
 
 ROLLBACK;
+CREATE OR REPLACE FUNCTION test_columnar_storage_write_new_page(relation regclass) RETURNS void
+STRICT LANGUAGE c AS 'citus', 'test_columnar_storage_write_new_page';
+CREATE TABLE aborted_write (a int, b int) USING columnar;
+SELECT test_columnar_storage_write_new_page('aborted_write');
+ test_columnar_storage_write_new_page
+---------------------------------------------------------------------
+
+(1 row)
+
+SET client_min_messages TO DEBUG4;
+INSERT INTO aborted_write VALUES (5);
+DEBUG:  Flushing Stripe of size 1
+DEBUG:  overwriting page 2
+DETAIL:  This can happen after a roll-back.
 RESET search_path;
 SET client_min_messages TO WARNING;
 DROP SCHEMA columnar_insert CASCADE;

src/test/regress/expected/coordinator_shouldhaveshards.out

@@ -847,8 +847,8 @@ HAVING (max(table_2.value) >= (SELECT value FROM a));
 DEBUG:  Group by list without distribution column is not allowed  in distributed INSERT ... SELECT queries
 DEBUG:  generating subplan XXX_1 for CTE a: SELECT key, value FROM coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1
 DEBUG:  push down of limit count: 1
-DEBUG:  generating subplan XXX_2 for subquery SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1))
+DEBUG:  generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1))
-DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT int4(count) AS key, (key)::text AS value FROM (SELECT intermediate_result.count, intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint, key integer)) citus_insert_select_subquery
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery
 DEBUG:  Collecting INSERT ... SELECT results on coordinator
 DEBUG:  Subplan XXX_1 will be written to local file
 DEBUG:  Subplan XXX_1 will be sent to localhost:xxxxx
@@ -856,9 +856,9 @@ DEBUG:  Subplan XXX_1 will be sent to localhost:xxxxx
 NOTICE:  executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503102 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint
 NOTICE:  executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503105 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint
 DEBUG:  Subplan XXX_2 will be written to local file
-NOTICE:  executing the command locally: SELECT count(*) AS count, worker_column_1 AS key, max(worker_column_2) AS worker_column_3 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503106 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1
+NOTICE:  executing the command locally: SELECT count(*) AS auto_coerced_by_citus_0, (worker_column_1)::text AS auto_coerced_by_citus_1, worker_column_1 AS discarded_target_item_1, max(worker_column_2) AS worker_column_4 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503106 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1
-NOTICE:  executing the command locally: SELECT count(*) AS count, worker_column_1 AS key, max(worker_column_2) AS worker_column_3 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503109 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1
+NOTICE:  executing the command locally: SELECT count(*) AS auto_coerced_by_citus_0, (worker_column_1)::text AS auto_coerced_by_citus_1, worker_column_1 AS discarded_target_item_1, max(worker_column_2) AS worker_column_4 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503109 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1
-NOTICE:  executing the command locally: SELECT int4(count) AS key, (key)::text AS value FROM (SELECT intermediate_result.count, intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint, key integer)) citus_insert_select_subquery
+NOTICE:  executing the command locally: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery
 NOTICE:  executing the copy locally for shard xxxxx
 WITH stats AS (
   SELECT count(key) m FROM table_1

src/test/regress/expected/distributed_functions.out

@@ -832,6 +832,266 @@ SELECT * FROM test ORDER BY id;
 (2 rows)
 
 DROP TABLE test;
+-- verify that recreating distributed functions with TABLE params gets propagated to workers
+CREATE OR REPLACE FUNCTION func_with_return_table(int)
+RETURNS  TABLE (date date)
+LANGUAGE plpgsql AS $$
+BEGIN
+    RETURN query SELECT '2011-01-01'::date;
+END;
+$$;
+SELECT create_distributed_function('func_with_return_table(int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION func_with_return_table(int)
+RETURNS  TABLE (date date)
+LANGUAGE plpgsql AS $$
+BEGIN
+    RETURN query SELECT '2011-01-02'::date;
+END;
+$$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'func_with_return_table';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'func_with_return_table')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed functions with OUT params gets propagated to workers
+CREATE OR REPLACE FUNCTION func_with_out_param(a int, out b int)
+  RETURNS int
+LANGUAGE sql AS $$ select 1; $$;
+SELECT create_distributed_function('func_with_out_param(int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+SET client_min_messages TO ERROR;
+CREATE ROLE r1;
+SELECT 1 FROM run_command_on_workers($$CREATE ROLE r1;$$);
+ ?column?
+---------------------------------------------------------------------
+        1
+        1
+(2 rows)
+
+GRANT EXECUTE ON FUNCTION func_with_out_param TO r1;
+SELECT 1 FROM run_command_on_workers($$GRANT EXECUTE ON FUNCTION func_with_out_param TO r1;$$);
+ ?column?
+---------------------------------------------------------------------
+        1
+        1
+(2 rows)
+
+RESET client_min_messages;
+CREATE OR REPLACE FUNCTION func_with_out_param(a int, out b int)
+  RETURNS int
+LANGUAGE sql AS $$ select 2; $$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc, pg_proc.proowner) from pg_proc where proname = 'func_with_out_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc, pg_proc.proowner)::text from pg_proc where proname = 'func_with_out_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed functions with INOUT params gets propagated to workers
+CREATE OR REPLACE FUNCTION func_with_inout_param(a int, inout b int)
+  RETURNS int
+LANGUAGE sql AS $$ select 1; $$;
+-- this should error out
+SELECT create_distributed_function('func_with_inout_param(int)');
+ERROR:  function "func_with_inout_param(int)" does not exist
+-- this should work
+SELECT create_distributed_function('func_with_inout_param(int,int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION func_with_inout_param(a int, inout b int)
+  RETURNS int
+LANGUAGE sql AS $$ select 2; $$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'func_with_inout_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'func_with_inout_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed functions with VARIADIC params gets propagated to workers
+CREATE OR REPLACE FUNCTION func_with_variadic_param(a int, variadic b int[])
+  RETURNS int
+LANGUAGE sql AS $$ select 1; $$;
+-- this should work
+SELECT create_distributed_function('func_with_variadic_param(int,int[])');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION func_with_variadic_param(a int, variadic b int[])
+  RETURNS int
+LANGUAGE sql AS $$ select 2; $$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'func_with_variadic_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'func_with_variadic_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed functions returning setof records gets propagated to workers
+CREATE OR REPLACE FUNCTION func_returning_setof_int(IN parm1 date, IN parm2 interval)
+  RETURNS SETOF integer AS
+$BODY$
+BEGIN
+    RETURN QUERY
+    SELECT 1;
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100;
+SELECT create_distributed_function('func_returning_setof_int(date,interval)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION func_returning_setof_int(IN parm1 date, IN parm2 interval)
+  RETURNS SETOF integer AS
+$BODY$
+BEGIN
+    RETURN QUERY
+    SELECT 2;
+
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'func_returning_setof_int';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'func_returning_setof_int')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed functions with variadic param returning setof records gets propagated to workers
+CREATE OR REPLACE FUNCTION func_returning_setof_int_with_variadic_param(IN parm1 date, VARIADIC parm2 int[])
+  RETURNS SETOF integer AS
+$BODY$
+BEGIN
+    RETURN QUERY
+    SELECT 1;
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100;
+SELECT create_distributed_function('func_returning_setof_int_with_variadic_param(date,int[])');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION func_returning_setof_int_with_variadic_param(IN parm1 date, VARIADIC parm2 int[])
+  RETURNS SETOF integer AS
+$BODY$
+BEGIN
+    RETURN QUERY
+    SELECT 2;
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'func_returning_setof_int_with_variadic_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'func_returning_setof_int_with_variadic_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed procedures with out params gets propagated to workers
+CREATE OR REPLACE PROCEDURE proc_with_variadic_param(IN parm1 date, VARIADIC parm2 int[])
+  LANGUAGE SQL
+AS $$
+    SELECT 1;
+$$;
+-- this should error out
+SELECT create_distributed_function('proc_with_variadic_param(date)');
+ERROR:  function "proc_with_variadic_param(date)" does not exist
+-- this should work
+SELECT create_distributed_function('proc_with_variadic_param(date,int[])');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE PROCEDURE proc_with_variadic_param(IN parm1 date, VARIADIC parm2 int[])
+  LANGUAGE SQL
+AS $$
+    SELECT 2;
+$$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'proc_with_variadic_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'proc_with_variadic_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- verify that recreating distributed procedures with INOUT param gets propagated to workers
+CREATE OR REPLACE PROCEDURE proc_with_inout_param(IN parm1 date, INOUT parm2 int)
+  LANGUAGE SQL
+AS $$
+    SELECT 1;
+$$;
+-- this should error out
+SELECT create_distributed_function('proc_with_inout_param(date)');
+ERROR:  function "proc_with_inout_param(date)" does not exist
+-- this should work
+SELECT create_distributed_function('proc_with_inout_param(date,int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE PROCEDURE proc_with_inout_param(IN parm1 date, INOUT parm2 int)
+  LANGUAGE SQL
+AS $$
+    SELECT 2;
+$$;
+SELECT count(*) FROM
+  (SELECT result FROM
+    run_command_on_workers($$select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc) from pg_proc where proname = 'proc_with_inout_param';$$)
+    UNION  select row(pg_proc.pronargs, pg_proc.proargtypes, pg_proc.prosrc)::text from pg_proc where proname = 'proc_with_inout_param')
+  as test;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
 SET client_min_messages TO error; -- suppress cascading objects dropping
 DROP SCHEMA function_tests CASCADE;
 DROP SCHEMA function_tests2 CASCADE;

src/test/regress/expected/distributed_functions_conflict.out

@@ -139,6 +139,27 @@ SELECT worker_create_or_replace_object('CREATE AGGREGATE proc_conflict.existing_
  f
 (1 row)
 
+-- test worker_create_or_replace_object with a function that returns table
+CREATE OR REPLACE FUNCTION func_with_return_table(int)
+RETURNS TABLE (date date)
+LANGUAGE plpgsql AS $$
+BEGIN
+    RETURN query SELECT '2011-01-01'::date;
+END;
+$$;
+SELECT worker_create_or_replace_object('CREATE OR REPLACE FUNCTION func_with_return_table(int) RETURNS TABLE (date date) LANGUAGE plpgsql AS $$ BEGIN RETURN query SELECT ''2011-01-01''::date; END; $$;');
+ worker_create_or_replace_object
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- verify that a backup function is created
+SELECT COUNT(*)=2 FROM pg_proc WHERE proname LIKE 'func_with_return_table%';
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
 -- hide cascades
 SET client_min_messages TO error;
 DROP SCHEMA proc_conflict CASCADE;

src/test/regress/expected/insert_select_into_local_table.out

@@ -149,6 +149,67 @@ SELECT * FROM non_dist_unique ORDER BY 1;
  5 |  8
 (5 rows)
 
+INSERT INTO non_dist_unique
+SELECT a+1, b FROM dist_table
+UNION ALL
+SELECT a+100, b FROM dist_table
+ON CONFLICT (a) DO NOTHING;
+SELECT * FROM non_dist_unique ORDER BY 1;
+  a  | b
+---------------------------------------------------------------------
+   1 |  6
+   2 |  7
+   3 | 14
+   4 | 15
+   5 |  8
+ 101 |  6
+ 102 |  7
+ 103 |  8
+(8 rows)
+
+INSERT INTO non_dist_unique
+SELECT a+1, b FROM dist_table
+UNION ALL
+SELECT a+100, b FROM dist_table
+ON CONFLICT (a) DO UPDATE SET b = EXCLUDED.b + 1;
+SELECT * FROM non_dist_unique ORDER BY 1;
+  a  | b
+---------------------------------------------------------------------
+   1 | 6
+   2 | 7
+   3 | 8
+   4 | 9
+   5 | 8
+ 101 | 7
+ 102 | 8
+ 103 | 9
+(8 rows)
+
+WITH cte1 AS (SELECT s FROM generate_series(1,10) s)
+INSERT INTO non_dist_unique
+WITH cte2 AS (SELECT s FROM generate_series(1,10) s)
+SELECT a+1, b FROM dist_table WHERE b IN (SELECT s FROM cte1)
+UNION ALL
+SELECT s, s FROM cte1
+ON CONFLICT (a) DO NOTHING;
+SELECT * FROM non_dist_unique ORDER BY 1;
+  a  | b
+---------------------------------------------------------------------
+   1 |  6
+   2 |  7
+   3 |  8
+   4 |  9
+   5 |  8
+   6 |  6
+   7 |  7
+   8 |  8
+   9 |  9
+  10 | 10
+ 101 |  7
+ 102 |  8
+ 103 |  9
+(13 rows)
+
 DROP TABLE non_dist_unique;
 -- test INSERT INTO a table with DEFAULT
 CREATE TABLE non_dist_default (a INT, c TEXT DEFAULT 'def');
@@ -168,6 +229,16 @@ SELECT * FROM non_dist_default ORDER BY 1, 2;
  3 | def
 (3 rows)
 
+SELECT alter_table_set_access_method('non_dist_default', 'columnar');
+NOTICE:  creating a new table for insert_select_into_local_table.non_dist_default
+NOTICE:  moving the data of insert_select_into_local_table.non_dist_default
+NOTICE:  dropping the old insert_select_into_local_table.non_dist_default
+NOTICE:  renaming the new table to insert_select_into_local_table.non_dist_default
+ alter_table_set_access_method
+---------------------------------------------------------------------
+
+(1 row)
+
 INSERT INTO non_dist_default SELECT a, c FROM dist_table WHERE a = 1;
 SELECT * FROM non_dist_default ORDER BY 1, 2;
  a |  c
@@ -354,6 +425,691 @@ SELECT * FROM non_dist_2 ORDER BY 1, 2;
 (3 rows)
 
 TRUNCATE non_dist_2;
+-- check issue https://github.com/citusdata/citus/issues/5858
+CREATE TABLE local_dest_table(
+  col_1 integer,
+  col_2 integer,
+  col_3 text,
+  col_4 text,
+  drop_col text,
+  col_5 bigint,
+  col_6 text,
+  col_7 text default 'col_7',
+  col_8 varchar
+);
+ALTER TABLE local_dest_table DROP COLUMN drop_col;
+CREATE TABLE dist_source_table_1(
+  int_col integer,
+  drop_col text,
+  text_col_1 text,
+  dist_col integer,
+  text_col_2 text
+);
+SELECT create_distributed_table('dist_source_table_1', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+ALTER TABLE dist_source_table_1 DROP COLUMN drop_col;
+INSERT INTO dist_source_table_1 VALUES (1, 'value', 1, 'value');
+INSERT INTO dist_source_table_1 VALUES (2, 'value2', 1, 'value');
+INSERT INTO dist_source_table_1 VALUES (3, 'value', 3, 'value3');
+CREATE TABLE dist_source_table_2(
+  dist_col integer,
+  int_col integer
+);
+SELECT create_distributed_table('dist_source_table_2', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO dist_source_table_2 VALUES (1, 1);
+INSERT INTO dist_source_table_2 VALUES (2, 2);
+INSERT INTO dist_source_table_2 VALUES (4, 4);
+CREATE TABLE local_source_table_1 AS SELECT * FROM dist_source_table_1;
+CREATE TABLE local_source_table_2 AS SELECT * FROM dist_source_table_2;
+/*
+ * query_results_equal compares the effect of two queries on local_dest_table.
+ * We use this to ensure that INSERT INTO local_dest_table SELECT behaves
+ * the same when selecting from a regular table (postgres handles it) and
+ * a distributed table (Citus handles it).
+ *
+ * The queries are generated by calling format() on query_table twice,
+ * once for each source_table argument.
+ */
+CREATE OR REPLACE FUNCTION query_results_equal(query_template text, source_table_1 text, source_table_2 text)
+RETURNS bool
+AS $$
+DECLARE
+	l1 local_dest_table[];
+	l2 local_dest_table[];
+BEGIN
+	/* get the results using source_table_1 as source */
+    TRUNCATE local_dest_table;
+	EXECUTE format(query_template, source_table_1);
+	SELECT array_agg(l) INTO l1
+	FROM (SELECT * FROM local_dest_table ORDER BY 1, 2, 3, 4, 5, 6, 7, 8) l;
+
+	/* get the results using source_table_2 as source */
+    TRUNCATE local_dest_table;
+	EXECUTE format(query_template, source_table_2);
+	SELECT array_agg(l) INTO l2
+	FROM (SELECT * FROM local_dest_table ORDER BY 1, 2, 3, 4, 5, 6, 7, 8) l;
+
+	RAISE NOTICE 'l2=%', l1;
+	RAISE NOTICE 'l2=%', l2;
+	RETURN l1 = l2;
+END;
+$$ LANGUAGE plpgsql;
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table
+  SELECT
+    t1.dist_col,
+    1,
+    'string1',
+    'string2',
+    2,
+    'string3',
+    t1.text_col_1,
+    t1.text_col_2
+  FROM %1$s_1 t1
+  WHERE t1.int_col IN (SELECT int_col FROM %1$s_2)
+$$, 'local_source_table', 'dist_source_table');
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table
+  SELECT
+    t1.dist_col,
+    1,
+    'string1',
+    'string2',
+    2,
+    'string3',
+    t1.text_col_1,
+    t1.text_col_2
+  FROM %1$s t1
+  returning *
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)","(3,1,string1,string2,2,string3,value,value3)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)","(3,1,string1,string2,2,string3,value,value3)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_3, col_4) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_7, col_4) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,string2,,,string1,)","(,,,string2,,,string1,)","(,,,string2,,,string1,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,string2,,,string1,)","(,,,string2,,,string1,)","(,,,string2,,,string1,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_4, col_3) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  WHERE dist_col = 1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string2,string1,,,col_7,)","(,,string2,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string2,string1,,,col_7,)","(,,string2,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s
+  UNION ALL
+  SELECT
+    'string',
+     int_col
+  FROM %1$s;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  WITH cte1 AS (SELECT s FROM generate_series(1,10) s)
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s WHERE int_col IN (SELECT s FROM cte1)
+  UNION ALL
+  SELECT
+    'string',
+     int_col
+  FROM %1$s WHERE int_col IN (SELECT s + 1 FROM cte1)
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  WITH cte1 AS (SELECT 'stringcte', s FROM generate_series(1,10) s)
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s WHERE int_col IN (SELECT s FROM cte1)
+  UNION ALL
+  SELECT
+    *
+  FROM cte1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,stringcte,,,col_7,)","(2,,,stringcte,,,col_7,)","(3,,,string1,,,col_7,)","(3,,,stringcte,,,col_7,)","(4,,,stringcte,,,col_7,)","(5,,,stringcte,,,col_7,)","(6,,,stringcte,,,col_7,)","(7,,,stringcte,,,col_7,)","(8,,,stringcte,,,col_7,)","(9,,,stringcte,,,col_7,)","(10,,,stringcte,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,stringcte,,,col_7,)","(2,,,stringcte,,,col_7,)","(3,,,string1,,,col_7,)","(3,,,stringcte,,,col_7,)","(4,,,stringcte,,,col_7,)","(5,,,stringcte,,,col_7,)","(6,,,stringcte,,,col_7,)","(7,,,stringcte,,,col_7,)","(8,,,stringcte,,,col_7,)","(9,,,stringcte,,,col_7,)","(10,,,stringcte,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_3)
+  SELECT t1.text_col_1
+  FROM %1$s t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,value,,,,col_7,)","(,,value2,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,value,,,,col_7,)","(,,value2,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_1, col_2, col_3, col_5, col_6, col_7, col_8)
+  SELECT
+    max(t1.dist_col),
+    3,
+    'string_3',
+    4,
+    44,
+    t1.text_col_1,
+    'string_1000'
+  FROM %1$s t1
+  GROUP BY t1.text_col_2, t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,3,string_3,,4,44,value,string_1000)","(1,3,string_3,,4,44,value2,string_1000)","(3,3,string_3,,4,44,value,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,3,string_3,,4,44,value,string_1000)","(1,3,string_3,,4,44,value2,string_1000)","(3,3,string_3,,4,44,value,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_7, col_8)
+  SELECT
+    t1.text_col_1,
+    'string_1000'
+  FROM dist_source_table_1 t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,,,,value,string_1000)","(,,,,,,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,,,,value,string_1000)","(,,,,,,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_6, col_7, col_8)
+  SELECT
+    'string_4',
+    t1.text_col_1,
+    'string_1000'
+  FROM %1$s t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,,,string_4,value,string_1000)","(,,,,,string_4,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,,,string_4,value,string_1000)","(,,,,,string_4,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_5, col_3)
+  SELECT 12, 'string_11' FROM %1$s t1
+  UNION
+  SELECT int_col, 'string' FROM %1$s;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string,,1,,col_7,)","(,,string,,2,,col_7,)","(,,string,,3,,col_7,)","(,,string_11,,12,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string,,1,,col_7,)","(,,string,,2,,col_7,)","(,,string,,3,,col_7,)","(,,string_11,,12,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table(col_3, col_2)
+  SELECT text_col_1, count(*) FROM %1$s GROUP BY 1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,1,value2,,,,col_7,)","(,2,value,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,1,value2,,,,col_7,)","(,2,value,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table(col_3, col_5)
+  SELECT text_col_1, count(*)::int FROM %1$s GROUP BY 1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,value,,2,,col_7,)","(,,value2,,1,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,value,,2,,col_7,)","(,,value2,,1,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- repeat above tests with Citus local table
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table
+  SELECT
+    t1.dist_col,
+    1,
+    'string1',
+    'string2',
+    2,
+    'string3',
+    t1.text_col_1,
+    t1.text_col_2
+  FROM %1$s_1 t1
+  WHERE t1.int_col IN (SELECT int_col FROM %1$s_2)
+$$, 'local_source_table', 'dist_source_table');
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table
+  SELECT
+    t1.dist_col,
+    1,
+    'string1',
+    'string2',
+    2,
+    'string3',
+    t1.text_col_1,
+    t1.text_col_2
+  FROM %1$s t1
+  returning *
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)","(3,1,string1,string2,2,string3,value,value3)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,1,string1,string2,2,string3,value,value)","(1,1,string1,string2,2,string3,value2,value)","(3,1,string1,string2,2,string3,value,value3)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_3, col_4) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)","(,,string1,string2,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_7, col_4) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,string2,,,string1,)","(,,,string2,,,string1,)","(,,,string2,,,string1,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,string2,,,string1,)","(,,,string2,,,string1,)","(,,,string2,,,string1,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_4, col_3) SELECT
+    'string1',
+    'string2'::text
+  FROM %1$s t1
+  WHERE dist_col = 1
+  returning *;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string2,string1,,,col_7,)","(,,string2,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string2,string1,,,col_7,)","(,,string2,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s
+  UNION ALL
+  SELECT
+    'string',
+     int_col
+  FROM %1$s;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  WITH cte1 AS (SELECT s FROM generate_series(1,10) s)
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s WHERE int_col IN (SELECT s FROM cte1)
+  UNION ALL
+  SELECT
+    'string',
+     int_col
+  FROM %1$s WHERE int_col IN (SELECT s + 1 FROM cte1)
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(2,,,string,,,col_7,)","(3,,,string,,,col_7,)","(3,,,string1,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  WITH cte1 AS (SELECT 'stringcte', s FROM generate_series(1,10) s)
+  INSERT INTO local_dest_table (col_4, col_1)
+  SELECT
+    'string1',
+     dist_col
+  FROM %1$s WHERE int_col IN (SELECT s FROM cte1)
+  UNION ALL
+  SELECT
+    *
+  FROM cte1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,stringcte,,,col_7,)","(2,,,stringcte,,,col_7,)","(3,,,string1,,,col_7,)","(3,,,stringcte,,,col_7,)","(4,,,stringcte,,,col_7,)","(5,,,stringcte,,,col_7,)","(6,,,stringcte,,,col_7,)","(7,,,stringcte,,,col_7,)","(8,,,stringcte,,,col_7,)","(9,,,stringcte,,,col_7,)","(10,,,stringcte,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,,,string1,,,col_7,)","(1,,,string1,,,col_7,)","(1,,,stringcte,,,col_7,)","(2,,,stringcte,,,col_7,)","(3,,,string1,,,col_7,)","(3,,,stringcte,,,col_7,)","(4,,,stringcte,,,col_7,)","(5,,,stringcte,,,col_7,)","(6,,,stringcte,,,col_7,)","(7,,,stringcte,,,col_7,)","(8,,,stringcte,,,col_7,)","(9,,,stringcte,,,col_7,)","(10,,,stringcte,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_3)
+  SELECT t1.text_col_1
+  FROM %1$s t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,value,,,,col_7,)","(,,value2,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,value,,,,col_7,)","(,,value2,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_1, col_2, col_3, col_5, col_6, col_7, col_8)
+  SELECT
+    max(t1.dist_col),
+    3,
+    'string_3',
+    4,
+    44,
+    t1.text_col_1,
+    'string_1000'
+  FROM %1$s t1
+  GROUP BY t1.text_col_2, t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(1,3,string_3,,4,44,value,string_1000)","(1,3,string_3,,4,44,value2,string_1000)","(3,3,string_3,,4,44,value,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(1,3,string_3,,4,44,value,string_1000)","(1,3,string_3,,4,44,value2,string_1000)","(3,3,string_3,,4,44,value,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_7, col_8)
+  SELECT
+    t1.text_col_1,
+    'string_1000'
+  FROM dist_source_table_1 t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,,,,value,string_1000)","(,,,,,,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,,,,value,string_1000)","(,,,,,,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_6, col_7, col_8)
+  SELECT
+    'string_4',
+    t1.text_col_1,
+    'string_1000'
+  FROM %1$s t1
+  GROUP BY t1.text_col_1;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,,,,string_4,value,string_1000)","(,,,,,string_4,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,,,,string_4,value,string_1000)","(,,,,,string_4,value2,string_1000)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table (col_5, col_3)
+  SELECT 12, 'string_11' FROM %1$s t1
+  UNION
+  SELECT int_col, 'string' FROM %1$s;
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,string,,1,,col_7,)","(,,string,,2,,col_7,)","(,,string,,3,,col_7,)","(,,string_11,,12,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,string,,1,,col_7,)","(,,string,,2,,col_7,)","(,,string,,3,,col_7,)","(,,string_11,,12,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table(col_3, col_2)
+  SELECT text_col_1, count(*) FROM %1$s GROUP BY 1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,1,value2,,,,col_7,)","(,2,value,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,1,value2,,,,col_7,)","(,2,value,,,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM query_results_equal($$
+  INSERT INTO local_dest_table(col_3, col_5)
+  SELECT text_col_1, count(*)::int FROM %1$s GROUP BY 1
+$$, 'local_source_table_1', 'dist_source_table_1');
+NOTICE:  l2={"(,,value,,2,,col_7,)","(,,value2,,1,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+NOTICE:  l2={"(,,value,,2,,col_7,)","(,,value2,,1,,col_7,)"}
+CONTEXT:  PL/pgSQL function query_results_equal(text,text,text) line XX at RAISE
+ query_results_equal
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- go back to proper local table for remaining tests
+TRUNCATE local_dest_table;
+SELECT undistribute_table('local_source_table_1');
+ERROR:  cannot undistribute table because the table is not distributed
+-- use a sequence (cannot use query_results_equal, since sequence values would not match)
+CREATE SEQUENCE seq;
+BEGIN;
+INSERT INTO local_dest_table (col_5, col_3)
+SELECT 12, 'string_11' FROM dist_source_table_1
+UNION
+SELECT nextval('seq'), 'string' FROM dist_source_table_1;
+SELECT * FROM local_dest_table ORDER BY 1,2,3,4,5,6,7,8;
+ col_1 | col_2 |   col_3   | col_4 | col_5 | col_6 | col_7 | col_8
+---------------------------------------------------------------------
+       |       | string    |       |     1 |       | col_7 |
+       |       | string    |       |     2 |       | col_7 |
+       |       | string    |       |     3 |       | col_7 |
+       |       | string_11 |       |    12 |       | col_7 |
+(4 rows)
+
+ROLLBACK;
+-- add a bigserial column
+ALTER TABLE local_dest_table ADD COLUMN col_9 bigserial;
+-- not supported due to limitations in nextval handling
+INSERT INTO local_dest_table (col_5, col_3)
+SELECT 12, 'string_11' FROM dist_source_table_1
+UNION
+SELECT 11, 'string' FROM dist_source_table_1;
+SELECT * FROM local_dest_table ORDER BY 1,2,3,4,5,6,7,8;
+ col_1 | col_2 |   col_3   | col_4 | col_5 | col_6 | col_7 | col_8 | col_9
+---------------------------------------------------------------------
+       |       | string    |       |    11 |       | col_7 |       |     2
+       |       | string_11 |       |    12 |       | col_7 |       |     1
+(2 rows)
+
+BEGIN;
+INSERT INTO local_dest_table(col_3, col_2)
+SELECT text_col_1, count(*) FROM dist_source_table_1 GROUP BY 1;
+SELECT * FROM local_dest_table ORDER BY 1,2,3,4,5,6,7,8;
+ col_1 | col_2 |   col_3   | col_4 | col_5 | col_6 | col_7 | col_8 | col_9
+---------------------------------------------------------------------
+       |     1 | value2    |       |       |       | col_7 |       |     3
+       |     2 | value     |       |       |       | col_7 |       |     4
+       |       | string    |       |    11 |       | col_7 |       |     2
+       |       | string_11 |       |    12 |       | col_7 |       |     1
+(4 rows)
+
+ROLLBACK;
+BEGIN;
+INSERT INTO local_dest_table (col_4, col_3) SELECT
+  'string1',
+  'string2'::text
+FROM dist_source_table_1 t1
+WHERE dist_col = 1
+RETURNING *;
+ col_1 | col_2 |  col_3  |  col_4  | col_5 | col_6 | col_7 | col_8 | col_9
+---------------------------------------------------------------------
+       |       | string2 | string1 |       |       | col_7 |       |     5
+       |       | string2 | string1 |       |       | col_7 |       |     6
+(2 rows)
+
+ROLLBACK;
 \set VERBOSITY terse
 DROP SCHEMA insert_select_into_local_table CASCADE;
-NOTICE:  drop cascades to 5 other objects
+NOTICE:  drop cascades to 12 other objects

src/test/regress/expected/insert_select_repartition.out

@@ -500,7 +500,7 @@ INSERT INTO target_table
 SELECT mapped_key, c FROM t NATURAL JOIN source_table;
 DEBUG:  volatile functions are not allowed in distributed INSERT ... SELECT queries
 DEBUG:  generating subplan XXX_1 for CTE t: SELECT mapped_key, a, c FROM insert_select_repartition.source_table WHERE ((a)::double precision OPERATOR(pg_catalog.>) floor(random()))
-DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT mapped_key AS a, (c)::integer[] AS b FROM (SELECT t.mapped_key, t.c FROM ((SELECT intermediate_result.mapped_key, intermediate_result.a, intermediate_result.c FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(mapped_key integer, a integer, c double precision[])) t JOIN insert_select_repartition.source_table USING (mapped_key, a, c))) citus_insert_select_subquery
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT mapped_key AS a, auto_coerced_by_citus_1 AS b FROM (SELECT t.mapped_key, (t.c)::integer[] AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.mapped_key, intermediate_result.a, intermediate_result.c FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(mapped_key integer, a integer, c double precision[])) t JOIN insert_select_repartition.source_table USING (mapped_key, a, c))) citus_insert_select_subquery
 DEBUG:  performing repartitioned INSERT ... SELECT
 RESET client_min_messages;
 SELECT * FROM target_table ORDER BY a;

src/test/regress/expected/intermediate_results.out

@@ -573,9 +573,111 @@ WARNING:  Query could not find the intermediate result file "squares_2", it was
 ---------------------------------------------------------------------
 (0 rows)
 
+-- test refreshing mat views
+SET client_min_messages TO ERROR;
+SELECT run_command_on_workers($$CREATE USER some_other_user;$$);
+      run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,"CREATE ROLE")
+ (localhost,57638,t,"CREATE ROLE")
+(2 rows)
+
+CREATE USER some_other_user;
+SELECT run_command_on_workers($$GRANT ALL ON DATABASE regression TO some_other_user;$$);
+  run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,GRANT)
+ (localhost,57638,t,GRANT)
+(2 rows)
+
+GRANT ALL ON DATABASE regression TO some_other_user;
+RESET client_min_messages;
+\c - some_other_user
+CREATE SCHEMA other_schema;
+SET search_path TO other_schema;
+CREATE TABLE dist_table (a int, b int);
+INSERT INTO dist_table(a, b) SELECT n, n+1 FROM generate_series(1, 10) n;
+SELECT create_distributed_table('dist_table', 'a');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$other_schema.dist_table$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE MATERIALIZED VIEW mat_view AS
+SELECT *
+FROM (
+  SELECT * FROM dist_table
+  LIMIT 50000
+) q;
+CREATE MATERIALIZED VIEW mat_view_2 AS
+  SELECT count(*) FROM (SELECT * FROM dist_table LIMIT 50000) q, (SELECT * FROM dist_table LIMIT 100) r WHERE q.a > r.a;
+REFRESH MATERIALIZED VIEW other_schema.mat_view;
+REFRESH MATERIALIZED VIEW other_schema.mat_view_2;
+-- Now connect back as a different user and run REFRESH MATERIALIZED VIEW command,
+-- which in turn executes a repartition join query.
+\c - postgres
+REFRESH MATERIALIZED VIEW other_schema.mat_view;
+REFRESH MATERIALIZED VIEW other_schema.mat_view_2;
+\c - some_other_user
+-- test security definer funcs
+CREATE FUNCTION security_definer_in_files()
+RETURNS BOOLEAN AS $$
+DECLARE passed BOOLEAN;
+BEGIN
+        SELECT  count(*) > 0 INTO passed
+        FROM   	(SELECT * FROM other_schema.dist_table ORDER BY a LIMIT 1) as foo,
+        		(SELECT * FROM other_schema.dist_table ORDER BY a LIMIT 1) as bar
+        		WHERE foo.a > bar.a;
+
+        RETURN passed;
+END;
+$$  LANGUAGE plpgsql
+    SECURITY DEFINER;
+SELECT security_definer_in_files();
+ security_definer_in_files
+---------------------------------------------------------------------
+ f
+(1 row)
+
+\c - postgres
+SELECT security_definer_in_files();
+ security_definer_in_files
+---------------------------------------------------------------------
+ f
+(1 row)
+
+CREATE FUNCTION security_definer_in_files_2()
+RETURNS BOOLEAN AS $$
+DECLARE passed BOOLEAN;
+BEGIN
+        SELECT  count(*) > 0 INTO passed
+        FROM   	(SELECT * FROM other_schema.dist_table ORDER BY a LIMIT 1) as foo,
+        		(SELECT * FROM other_schema.dist_table ORDER BY a LIMIT 1) as bar
+        		WHERE foo.a > bar.a;
+
+        RETURN passed;
+END;
+$$  LANGUAGE plpgsql
+    SECURITY DEFINER;
+BEGIN;
+  SELECT * FROM security_definer_in_files_2(), security_definer_in_files();
+ security_definer_in_files_2 | security_definer_in_files
+---------------------------------------------------------------------
+ f                           | f
+(1 row)
+
+  SELECT * FROM security_definer_in_files_2(), security_definer_in_files();
+ security_definer_in_files_2 | security_definer_in_files
+---------------------------------------------------------------------
+ f                           | f
+(1 row)
+
+COMMIT;
+-- cleanup
+SET client_min_messages TO ERROR;
+DROP SCHEMA other_schema CASCADE;
 DROP SCHEMA intermediate_results CASCADE;
-NOTICE:  drop cascades to 4 other objects
-DETAIL:  drop cascades to table interesting_squares
-drop cascades to type square_type
-drop cascades to table stored_squares
-drop cascades to table squares

src/test/regress/expected/isolation_fix_partition_shard_index_names.out

@@ -0,0 +1,54 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1-begin s1-drop-table s2-fix-partition-shard-index-names s1-commit
+create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin:
+    BEGIN;
+
+step s1-drop-table:
+    DROP TABLE dist_partitioned_table;
+
+step s2-fix-partition-shard-index-names:
+    SET client_min_messages TO NOTICE;
+    SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ <waiting ...>
+step s1-commit: 
+    COMMIT;
+
+step s2-fix-partition-shard-index-names: <... completed>
+s2: NOTICE:  relation with OID XXXX does not exist, skipping
+fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+
+starting permutation: s2-begin s2-fix-partition-shard-index-names s1-drop-table s2-commit
+create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s2-fix-partition-shard-index-names:
+    SET client_min_messages TO NOTICE;
+    SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+
+fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-drop-table:
+    DROP TABLE dist_partitioned_table;
+ <waiting ...>
+step s2-commit:
+    COMMIT;
+
+step s1-drop-table: <... completed>

src/test/regress/expected/isolation_reference_table.out

@@ -0,0 +1,137 @@
+Parsed test spec with 2 sessions
+
+starting permutation: s1-begin s1-create s2-create s1-commit
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin:
+    BEGIN;
+
+step s1-create:
+    CREATE TABLE reference_table_s1(a int);
+ 	SELECT create_reference_table('reference_table_s1');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-create:
+    CREATE TABLE reference_table_s2(a int);
+ 	SELECT create_reference_table('reference_table_s2');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-commit:
+    COMMIT;
+
+
+starting permutation: s1-create s2-create s1-begin s1-drop s2-drop s1-commit
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create:
+    CREATE TABLE reference_table_s1(a int);
+ 	SELECT create_reference_table('reference_table_s1');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-create:
+    CREATE TABLE reference_table_s2(a int);
+ 	SELECT create_reference_table('reference_table_s2');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin:
+    BEGIN;
+
+step s1-drop:
+    DROP TABLE reference_table_s1;
+
+step s2-drop:
+    DROP TABLE reference_table_s2;
+
+step s1-commit:
+    COMMIT;
+
+
+starting permutation: s1-create s2-begin s2-create s1-drop s2-commit
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create:
+    CREATE TABLE reference_table_s1(a int);
+ 	SELECT create_reference_table('reference_table_s1');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s2-create:
+    CREATE TABLE reference_table_s2(a int);
+ 	SELECT create_reference_table('reference_table_s2');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-drop:
+    DROP TABLE reference_table_s1;
+
+step s2-commit:
+    COMMIT;
+
+
+starting permutation: s2-create s2-begin s2-drop s1-create s2-commit
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-create:
+    CREATE TABLE reference_table_s2(a int);
+ 	SELECT create_reference_table('reference_table_s2');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s2-drop:
+    DROP TABLE reference_table_s2;
+
+step s1-create:
+    CREATE TABLE reference_table_s1(a int);
+ 	SELECT create_reference_table('reference_table_s1');
+
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-commit:
+    COMMIT;
+

src/test/regress/expected/multi_explain.out

@@ -1347,21 +1347,20 @@ WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5)
 SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1);
 Custom Scan (Citus INSERT ... SELECT)
   INSERT/SELECT method: pull to coordinator
-  ->  Subquery Scan on citus_insert_select_subquery
+  ->  Result
+        One-Time Filter: $3
         CTE cte1
           ->  Function Scan on generate_series s
-        ->  Result
+        CTE cte1
-              One-Time Filter: $3
+          ->  Limit
-              CTE cte1
+                InitPlan 2 (returns $1)
-                ->  Limit
+                  ->  CTE Scan on cte1 cte1_1
-                      InitPlan 2 (returns $1)
+                ->  Result
-                        ->  CTE Scan on cte1 cte1_1
+                      One-Time Filter: $1
-                      ->  Result
+                      ->  CTE Scan on cte1 cte1_2
-                            One-Time Filter: $1
+        InitPlan 4 (returns $3)
-                            ->  CTE Scan on cte1 cte1_2
+          ->  CTE Scan on cte1 cte1_3
-              InitPlan 4 (returns $3)
+        ->  CTE Scan on cte1
-                ->  CTE Scan on cte1 cte1_3
-              ->  CTE Scan on cte1
 EXPLAIN (COSTS OFF)
 INSERT INTO lineitem_hash_part
 ( SELECT s FROM generate_series(1,5) s) UNION
@@ -3014,5 +3013,18 @@ Custom Scan (Citus Adaptive) (actual rows=1 loops=1)
   ->  Task
         Node: host=localhost port=xxxxx dbname=regression
         ->  Seq Scan on distributed_table_1_570032 distributed_table_xxx (actual rows=1 loops=1)
+CREATE TYPE multi_explain.int_wrapper_type AS (int_field int);
+CREATE TABLE tbl (a int, b multi_explain.int_wrapper_type);
+SELECT create_distributed_table('tbl', 'a');
+
+EXPLAIN :default_analyze_flags SELECT * FROM tbl;
+Custom Scan (Citus Adaptive) (actual rows=0 loops=1)
+  Task Count: 2
+  Tuple data received from nodes: 0 bytes
+  Tasks Shown: One of 2
+  ->  Task
+        Tuple data received from node: 0 bytes
+        Node: host=localhost port=xxxxx dbname=regression
+        ->  Seq Scan on tbl_570036 tbl (actual rows=0 loops=1)
 SET client_min_messages TO ERROR;
 DROP SCHEMA multi_explain CASCADE;

src/test/regress/expected/multi_extension.out

@@ -750,6 +750,41 @@ SELECT * FROM multi_extension.print_extension_changes();
                     | view public.citus_tables
 (2 rows)
 
+-- not print "HINT: " to hide current lib version
+\set VERBOSITY terse
+CREATE TABLE columnar_table(a INT, b INT) USING columnar;
+SET citus.enable_version_checks TO ON;
+-- all should throw an error due to version mismatch
+VACUUM FULL columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+INSERT INTO columnar_table SELECT i FROM generate_series(1, 10) i;
+ERROR:  loaded Citus library version differs from installed extension version
+VACUUM columnar_table;
+WARNING:  loaded Citus library version differs from installed extension version
+TRUNCATE columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+DROP TABLE columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+CREATE INDEX ON columnar_table (a);
+ERROR:  loaded Citus library version differs from installed extension version
+SELECT alter_columnar_table_set('columnar_table', compression => 'pglz');
+ERROR:  loaded Citus library version differs from installed extension version
+SELECT alter_columnar_table_reset('columnar_table');
+ERROR:  loaded Citus library version differs from installed extension version
+INSERT INTO columnar_table SELECT * FROM columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+SELECT 1 FROM columnar_table; -- columnar custom scan
+ERROR:  loaded Citus library version differs from installed extension version
+SET columnar.enable_custom_scan TO OFF;
+SELECT 1 FROM columnar_table; -- seq scan
+ERROR:  loaded Citus library version differs from installed extension version
+CREATE TABLE new_columnar_table (a int) USING columnar;
+ERROR:  loaded Citus library version differs from installed extension version
+-- do cleanup for the rest of the tests
+SET citus.enable_version_checks TO OFF;
+DROP TABLE columnar_table;
+RESET columnar.enable_custom_scan;
+\set VERBOSITY default
 -- Test downgrade to 10.0-4 from 10.1-1
 ALTER EXTENSION citus UPDATE TO '10.1-1';
 ALTER EXTENSION citus UPDATE TO '10.0-4';
@@ -813,12 +848,120 @@ SELECT * FROM multi_extension.print_extension_changes();
                                                             | function worker_nextval(regclass) integer
 (16 rows)
 
+-- Test downgrade to 10.2-1 from 10.2-2
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+ALTER EXTENSION citus UPDATE TO '10.2-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-2
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-2 from 10.2-3
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-3
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-3 from 10.2-4
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+-- Make sure that we don't delete pg_depend entries added in
+-- columnar--10.2-3--10.2-4.sql when downgrading to 10.2-3.
+SELECT COUNT(*)=10
+FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid AND
+      objid = (select oid from pg_am where amname = 'columnar') AND
+      objsubid = 0 AND
+      refclassid = 'pg_class'::regclass::oid AND
+      refobjsubid = 0 AND
+      deptype = 'n';
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-4
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                              current_object
+---------------------------------------------------------------------
+                 | function citus_internal.columnar_ensure_am_depends_catalog() void
+                 | function fix_all_partition_shard_index_names() SETOF regclass
+                 | function fix_partition_shard_index_names(regclass) void
+                 | function worker_fix_partition_shard_index_names(regclass,text,text) void
+(4 rows)
+
+-- Snapshot of state at 10.2-5
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-4 from 10.2-5
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+-- Make sure that we defined dependencies from all rel objects (tables,
+-- indexes, sequences ..) to columnar table access method ...
+SELECT pg_class.oid INTO columnar_schema_members
+FROM pg_class, pg_namespace
+WHERE pg_namespace.oid=pg_class.relnamespace AND
+      pg_namespace.nspname='columnar';
+SELECT refobjid INTO columnar_schema_members_pg_depend
+FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid AND
+      objid = (select oid from pg_am where amname = 'columnar') AND
+      objsubid = 0 AND
+      refclassid = 'pg_class'::regclass::oid AND
+      refobjsubid = 0 AND
+      deptype = 'n';
+-- ... , so this should be empty,
+(TABLE columnar_schema_members EXCEPT TABLE columnar_schema_members_pg_depend)
+UNION
+(TABLE columnar_schema_members_pg_depend EXCEPT TABLE columnar_schema_members);
+ oid
+---------------------------------------------------------------------
+(0 rows)
+
+-- ... , and both columnar_schema_members_pg_depend & columnar_schema_members
+-- should have 10 entries.
+SELECT COUNT(*)=10 FROM columnar_schema_members_pg_depend;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+DROP TABLE columnar_schema_members, columnar_schema_members_pg_depend;
 DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
 -- show running version
 SHOW citus.version;
  citus.version
 ---------------------------------------------------------------------
- 10.2devel
+ 10.2.8
 (1 row)
 
 -- ensure no unexpected objects were created outside pg_catalog

src/test/regress/expected/multi_fix_partition_shard_index_names.out

@@ -0,0 +1,638 @@
+---------------------------------------------------------------------
+-- multi_fix_partition_shard_index_names
+-- check the following two issues
+-- https://github.com/citusdata/citus/issues/4962
+-- https://github.com/citusdata/citus/issues/5138
+---------------------------------------------------------------------
+SET citus.next_shard_id TO 910000;
+SET citus.shard_replication_factor TO 1;
+CREATE SCHEMA fix_idx_names;
+SET search_path TO fix_idx_names, public;
+-- NULL input should automatically return NULL since
+-- fix_partition_shard_index_names is strict
+-- same for worker_fix_partition_shard_index_names
+SELECT fix_partition_shard_index_names(NULL);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT worker_fix_partition_shard_index_names(NULL, NULL, NULL);
+ worker_fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+-- fix_partition_shard_index_names cannot be called for distributed
+-- and not partitioned tables
+CREATE TABLE not_partitioned(id int);
+SELECT create_distributed_table('not_partitioned', 'id');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT fix_partition_shard_index_names('not_partitioned'::regclass);
+ERROR:  Fixing shard index names is only applicable to partitioned tables, and "not_partitioned" is not a partitioned table
+-- fix_partition_shard_index_names cannot be called for partitioned
+-- and not distributed tables
+CREATE TABLE not_distributed(created_at timestamptz) PARTITION BY RANGE (created_at);
+SELECT fix_partition_shard_index_names('not_distributed'::regclass);
+ERROR:  fix_partition_shard_index_names can only be called for distributed partitioned tables
+-- test with proper table
+CREATE TABLE dist_partitioned_table (dist_col int, another_col int, partition_col timestamp) PARTITION BY RANGE (partition_col);
+SELECT create_distributed_table('dist_partitioned_table', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- create a partition with a long name and another with a short name
+CREATE TABLE partition_table_with_very_long_name PARTITION OF dist_partitioned_table FOR VALUES FROM ('2018-01-01') TO ('2019-01-01');
+CREATE TABLE p PARTITION OF dist_partitioned_table FOR VALUES FROM ('2019-01-01') TO ('2020-01-01');
+-- create an index on parent table
+-- we will see that it doesn't matter whether we name the index on parent or not
+-- indexes auto-generated on partitions will not use this name
+CREATE INDEX short ON dist_partitioned_table USING btree (another_col, partition_col);
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+              tablename              |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table              | short
+ p                                   | p_another_col_partition_col_idx
+ partition_table_with_very_long_name | partition_table_with_very_long_na_another_col_partition_col_idx
+(3 rows)
+
+\c - - - :worker_1_port
+-- Note that, the shell table from above partition_table_with_very_long_name
+-- and its shard partition_table_with_very_long_name_910008
+-- have the same index name: partition_table_with_very_long_na_another_col_partition_col_idx
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | short_910004
+ dist_partitioned_table_910006              | short_910006
+ p_910012                                   | p_910012_another_col_partition_col_idx
+ p_910014                                   | p_910014_another_col_partition_col_idx
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_na_another_col_partition_col_idx
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_n_another_col_partition_col_idx1
+(6 rows)
+
+\c - - - :master_port
+-- this should fail because of the name clash explained above
+SELECT start_metadata_sync_to_node('localhost', :worker_1_port);
+ERROR:  relation "partition_table_with_very_long_na_another_col_partition_col_idx" already exists
+CONTEXT:  while executing command on localhost:xxxxx
+-- let's fix the problematic table
+SET search_path TO fix_idx_names, public;
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+\c - - - :worker_1_port
+-- shard id has been appended to all index names which didn't end in shard id
+-- this goes in line with Citus's way of naming indexes of shards: always append shardid to the end
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | short_910004
+ dist_partitioned_table_910006              | short_910006
+ p_910012                                   | p_another_col_partition_col_idx_910012
+ p_910014                                   | p_another_col_partition_col_idx_910014
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_na_another_col_p_dd884a3b_910008
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_na_another_col_p_dd884a3b_910010
+(6 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- this should now work
+SELECT start_metadata_sync_to_node('localhost', :worker_1_port);
+ start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+-- if we run this command again, the names will not change anymore since shardid is appended to them
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT fix_all_partition_shard_index_names();
+ fix_all_partition_shard_index_names
+---------------------------------------------------------------------
+ dist_partitioned_table
+(1 row)
+
+\c - - - :worker_1_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table                     | short
+ dist_partitioned_table_910004              | short_910004
+ dist_partitioned_table_910006              | short_910006
+ p                                          | p_another_col_partition_col_idx
+ p_910012                                   | p_another_col_partition_col_idx_910012
+ p_910014                                   | p_another_col_partition_col_idx_910014
+ partition_table_with_very_long_name        | partition_table_with_very_long_na_another_col_partition_col_idx
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_na_another_col_p_dd884a3b_910008
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_na_another_col_p_dd884a3b_910010
+(9 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+SET citus.shard_replication_factor TO 1;
+SET citus.next_shard_id TO 910020;
+-- if we explicitly create index on partition-to-be table, Citus handles the naming
+-- hence we would have no broken index names
+CREATE TABLE another_partition_table_with_very_long_name (dist_col int, another_col int, partition_col timestamp);
+SELECT create_distributed_table('another_partition_table_with_very_long_name', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE INDEX ON another_partition_table_with_very_long_name USING btree (another_col, partition_col);
+ALTER TABLE dist_partitioned_table ATTACH PARTITION another_partition_table_with_very_long_name FOR VALUES FROM ('2020-01-01') TO ('2021-01-01');
+-- check it works even if we give a weird index name
+CREATE TABLE yet_another_partition_table (dist_col int, another_col int, partition_col timestamp);
+SELECT create_distributed_table('yet_another_partition_table', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE INDEX "really weird index name !!" ON yet_another_partition_table USING btree (another_col, partition_col);
+ALTER TABLE dist_partitioned_table ATTACH PARTITION yet_another_partition_table FOR VALUES FROM ('2021-01-01') TO ('2022-01-01');
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                  tablename                  |                            indexname
+---------------------------------------------------------------------
+ another_partition_table_with_very_long_name | another_partition_table_with_very_another_col_partition_col_idx
+ dist_partitioned_table                      | short
+ p                                           | p_another_col_partition_col_idx
+ partition_table_with_very_long_name         | partition_table_with_very_long_na_another_col_partition_col_idx
+ yet_another_partition_table                 | really weird index name !!
+(5 rows)
+
+\c - - - :worker_1_port
+-- notice indexes of shards of another_partition_table_with_very_long_name already have shardid appended to the end
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                     tablename                      |                            indexname
+---------------------------------------------------------------------
+ another_partition_table_with_very_long_name        | another_partition_table_with_very_another_col_partition_col_idx
+ another_partition_table_with_very_long_name_910020 | another_partition_table_with_very_another_col_p_a02939b4_910020
+ another_partition_table_with_very_long_name_910022 | another_partition_table_with_very_another_col_p_a02939b4_910022
+ dist_partitioned_table                             | short
+ dist_partitioned_table_910004                      | short_910004
+ dist_partitioned_table_910006                      | short_910006
+ p                                                  | p_another_col_partition_col_idx
+ p_910012                                           | p_another_col_partition_col_idx_910012
+ p_910014                                           | p_another_col_partition_col_idx_910014
+ partition_table_with_very_long_name                | partition_table_with_very_long_na_another_col_partition_col_idx
+ partition_table_with_very_long_name_910008         | partition_table_with_very_long_na_another_col_p_dd884a3b_910008
+ partition_table_with_very_long_name_910010         | partition_table_with_very_long_na_another_col_p_dd884a3b_910010
+ yet_another_partition_table                        | really weird index name !!
+ yet_another_partition_table_910024                 | really weird index name !!_910024
+ yet_another_partition_table_910026                 | really weird index name !!_910026
+(15 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- this command would not do anything
+SELECT fix_all_partition_shard_index_names();
+ fix_all_partition_shard_index_names
+---------------------------------------------------------------------
+ dist_partitioned_table
+(1 row)
+
+\c - - - :worker_1_port
+-- names are the same as before
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                     tablename                      |                            indexname
+---------------------------------------------------------------------
+ another_partition_table_with_very_long_name        | another_partition_table_with_very_another_col_partition_col_idx
+ another_partition_table_with_very_long_name_910020 | another_partition_table_with_very_another_col_p_a02939b4_910020
+ another_partition_table_with_very_long_name_910022 | another_partition_table_with_very_another_col_p_a02939b4_910022
+ dist_partitioned_table                             | short
+ dist_partitioned_table_910004                      | short_910004
+ dist_partitioned_table_910006                      | short_910006
+ p                                                  | p_another_col_partition_col_idx
+ p_910012                                           | p_another_col_partition_col_idx_910012
+ p_910014                                           | p_another_col_partition_col_idx_910014
+ partition_table_with_very_long_name                | partition_table_with_very_long_na_another_col_partition_col_idx
+ partition_table_with_very_long_name_910008         | partition_table_with_very_long_na_another_col_p_dd884a3b_910008
+ partition_table_with_very_long_name_910010         | partition_table_with_very_long_na_another_col_p_dd884a3b_910010
+ yet_another_partition_table                        | really weird index name !!
+ yet_another_partition_table_910024                 | really weird index name !!_910024
+ yet_another_partition_table_910026                 | really weird index name !!_910026
+(15 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
+NOTICE:  dropping metadata on the node (localhost,57637)
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP INDEX short;
+DROP TABLE yet_another_partition_table, another_partition_table_with_very_long_name;
+-- this will create constraint1 index on parent
+ALTER TABLE dist_partitioned_table ADD CONSTRAINT constraint1 UNIQUE (dist_col, partition_col);
+CREATE TABLE fk_table (id int, fk_column timestamp, FOREIGN KEY (id, fk_column) REFERENCES dist_partitioned_table (dist_col, partition_col));
+-- try creating index to foreign key
+CREATE INDEX ON dist_partitioned_table USING btree (dist_col, partition_col);
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+              tablename              |                           indexname
+---------------------------------------------------------------------
+ dist_partitioned_table              | constraint1
+ dist_partitioned_table              | dist_partitioned_table_dist_col_partition_col_idx
+ p                                   | p_dist_col_partition_col_idx
+ p                                   | p_dist_col_partition_col_key
+ partition_table_with_very_long_name | partition_table_with_very_long_name_dist_col_partition_col_idx
+ partition_table_with_very_long_name | partition_table_with_very_long_name_dist_col_partition_col_key
+(6 rows)
+
+\c - - - :worker_1_port
+-- index names don't end in shardid for partitions
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | constraint1_910004
+ dist_partitioned_table_910004              | dist_partitioned_table_dist_col_partition_col_idx_910004
+ dist_partitioned_table_910006              | constraint1_910006
+ dist_partitioned_table_910006              | dist_partitioned_table_dist_col_partition_col_idx_910006
+ p_910012                                   | p_910012_dist_col_partition_col_idx
+ p_910012                                   | p_910012_dist_col_partition_col_key
+ p_910014                                   | p_910014_dist_col_partition_col_idx
+ p_910014                                   | p_910014_dist_col_partition_col_key
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name__dist_col_partition_col_idx
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name__dist_col_partition_col_key
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_dist_col_partition_col_idx1
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_dist_col_partition_col_key1
+(12 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+SELECT fix_all_partition_shard_index_names();
+ fix_all_partition_shard_index_names
+---------------------------------------------------------------------
+ dist_partitioned_table
+(1 row)
+
+\c - - - :worker_1_port
+-- now index names end in shardid
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | constraint1_910004
+ dist_partitioned_table_910004              | dist_partitioned_table_dist_col_partition_col_idx_910004
+ dist_partitioned_table_910006              | constraint1_910006
+ dist_partitioned_table_910006              | dist_partitioned_table_dist_col_partition_col_idx_910006
+ p_910012                                   | p_dist_col_partition_col_idx_910012
+ p_910012                                   | p_dist_col_partition_col_key_910012
+ p_910014                                   | p_dist_col_partition_col_idx_910014
+ p_910014                                   | p_dist_col_partition_col_key_910014
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_dist_col_pa_781a5400_910008
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_dist_col_pa_ef25fb77_910008
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_dist_col_pa_781a5400_910010
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_dist_col_pa_ef25fb77_910010
+(12 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+ALTER TABLE dist_partitioned_table DROP CONSTRAINT constraint1 CASCADE;
+NOTICE:  drop cascades to constraint fk_table_id_fk_column_fkey on table fk_table
+DROP INDEX dist_partitioned_table_dist_col_partition_col_idx;
+-- try with index on only parent
+-- this is also an invalid index
+-- also try with hash method, not btree
+CREATE INDEX short_parent ON ONLY dist_partitioned_table USING hash (dist_col);
+-- only another_partition will have the index on dist_col inherited from short_parent
+-- hence short_parent will still be invalid
+CREATE TABLE another_partition (dist_col int, another_col int, partition_col timestamp);
+ALTER TABLE dist_partitioned_table ATTACH PARTITION another_partition FOR VALUES FROM ('2017-01-01') TO ('2018-01-01');
+SELECT c.relname AS indexname
+FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n, pg_catalog.pg_index i
+WHERE  (i.indisvalid = false) AND i.indexrelid = c.oid AND c.relnamespace = n.oid AND n.nspname = 'fix_idx_names';
+  indexname
+---------------------------------------------------------------------
+ short_parent
+(1 row)
+
+-- try with index on only partition
+CREATE INDEX short_child ON ONLY p USING hash (dist_col);
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+       tablename        |           indexname
+---------------------------------------------------------------------
+ another_partition      | another_partition_dist_col_idx
+ dist_partitioned_table | short_parent
+ p                      | short_child
+(3 rows)
+
+\c - - - :worker_1_port
+-- index names are already correct except for inherited index for another_partition
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+           tablename           |               indexname
+---------------------------------------------------------------------
+ another_partition_361176      | another_partition_361176_dist_col_idx
+ another_partition_361178      | another_partition_361178_dist_col_idx
+ dist_partitioned_table_910004 | short_parent_910004
+ dist_partitioned_table_910006 | short_parent_910006
+ p_910012                      | short_child_910012
+ p_910014                      | short_child_910014
+(6 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- this will fix inherited index for another_partition
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+-- this will error out becuase p is not partitioned, it is rather a partition
+SELECT fix_partition_shard_index_names('p'::regclass);
+ERROR:  Fixing shard index names is only applicable to partitioned tables, and "p" is not a partitioned table
+\c - - - :worker_1_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+           tablename           |               indexname
+---------------------------------------------------------------------
+ another_partition_361176      | another_partition_dist_col_idx_361176
+ another_partition_361178      | another_partition_dist_col_idx_361178
+ dist_partitioned_table_910004 | short_parent_910004
+ dist_partitioned_table_910006 | short_parent_910006
+ p_910012                      | short_child_910012
+ p_910014                      | short_child_910014
+(6 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+DROP INDEX short_parent;
+DROP INDEX short_child;
+DROP TABLE another_partition;
+-- expression indexes have the same problem with naming
+CREATE INDEX expression_index ON dist_partitioned_table ((dist_col || ' ' || another_col));
+-- try with statistics on index
+CREATE INDEX statistics_on_index on dist_partitioned_table ((dist_col+another_col), (dist_col-another_col));
+ALTER INDEX statistics_on_index ALTER COLUMN 1 SET STATISTICS 3737;
+ALTER INDEX statistics_on_index ALTER COLUMN 2 SET STATISTICS 3737;
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+              tablename              |                     indexname
+---------------------------------------------------------------------
+ dist_partitioned_table              | expression_index
+ dist_partitioned_table              | statistics_on_index
+ p                                   | p_expr_expr1_idx
+ p                                   | p_expr_idx
+ partition_table_with_very_long_name | partition_table_with_very_long_name_expr_expr1_idx
+ partition_table_with_very_long_name | partition_table_with_very_long_name_expr_idx
+(6 rows)
+
+\c - - - :worker_1_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                         indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | expression_index_910004
+ dist_partitioned_table_910004              | statistics_on_index_910004
+ dist_partitioned_table_910006              | expression_index_910006
+ dist_partitioned_table_910006              | statistics_on_index_910006
+ p_910012                                   | p_910012_expr_expr1_idx
+ p_910012                                   | p_910012_expr_idx
+ p_910014                                   | p_910014_expr_expr1_idx
+ p_910014                                   | p_910014_expr_idx
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_910008_expr_expr1_idx
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_910008_expr_idx
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_910010_expr_expr1_idx
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_910010_expr_idx
+(12 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+\c - - - :worker_1_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                         indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004              | expression_index_910004
+ dist_partitioned_table_910004              | statistics_on_index_910004
+ dist_partitioned_table_910006              | expression_index_910006
+ dist_partitioned_table_910006              | statistics_on_index_910006
+ p_910012                                   | p_expr_expr1_idx_910012
+ p_910012                                   | p_expr_idx_910012
+ p_910014                                   | p_expr_expr1_idx_910014
+ p_910014                                   | p_expr_idx_910014
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_expr_expr1_idx_910008
+ partition_table_with_very_long_name_910008 | partition_table_with_very_long_name_expr_idx_910008
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_expr_expr1_idx_910010
+ partition_table_with_very_long_name_910010 | partition_table_with_very_long_name_expr_idx_910010
+(12 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- try with a table with no partitions
+ALTER TABLE dist_partitioned_table DETACH PARTITION p;
+ALTER TABLE dist_partitioned_table DETACH PARTITION partition_table_with_very_long_name;
+DROP TABLE p;
+DROP TABLE partition_table_with_very_long_name;
+-- still dist_partitioned_table has indexes
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+       tablename        |      indexname
+---------------------------------------------------------------------
+ dist_partitioned_table | expression_index
+ dist_partitioned_table | statistics_on_index
+(2 rows)
+
+-- this does nothing
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+\c - - - :worker_1_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+           tablename           |         indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910004 | expression_index_910004
+ dist_partitioned_table_910004 | statistics_on_index_910004
+ dist_partitioned_table_910006 | expression_index_910006
+ dist_partitioned_table_910006 | statistics_on_index_910006
+(4 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+DROP TABLE dist_partitioned_table;
+-- add test with replication factor = 2
+SET citus.shard_replication_factor TO 2;
+SET citus.next_shard_id TO 910050;
+CREATE TABLE dist_partitioned_table (dist_col int, another_col int, partition_col timestamp) PARTITION BY RANGE (partition_col);
+SELECT create_distributed_table('dist_partitioned_table', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- create a partition with a long name
+CREATE TABLE partition_table_with_very_long_name PARTITION OF dist_partitioned_table FOR VALUES FROM ('2018-01-01') TO ('2019-01-01');
+-- create an index on parent table
+CREATE INDEX index_rep_factor_2 ON dist_partitioned_table USING btree (another_col, partition_col);
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+              tablename              |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table              | index_rep_factor_2
+ partition_table_with_very_long_name | partition_table_with_very_long_na_another_col_partition_col_idx
+(2 rows)
+
+\c - - - :worker_2_port
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910050              | index_rep_factor_2_910050
+ dist_partitioned_table_910051              | index_rep_factor_2_910051
+ dist_partitioned_table_910052              | index_rep_factor_2_910052
+ dist_partitioned_table_910053              | index_rep_factor_2_910053
+ partition_table_with_very_long_name_910054 | partition_table_with_very_long_na_another_col_partition_col_idx
+ partition_table_with_very_long_name_910055 | partition_table_with_very_long_n_another_col_partition_col_idx1
+ partition_table_with_very_long_name_910056 | partition_table_with_very_long_n_another_col_partition_col_idx2
+ partition_table_with_very_long_name_910057 | partition_table_with_very_long_n_another_col_partition_col_idx3
+(8 rows)
+
+\c - - - :master_port
+-- let's fix the problematic table
+SET search_path TO fix_idx_names, public;
+SELECT fix_partition_shard_index_names('dist_partitioned_table'::regclass);
+ fix_partition_shard_index_names
+---------------------------------------------------------------------
+
+(1 row)
+
+\c - - - :worker_2_port
+-- shard id has been appended to all index names which didn't end in shard id
+-- this goes in line with Citus's way of naming indexes of shards: always append shardid to the end
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910050              | index_rep_factor_2_910050
+ dist_partitioned_table_910051              | index_rep_factor_2_910051
+ dist_partitioned_table_910052              | index_rep_factor_2_910052
+ dist_partitioned_table_910053              | index_rep_factor_2_910053
+ partition_table_with_very_long_name_910054 | partition_table_with_very_long_na_another_col_p_dd884a3b_910054
+ partition_table_with_very_long_name_910055 | partition_table_with_very_long_na_another_col_p_dd884a3b_910055
+ partition_table_with_very_long_name_910056 | partition_table_with_very_long_na_another_col_p_dd884a3b_910056
+ partition_table_with_very_long_name_910057 | partition_table_with_very_long_na_another_col_p_dd884a3b_910057
+(8 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- test with role that is not superuser
+SET client_min_messages TO warning;
+SET citus.enable_ddl_propagation TO off;
+CREATE USER user1;
+RESET client_min_messages;
+RESET citus.enable_ddl_propagation;
+SET ROLE user1;
+SELECT fix_partition_shard_index_names('fix_idx_names.dist_partitioned_table'::regclass);
+ERROR:  permission denied for schema fix_idx_names
+RESET ROLE;
+SET search_path TO fix_idx_names, public;
+DROP TABLE dist_partitioned_table;
+-- also, we cannot do any further operations (e.g. rename) on the indexes of partitions because
+-- the index names on shards of partitions have been generated by Postgres, not Citus
+-- it doesn't matter here whether the partition name is long or short
+-- replicate scenario from above but this time with one shard so that this test isn't flaky
+SET citus.shard_count TO 1;
+SET citus.shard_replication_factor TO 1;
+SET citus.next_shard_id TO 910030;
+CREATE TABLE dist_partitioned_table (dist_col int, another_col int, partition_col timestamp) PARTITION BY RANGE (partition_col);
+SELECT create_distributed_table('dist_partitioned_table', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE partition_table_with_very_long_name PARTITION OF dist_partitioned_table FOR VALUES FROM ('2018-01-01') TO ('2019-01-01');
+CREATE TABLE p PARTITION OF dist_partitioned_table FOR VALUES FROM ('2019-01-01') TO ('2020-01-01');
+CREATE INDEX short ON dist_partitioned_table USING btree (another_col, partition_col);
+-- rename shouldn't work
+ALTER INDEX partition_table_with_very_long_na_another_col_partition_col_idx RENAME TO partition_table_with_very_long_name_idx;
+ERROR:  relation "fix_idx_names.partition_table_with_very_long_na_another_col_p_dd884a3b_910031" does not exist
+CONTEXT:  while executing command on localhost:xxxxx
+-- we currently can't drop index on detached partition
+-- https://github.com/citusdata/citus/issues/5138
+ALTER TABLE dist_partitioned_table DETACH PARTITION p;
+DROP INDEX p_another_col_partition_col_idx;
+ERROR:  index "p_another_col_partition_col_idx_910032" does not exist
+CONTEXT:  while executing command on localhost:xxxxx
+-- let's reattach and retry after fixing index names
+ALTER TABLE dist_partitioned_table ATTACH PARTITION p FOR VALUES FROM ('2019-01-01') TO ('2020-01-01');
+\c - - - :worker_1_port
+-- check the current broken index names
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910030              | short_910030
+ p_910032                                   | p_910032_another_col_partition_col_idx
+ partition_table_with_very_long_name_910031 | partition_table_with_very_long_na_another_col_partition_col_idx
+(3 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- fix index names
+SELECT fix_all_partition_shard_index_names();
+ fix_all_partition_shard_index_names
+---------------------------------------------------------------------
+ dist_partitioned_table
+(1 row)
+
+\c - - - :worker_1_port
+-- check the fixed index names
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                            indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910030              | short_910030
+ p_910032                                   | p_another_col_partition_col_idx_910032
+ partition_table_with_very_long_name_910031 | partition_table_with_very_long_na_another_col_p_dd884a3b_910031
+(3 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+-- should now work
+ALTER INDEX partition_table_with_very_long_na_another_col_partition_col_idx RENAME TO partition_table_with_very_long_name_idx;
+-- now we can drop index on detached partition
+ALTER TABLE dist_partitioned_table DETACH PARTITION p;
+DROP INDEX p_another_col_partition_col_idx;
+\c - - - :worker_1_port
+-- check that indexes have been renamed
+-- and that index on p has been dropped (it won't appear)
+SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' ORDER BY 1, 2;
+                 tablename                  |                   indexname
+---------------------------------------------------------------------
+ dist_partitioned_table_910030              | short_910030
+ partition_table_with_very_long_name_910031 | partition_table_with_very_long_name_idx_910031
+(2 rows)
+
+\c - - - :master_port
+SET search_path TO fix_idx_names, public;
+DROP SCHEMA fix_idx_names CASCADE;
+NOTICE:  drop cascades to 5 other objects
+DETAIL:  drop cascades to table not_partitioned
+drop cascades to table not_distributed
+drop cascades to table fk_table
+drop cascades to table dist_partitioned_table
+drop cascades to table p
+SELECT run_command_on_workers($$ DROP SCHEMA IF EXISTS fix_idx_names CASCADE $$);
+      run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,"DROP SCHEMA")
+ (localhost,57638,t,"DROP SCHEMA")
+(2 rows)
+

src/test/regress/expected/multi_index_statements.out

@@ -7,6 +7,7 @@
 -- CREATE TEST TABLES
 --
 CREATE SCHEMA multi_index_statements;
+CREATE SCHEMA multi_index_statements_2;
 SET search_path TO multi_index_statements;
 SET citus.next_shard_id TO 102080;
 CREATE TABLE index_test_range(a int, b int, c int);
@@ -75,6 +76,34 @@ CREATE UNIQUE INDEX index_test_hash_index_a_b_partial ON index_test_hash(a,b) WH
 CREATE UNIQUE INDEX index_test_range_index_a_b_partial ON index_test_range(a,b) WHERE c IS NOT NULL;
 CREATE UNIQUE INDEX index_test_hash_index_a_b_c ON index_test_hash(a) INCLUDE (b,c);
 RESET client_min_messages;
+-- Verify that we can create expression indexes and be robust to different schemas
+CREATE OR REPLACE FUNCTION value_plus_one(a int)
+RETURNS int IMMUTABLE AS $$
+BEGIN
+	RETURN a + 1;
+END;
+$$ LANGUAGE plpgsql;
+SELECT create_distributed_function('value_plus_one(int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE OR REPLACE FUNCTION multi_index_statements_2.value_plus_one(a int)
+RETURNS int IMMUTABLE AS $$
+BEGIN
+	RETURN a + 1;
+END;
+$$ LANGUAGE plpgsql;
+SELECT create_distributed_function('multi_index_statements_2.value_plus_one(int)');
+ create_distributed_function
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE INDEX ON index_test_hash ((value_plus_one(b)));
+CREATE INDEX ON index_test_hash ((multi_index_statements.value_plus_one(b)));
+CREATE INDEX ON index_test_hash ((multi_index_statements_2.value_plus_one(b)));
 -- Verify that we handle if not exists statements correctly
 CREATE INDEX lineitem_orderkey_index on public.lineitem(l_orderkey);
 ERROR:  relation "lineitem_orderkey_index" already exists
@@ -104,6 +133,9 @@ DROP TABLE local_table;
 SELECT * FROM pg_indexes WHERE tablename = 'lineitem' or tablename like 'index_test_%' ORDER BY indexname;
        schemaname       |    tablename     |             indexname              | tablespace |                                                                  indexdef
 ---------------------------------------------------------------------
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx           |            | CREATE INDEX index_test_hash_expr_idx ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx1          |            | CREATE INDEX index_test_hash_expr_idx1 ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx2          |            | CREATE INDEX index_test_hash_expr_idx2 ON multi_index_statements.index_test_hash USING btree (multi_index_statements_2.value_plus_one(b))
  multi_index_statements | index_test_hash  | index_test_hash_index_a            |            | CREATE UNIQUE INDEX index_test_hash_index_a ON multi_index_statements.index_test_hash USING btree (a)
  multi_index_statements | index_test_hash  | index_test_hash_index_a_b          |            | CREATE UNIQUE INDEX index_test_hash_index_a_b ON multi_index_statements.index_test_hash USING btree (a, b)
  multi_index_statements | index_test_hash  | index_test_hash_index_a_b_c        |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c ON multi_index_statements.index_test_hash USING btree (a) INCLUDE (b, c)
@@ -120,7 +152,7 @@ SELECT * FROM pg_indexes WHERE tablename = 'lineitem' or tablename like 'index_t
  public                 | lineitem         | lineitem_partkey_desc_index        |            | CREATE INDEX lineitem_partkey_desc_index ON public.lineitem USING btree (l_partkey DESC)
  public                 | lineitem         | lineitem_pkey                      |            | CREATE UNIQUE INDEX lineitem_pkey ON public.lineitem USING btree (l_orderkey, l_linenumber)
  public                 | lineitem         | lineitem_time_index                |            | CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate)
-(16 rows)
+(19 rows)
 
 \c - - - :worker_1_port
 SELECT count(*) FROM pg_indexes WHERE tablename = (SELECT relname FROM pg_class WHERE relname LIKE 'lineitem%' ORDER BY relname LIMIT 1);
@@ -132,7 +164,7 @@ SELECT count(*) FROM pg_indexes WHERE tablename = (SELECT relname FROM pg_class
 SELECT count(*) FROM pg_indexes WHERE tablename LIKE 'index_test_hash%';
  count
 ---------------------------------------------------------------------
-    32
+    56
 (1 row)
 
 SELECT count(*) FROM pg_indexes WHERE tablename LIKE 'index_test_range%';
@@ -186,6 +218,9 @@ SELECT * FROM pg_indexes WHERE tablename = 'lineitem' or tablename like 'index_t
        schemaname       |    tablename     |             indexname              | tablespace |                                                                  indexdef
 ---------------------------------------------------------------------
  multi_index_statements | index_test_hash  | index_test_hash_a_idx              |            | CREATE UNIQUE INDEX index_test_hash_a_idx ON multi_index_statements.index_test_hash USING btree (a)
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx           |            | CREATE INDEX index_test_hash_expr_idx ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx1          |            | CREATE INDEX index_test_hash_expr_idx1 ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash  | index_test_hash_expr_idx2          |            | CREATE INDEX index_test_hash_expr_idx2 ON multi_index_statements.index_test_hash USING btree (multi_index_statements_2.value_plus_one(b))
  multi_index_statements | index_test_hash  | index_test_hash_index_a            |            | CREATE UNIQUE INDEX index_test_hash_index_a ON multi_index_statements.index_test_hash USING btree (a)
  multi_index_statements | index_test_hash  | index_test_hash_index_a_b          |            | CREATE UNIQUE INDEX index_test_hash_index_a_b ON multi_index_statements.index_test_hash USING btree (a, b)
  multi_index_statements | index_test_hash  | index_test_hash_index_a_b_c        |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c ON multi_index_statements.index_test_hash USING btree (a) INCLUDE (b, c)
@@ -204,7 +239,7 @@ SELECT * FROM pg_indexes WHERE tablename = 'lineitem' or tablename like 'index_t
  public                 | lineitem         | lineitem_partkey_desc_index        |            | CREATE INDEX lineitem_partkey_desc_index ON public.lineitem USING btree (l_partkey DESC)
  public                 | lineitem         | lineitem_pkey                      |            | CREATE UNIQUE INDEX lineitem_pkey ON public.lineitem USING btree (l_orderkey, l_linenumber)
  public                 | lineitem         | lineitem_time_index                |            | CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate)
-(19 rows)
+(22 rows)
 
 --
 -- REINDEX
@@ -258,11 +293,14 @@ SELECT indrelid::regclass, indexrelid::regclass FROM pg_index WHERE indrelid = (
 (2 rows)
 
 SELECT * FROM pg_indexes WHERE tablename LIKE 'index_test_%' ORDER BY indexname;
-       schemaname       |    tablename    |          indexname          | tablespace |                                                         indexdef
+       schemaname       |    tablename    |          indexname          | tablespace |                                                                 indexdef
 ---------------------------------------------------------------------
  multi_index_statements | index_test_hash | index_test_hash_a_idx       |            | CREATE UNIQUE INDEX index_test_hash_a_idx ON multi_index_statements.index_test_hash USING btree (a)
+ multi_index_statements | index_test_hash | index_test_hash_expr_idx    |            | CREATE INDEX index_test_hash_expr_idx ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash | index_test_hash_expr_idx1   |            | CREATE INDEX index_test_hash_expr_idx1 ON multi_index_statements.index_test_hash USING btree (value_plus_one(b))
+ multi_index_statements | index_test_hash | index_test_hash_expr_idx2   |            | CREATE INDEX index_test_hash_expr_idx2 ON multi_index_statements.index_test_hash USING btree (multi_index_statements_2.value_plus_one(b))
  multi_index_statements | index_test_hash | index_test_hash_index_a_b_c |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c ON multi_index_statements.index_test_hash USING btree (a) INCLUDE (b, c)
-(2 rows)
+(5 rows)
 
 \c - - - :worker_1_port
 SELECT indrelid::regclass, indexrelid::regclass FROM pg_index WHERE indrelid = (SELECT relname FROM pg_class WHERE relname LIKE 'lineitem%' ORDER BY relname LIMIT 1)::regclass AND NOT indisprimary AND indexrelid::regclass::text NOT LIKE 'lineitem_time_index%' ORDER BY 1,2;
@@ -273,7 +311,7 @@ SELECT indrelid::regclass, indexrelid::regclass FROM pg_index WHERE indrelid = (
 (2 rows)
 
 SELECT * FROM pg_indexes WHERE tablename LIKE 'index_test_%' ORDER BY indexname;
-       schemaname       |       tablename        |             indexname              | tablespace |                                                                indexdef
+       schemaname       |       tablename        |             indexname              | tablespace |                                                                        indexdef
 ---------------------------------------------------------------------
  multi_index_statements | index_test_hash_102082 | index_test_hash_a_idx_102082       |            | CREATE UNIQUE INDEX index_test_hash_a_idx_102082 ON multi_index_statements.index_test_hash_102082 USING btree (a)
  multi_index_statements | index_test_hash_102083 | index_test_hash_a_idx_102083       |            | CREATE UNIQUE INDEX index_test_hash_a_idx_102083 ON multi_index_statements.index_test_hash_102083 USING btree (a)
@@ -283,6 +321,30 @@ SELECT * FROM pg_indexes WHERE tablename LIKE 'index_test_%' ORDER BY indexname;
  multi_index_statements | index_test_hash_102087 | index_test_hash_a_idx_102087       |            | CREATE UNIQUE INDEX index_test_hash_a_idx_102087 ON multi_index_statements.index_test_hash_102087 USING btree (a)
  multi_index_statements | index_test_hash_102088 | index_test_hash_a_idx_102088       |            | CREATE UNIQUE INDEX index_test_hash_a_idx_102088 ON multi_index_statements.index_test_hash_102088 USING btree (a)
  multi_index_statements | index_test_hash_102089 | index_test_hash_a_idx_102089       |            | CREATE UNIQUE INDEX index_test_hash_a_idx_102089 ON multi_index_statements.index_test_hash_102089 USING btree (a)
+ multi_index_statements | index_test_hash_102082 | index_test_hash_expr_idx1_102082   |            | CREATE INDEX index_test_hash_expr_idx1_102082 ON multi_index_statements.index_test_hash_102082 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102083 | index_test_hash_expr_idx1_102083   |            | CREATE INDEX index_test_hash_expr_idx1_102083 ON multi_index_statements.index_test_hash_102083 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102084 | index_test_hash_expr_idx1_102084   |            | CREATE INDEX index_test_hash_expr_idx1_102084 ON multi_index_statements.index_test_hash_102084 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102085 | index_test_hash_expr_idx1_102085   |            | CREATE INDEX index_test_hash_expr_idx1_102085 ON multi_index_statements.index_test_hash_102085 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102086 | index_test_hash_expr_idx1_102086   |            | CREATE INDEX index_test_hash_expr_idx1_102086 ON multi_index_statements.index_test_hash_102086 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102087 | index_test_hash_expr_idx1_102087   |            | CREATE INDEX index_test_hash_expr_idx1_102087 ON multi_index_statements.index_test_hash_102087 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102088 | index_test_hash_expr_idx1_102088   |            | CREATE INDEX index_test_hash_expr_idx1_102088 ON multi_index_statements.index_test_hash_102088 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102089 | index_test_hash_expr_idx1_102089   |            | CREATE INDEX index_test_hash_expr_idx1_102089 ON multi_index_statements.index_test_hash_102089 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102082 | index_test_hash_expr_idx2_102082   |            | CREATE INDEX index_test_hash_expr_idx2_102082 ON multi_index_statements.index_test_hash_102082 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102083 | index_test_hash_expr_idx2_102083   |            | CREATE INDEX index_test_hash_expr_idx2_102083 ON multi_index_statements.index_test_hash_102083 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102084 | index_test_hash_expr_idx2_102084   |            | CREATE INDEX index_test_hash_expr_idx2_102084 ON multi_index_statements.index_test_hash_102084 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102085 | index_test_hash_expr_idx2_102085   |            | CREATE INDEX index_test_hash_expr_idx2_102085 ON multi_index_statements.index_test_hash_102085 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102086 | index_test_hash_expr_idx2_102086   |            | CREATE INDEX index_test_hash_expr_idx2_102086 ON multi_index_statements.index_test_hash_102086 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102087 | index_test_hash_expr_idx2_102087   |            | CREATE INDEX index_test_hash_expr_idx2_102087 ON multi_index_statements.index_test_hash_102087 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102088 | index_test_hash_expr_idx2_102088   |            | CREATE INDEX index_test_hash_expr_idx2_102088 ON multi_index_statements.index_test_hash_102088 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102089 | index_test_hash_expr_idx2_102089   |            | CREATE INDEX index_test_hash_expr_idx2_102089 ON multi_index_statements.index_test_hash_102089 USING btree (multi_index_statements_2.value_plus_one(b))
+ multi_index_statements | index_test_hash_102082 | index_test_hash_expr_idx_102082    |            | CREATE INDEX index_test_hash_expr_idx_102082 ON multi_index_statements.index_test_hash_102082 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102083 | index_test_hash_expr_idx_102083    |            | CREATE INDEX index_test_hash_expr_idx_102083 ON multi_index_statements.index_test_hash_102083 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102084 | index_test_hash_expr_idx_102084    |            | CREATE INDEX index_test_hash_expr_idx_102084 ON multi_index_statements.index_test_hash_102084 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102085 | index_test_hash_expr_idx_102085    |            | CREATE INDEX index_test_hash_expr_idx_102085 ON multi_index_statements.index_test_hash_102085 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102086 | index_test_hash_expr_idx_102086    |            | CREATE INDEX index_test_hash_expr_idx_102086 ON multi_index_statements.index_test_hash_102086 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102087 | index_test_hash_expr_idx_102087    |            | CREATE INDEX index_test_hash_expr_idx_102087 ON multi_index_statements.index_test_hash_102087 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102088 | index_test_hash_expr_idx_102088    |            | CREATE INDEX index_test_hash_expr_idx_102088 ON multi_index_statements.index_test_hash_102088 USING btree (multi_index_statements.value_plus_one(b))
+ multi_index_statements | index_test_hash_102089 | index_test_hash_expr_idx_102089    |            | CREATE INDEX index_test_hash_expr_idx_102089 ON multi_index_statements.index_test_hash_102089 USING btree (multi_index_statements.value_plus_one(b))
  multi_index_statements | index_test_hash_102082 | index_test_hash_index_a_b_c_102082 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102082 ON multi_index_statements.index_test_hash_102082 USING btree (a) INCLUDE (b, c)
  multi_index_statements | index_test_hash_102083 | index_test_hash_index_a_b_c_102083 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102083 ON multi_index_statements.index_test_hash_102083 USING btree (a) INCLUDE (b, c)
  multi_index_statements | index_test_hash_102084 | index_test_hash_index_a_b_c_102084 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102084 ON multi_index_statements.index_test_hash_102084 USING btree (a) INCLUDE (b, c)
@@ -291,7 +353,7 @@ SELECT * FROM pg_indexes WHERE tablename LIKE 'index_test_%' ORDER BY indexname;
  multi_index_statements | index_test_hash_102087 | index_test_hash_index_a_b_c_102087 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102087 ON multi_index_statements.index_test_hash_102087 USING btree (a) INCLUDE (b, c)
  multi_index_statements | index_test_hash_102088 | index_test_hash_index_a_b_c_102088 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102088 ON multi_index_statements.index_test_hash_102088 USING btree (a) INCLUDE (b, c)
  multi_index_statements | index_test_hash_102089 | index_test_hash_index_a_b_c_102089 |            | CREATE UNIQUE INDEX index_test_hash_index_a_b_c_102089 ON multi_index_statements.index_test_hash_102089 USING btree (a) INCLUDE (b, c)
-(16 rows)
+(40 rows)
 
 -- create index that will conflict with master operations
 CREATE INDEX CONCURRENTLY ith_b_idx_102089 ON multi_index_statements.index_test_hash_102089(b);
@@ -614,3 +676,4 @@ SELECT indisvalid AS "Index Valid?" FROM pg_index WHERE indexrelid='ith_b_idx'::
 -- final clean up
 DROP INDEX CONCURRENTLY IF EXISTS ith_b_idx;
 DROP SCHEMA multi_index_statements CASCADE;
+DROP SCHEMA multi_index_statements_2 CASCADE;

src/test/regress/expected/multi_insert_select.out

@@ -667,7 +667,7 @@ DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
 DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE fist_table_agg: SELECT (max(value_1) OPERATOR(pg_catalog.+) 1) AS v1_agg, user_id FROM public.raw_events_first GROUP BY user_id
 DEBUG:  Router planner cannot handle multi-shard select queries
-DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT user_id, v1_agg AS value_1_agg FROM (SELECT fist_table_agg.v1_agg, fist_table_agg.user_id FROM (SELECT intermediate_result.v1_agg, intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v1_agg integer, user_id integer)) fist_table_agg) citus_insert_select_subquery
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT user_id, v1_agg AS value_1_agg FROM (SELECT fist_table_agg.user_id, fist_table_agg.v1_agg FROM (SELECT intermediate_result.v1_agg, intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v1_agg integer, user_id integer)) fist_table_agg) citus_insert_select_subquery
 DEBUG:  Creating router plan
 DEBUG:  Collecting INSERT ... SELECT results on coordinator
 ROLLBACK;
@@ -2712,7 +2712,6 @@ WITH top10 AS (
 )
 INSERT INTO dist_table_with_sequence (value_1)
 SELECT * FROM top10;
-ERROR:  cannot handle complex subqueries when the router executor is disabled
 SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1;
  user_id | value_1
 ---------------------------------------------------------------------
@@ -2797,7 +2796,6 @@ WITH top10 AS (
 )
 INSERT INTO dist_table_with_user_sequence (value_1)
 SELECT * FROM top10;
-ERROR:  cannot handle complex subqueries when the router executor is disabled
 SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1;
  user_id | value_1
 ---------------------------------------------------------------------
@@ -3234,6 +3232,47 @@ INSERT INTO raw_events_first
 SELECT * FROM raw_events_first OFFSET 0
 ON CONFLICT DO NOTHING;
 ABORT;
+-- test fix for issue https://github.com/citusdata/citus/issues/5891
+CREATE TABLE dist_table_1(
+dist_col integer,
+int_col integer,
+text_col_1 text,
+text_col_2 text
+);
+SELECT create_distributed_table('dist_table_1', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO dist_table_1 VALUES (1, 1, 'string', 'string');
+CREATE TABLE dist_table_2(
+dist_col integer,
+int_col integer
+);
+SELECT create_distributed_table('dist_table_2', 'dist_col');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO dist_table_2 VALUES (1, 1);
+with a as (select random()) INSERT INTO dist_table_1
+SELECT
+t1.dist_col,
+1,
+'string',
+'string'
+FROM a, dist_table_1 t1
+join dist_table_2 t2 using (dist_col)
+limit 1
+returning text_col_1;
+ text_col_1
+---------------------------------------------------------------------
+ string
+(1 row)
+
+DROP TABLE dist_table_1, dist_table_2;
 -- wrap in a transaction to improve performance
 BEGIN;
 DROP TABLE coerce_events;

src/test/regress/expected/multi_multiuser.out

@@ -265,19 +265,26 @@ SELECT * FROM columnar.stripe;
 
 -- alter a columnar setting
 SET columnar.chunk_group_row_limit = 1050;
-DO $proc$
+-- create columnar table
-BEGIN
+CREATE TABLE columnar_table (a int) USING columnar;
-IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
+-- alter a columnar table that is created by that unprivileged user
-  EXECUTE $$
+SELECT alter_columnar_table_set('columnar_table', chunk_group_row_limit => 2000);
-    -- create columnar table
+ alter_columnar_table_set
-    CREATE TABLE columnar_table (a int) USING columnar;
+---------------------------------------------------------------------
-    -- alter a columnar table that is created by that unprivileged user
+
-    SELECT alter_columnar_table_set('columnar_table', chunk_group_row_limit => 2000);
+(1 row)
-    -- and drop it
+
-    DROP TABLE columnar_table;
+-- insert some data and read
-  $$;
+INSERT INTO columnar_table VALUES (1), (1);
-END IF;
+SELECT * FROM columnar_table;
-END$proc$;
+ a
+---------------------------------------------------------------------
+ 1
+ 1
+(2 rows)
+
+-- and drop it
+DROP TABLE columnar_table;
 -- cannot modify columnar metadata table as unprivileged user
 INSERT INTO columnar.stripe VALUES(99);
 ERROR:  permission denied for table stripe
@@ -286,6 +293,9 @@ ERROR:  permission denied for table stripe
 -- (since citus extension has a dependency to it)
 DROP TABLE columnar.chunk;
 ERROR:  must be owner of table chunk
+-- cannot read columnar.chunk since it could expose chunk min/max values
+SELECT * FROM columnar.chunk;
+ERROR:  permission denied for table chunk
 -- test whether a read-only user can read from citus_tables view
 SELECT distribution_column FROM citus_tables WHERE table_name = 'test'::regclass;
  distribution_column