Adds PG16Beta3 support (#6952)

DESCRIPTION: Adds PG16Beta3 support

This is the final commit that adds
PG16 compatibility with Citus's current features.

You can use Citus community with PG16Beta3. This commit:

- Enables PG16 in the configure script.
- Adds PG16 tests to CI using test images that have 16beta3
- Skips wal2json cdc test since wal2json package is not available for PG16 yet
- Fixes an isolation test

Several PG16 Compatibility commits have been merged before this final one.
All these subtasks are done https://github.com/citusdata/citus/issues/7017
See the list below:

1 - 42d956888d
Resolve compilation issues
2 - 0d503dd5ac
Ruleutils and successful CREATE EXTENSION
3 - 907d72e60d
Some test outputs
4 - 7c6b4ce103
Outer join checks, subscription password, crash fixes
5 - 6056cb2c29
get_relation_info hook to avoid crash from adjusted partitioning
6 - b36c431abb
Rework PlannedStmt and Query's Permission Info
7 - ee3153fe50
More test output fixes
8 - 2c50b5f7ff
varnullingrels additions
9 - b2291374b4
More test output fixes
10- a2315fdc67
New options to vacuum and analyze
11- 9fa72545e2
Fix AM dependency and grant's admin option
12- 2d6cf8e79a
One more outer join check

Stay tuned for PG16 new features in Citus :)

.circleci/config.yml

@@ -6,16 +6,19 @@ orbs:
 parameters:
   image_suffix:
     type: string
-    default: '-v1b94240'
+    default: '-v0c8d80c'
   pg14_version:
     type: string
     default: '14.9'
   pg15_version:
     type: string
     default: '15.4'
+  pg16_version:
+    type: string
+    default: '16beta3'
   upgrade_pg_versions:
     type: string
-    default: '14.9-15.4'
+    default: '14.9-15.4-16beta3'
   style_checker_tools_version:
     type: string
     default: '0.8.18'
@@ -722,6 +725,10 @@ workflows:
           name: build-15
           pg_major: 15
           image_tag: '<< pipeline.parameters.pg15_version >>'
+      - build:
+          name: build-16
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
 
       - check-style
       - check-sql-snapshots
@@ -872,6 +879,79 @@ workflows:
           image: citus/failtester
           make: check-failure
 
+      - test-citus: &test-citus-16
+          name: 'test-16_check-split'
+          make: check-split
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise'
+          make: check-enterprise
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation'
+          make: check-enterprise-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-1'
+          make: check-enterprise-isolation-logicalrep-1
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-2'
+          make: check-enterprise-isolation-logicalrep-2
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-3'
+          make: check-enterprise-isolation-logicalrep-3
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-failure'
+          image: citus/failtester
+          make: check-enterprise-failure
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-multi'
+          make: check-multi
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-multi-1'
+          make: check-multi-1
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-mx'
+          make: check-multi-mx
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-vanilla'
+          make: check-vanilla
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-isolation'
+          make: check-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-operations'
+          make: check-operations
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-follower-cluster'
+          make: check-follower-cluster
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-columnar'
+          make: check-columnar
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-columnar-isolation'
+          make: check-columnar-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-failure'
+          image: citus/failtester
+          make: check-failure
+
       - test-pytest:
           name: 'test-14_pytest'
           pg_major: 14
@@ -884,6 +964,12 @@ workflows:
           image_tag: '<< pipeline.parameters.pg15_version >>'
           requires: [build-15]
 
+      - test-pytest:
+          name: 'test-16_pytest'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
       - tap-test-citus:
           name: 'test-15_tap-cdc'
           suite: cdc
@@ -891,6 +977,13 @@ workflows:
           image_tag: '<< pipeline.parameters.pg15_version >>'
           requires: [build-15]
 
+      - tap-test-citus:
+          name: 'test-16_tap-cdc'
+          suite: cdc
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
       - test-arbitrary-configs:
           name: 'test-14_check-arbitrary-configs'
           pg_major: 14
@@ -903,6 +996,12 @@ workflows:
           image_tag: '<< pipeline.parameters.pg15_version >>'
           requires: [build-15]
 
+      - test-arbitrary-configs:
+          name: 'test-16_check-arbitrary-configs'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
       - test-query-generator:
           name: 'test-14_check-query-generator'
           pg_major: 14
@@ -915,6 +1014,12 @@ workflows:
           image_tag: '<< pipeline.parameters.pg15_version >>'
           requires: [build-15]
 
+      - test-query-generator:
+          name: 'test-16_check-query-generator'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
       - test-pg-upgrade:
           name: 'test-14-15_check-pg-upgrade'
           old_pg_major: 14
@@ -922,6 +1027,20 @@ workflows:
           image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
           requires: [build-14, build-15]
 
+      - test-pg-upgrade:
+          name: 'test-15-16_check-pg-upgrade'
+          old_pg_major: 15
+          new_pg_major: 16
+          image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
+          requires: [build-15, build-16]
+
+      - test-pg-upgrade:
+          name: 'test-14-16_check-pg-upgrade'
+          old_pg_major: 14
+          new_pg_major: 16
+          image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
+          requires: [build-14, build-16]
+
       - test-citus-upgrade:
           name: test-14_check-citus-upgrade
           pg_major: 14
@@ -968,7 +1087,28 @@ workflows:
             - test-15_check-split
             - test-15_check-arbitrary-configs
             - test-15_check-query-generator
+            - test-16_check-multi
+            - test-16_check-multi-1
+            - test-16_check-mx
+            - test-16_check-vanilla
+            - test-16_check-isolation
+            - test-16_check-operations
+            - test-16_check-follower-cluster
+            - test-16_check-columnar
+            - test-16_check-columnar-isolation
+            - test-16_check-failure
+            - test-16_check-enterprise
+            - test-16_check-enterprise-isolation
+            - test-16_check-enterprise-isolation-logicalrep-1
+            - test-16_check-enterprise-isolation-logicalrep-2
+            - test-16_check-enterprise-isolation-logicalrep-3
+            - test-16_check-enterprise-failure
+            - test-16_check-split
+            - test-16_check-arbitrary-configs
+            - test-16_check-query-generator
             - test-14-15_check-pg-upgrade
+            - test-15-16_check-pg-upgrade
+            - test-14-16_check-pg-upgrade
             - test-14_check-citus-upgrade
 
       - ch_benchmark:

configure

@@ -2588,7 +2588,7 @@ fi
 if test "$with_pg_version_check" = no; then
     { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5
 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;}
-elif test "$version_num" != '14' -a "$version_num" != '15'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
    as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5
 else
    { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5

configure.ac

@@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check)
 
 if test "$with_pg_version_check" = no; then
     AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)])
-elif test "$version_num" != '14' -a "$version_num" != '15'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
    AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.])
 else
    AC_MSG_NOTICE([building against PostgreSQL $version_num])

src/backend/distributed/commands/index.c

@@ -561,7 +561,7 @@ ReindexStmtFindRelationOid(ReindexStmt *reindexStmt, bool missingOk)
 	{
 		relationId = RangeVarGetRelidExtended(reindexStmt->relation, lockmode,
 											  (missingOk) ? RVR_MISSING_OK : 0,
-											  RANGE_VAR_TABLE_CALLBACK, NULL);
+											  RangeVarCallbackOwnsTable, NULL);
 	}
 
 	return relationId;

src/include/pg_version_compat.h

@@ -27,8 +27,6 @@
 
 #define float_abs(a) fabs(a)
 
-#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackMaintainsTable
-
 #define tuplesort_getdatum_compat(a, b, c, d, e, f) tuplesort_getdatum(a, b, c, d, e, f)
 
 static inline struct config_generic **
@@ -57,8 +55,6 @@ get_guc_variables_compat(int *gucCount)
 #include "utils/guc.h"
 #include "utils/guc_tables.h"
 
-#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackOwnsTable
-
 #define pg_clean_ascii_compat(a, b) pg_clean_ascii(a)
 
 #define RelationPhysicalIdentifier_compat(a) ((a)->rd_node)

src/test/cdc/t/016_cdc_wal2json.pl

@@ -9,6 +9,13 @@ use cdctestlib;
 
 use threads;
 
+my $pg_major_version =  int($ENV{'pg_major_version'});
+print("working with PG major version : $pg_major_version\n");
+if ($pg_major_version >= 16) {
+	plan skip_all => 'wal2json is not available for PG16 yet';
+    exit 0;
+}
+
 # Initialize co-ordinator node
 my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;);
 my $result = 0;

src/test/regress/citus_tests/common.py

@@ -92,6 +92,7 @@ PG_MAJOR_VERSION = get_pg_major_version()
 OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = {
     14: "10.2.0",
     15: "11.1.5",
+    16: "12.1devel",
 }
 
 OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION]

src/test/regress/expected/isolation_add_remove_node.out

@@ -698,7 +698,7 @@ master_remove_node
 (1 row)
 
 
-starting permutation: s1-add-node-1 s1-begin s1-disable-node-1 s2-disable-node-1 s1-abort s1-show-nodes
+starting permutation: s1-add-node-1 s1-add-node-2 s1-begin s1-disable-node-2 s2-disable-node-2 s1-abort s2-wait-metadata-sync s1-show-nodes
 step s1-add-node-1:
  SELECT 1 FROM master_add_node('localhost', 57637);
 
@@ -707,11 +707,19 @@ step s1-add-node-1:
        1
 (1 row)
 
+step s1-add-node-2:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
 step s1-begin:
  BEGIN;
 
-step s1-disable-node-1:
+step s1-disable-node-2:
-	SELECT 1 FROM master_disable_node('localhost', 57637);
+ SELECT 1 FROM master_disable_node('localhost', 57638);
  SELECT public.wait_until_metadata_sync();
 
 ?column?
@@ -724,19 +732,21 @@ wait_until_metadata_sync
 
 (1 row)
 
-step s2-disable-node-1:
+step s2-disable-node-2:
-	SELECT 1 FROM master_disable_node('localhost', 57637);
+ SELECT 1 FROM master_disable_node('localhost', 57638);
-	SELECT public.wait_until_metadata_sync();
  <waiting ...>
 step s1-abort: 
  ABORT;
 
-step s2-disable-node-1: <... completed>
+step s2-disable-node-2: <... completed>
 ?column?
 ---------------------------------------------------------------------
        1
 (1 row)
 
+step s2-wait-metadata-sync:
+ SELECT public.wait_until_metadata_sync();
+
 wait_until_metadata_sync
 ---------------------------------------------------------------------
 
@@ -747,11 +757,13 @@ step s1-show-nodes:
 
 nodename |nodeport|isactive
 ---------------------------------------------------------------------
-localhost|   57637|f
+localhost|   57637|t
-(1 row)
+localhost|   57638|f
+(2 rows)
 
 master_remove_node
 ---------------------------------------------------------------------
 
-(1 row)
+
+(2 rows)
 

src/test/regress/expected/isolation_update_node.out

@@ -3,8 +3,8 @@ Parsed test spec with 3 sessions
 starting permutation: s1-begin s1-update-node-1 s2-update-node-2 s1-commit s1-show-nodes s3-update-node-1-back s3-update-node-2-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    22|localhost|   57638
+    23|localhost|   57638
-    21|localhost|   57637
+    22|localhost|   57637
 (2 rows)
 
 step s1-begin:
@@ -43,8 +43,8 @@ step s1-show-nodes:
 
 nodeid|nodename |nodeport|isactive
 ---------------------------------------------------------------------
-    21|localhost|   58637|t
+    22|localhost|   58637|t
-    22|localhost|   58638|t
+    23|localhost|   58638|t
 (2 rows)
 
 step s3-update-node-1-back:
@@ -93,8 +93,8 @@ nodeid|nodename|nodeport
 starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes s3-update-node-1-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    24|localhost|   57638
+    25|localhost|   57638
-    23|localhost|   57637
+    24|localhost|   57637
 (2 rows)
 
 step s1-begin:
@@ -139,8 +139,8 @@ step s1-show-nodes:
 
 nodeid|nodename |nodeport|isactive
 ---------------------------------------------------------------------
-    24|localhost|   57638|t
+    25|localhost|   57638|t
-    23|localhost|   58637|t
+    24|localhost|   58637|t
 (2 rows)
 
 step s3-update-node-1-back:
@@ -178,8 +178,8 @@ nodeid|nodename|nodeport
 starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    26|localhost|   57638
+    27|localhost|   57638
-    25|localhost|   57637
+    26|localhost|   57637
 (2 rows)
 
 step s2-create-table:

src/test/regress/spec/isolation_add_remove_node.spec

@@ -45,6 +45,12 @@ step "s1-disable-node-1"
 	SELECT public.wait_until_metadata_sync();
 }
 
+step "s1-disable-node-2"
+{
+	SELECT 1 FROM master_disable_node('localhost', 57638);
+	SELECT public.wait_until_metadata_sync();
+}
+
 step "s1-remove-node-1"
 {
 	SELECT * FROM master_remove_node('localhost', 57637);
@@ -88,6 +94,16 @@ step "s2-disable-node-1"
 	SELECT public.wait_until_metadata_sync();
 }
 
+step "s2-disable-node-2"
+{
+	SELECT 1 FROM master_disable_node('localhost', 57638);
+}
+
+step "s2-wait-metadata-sync"
+{
+	SELECT public.wait_until_metadata_sync();
+}
+
 step "s2-remove-node-1"
 {
 	SELECT * FROM master_remove_node('localhost', 57637);
@@ -135,4 +151,4 @@ permutation "s1-add-inactive-1" "s1-begin" "s1-disable-node-1" "s2-activate-node
 permutation "s1-add-inactive-1" "s1-begin" "s1-activate-node-1" "s2-disable-node-1" "s1-commit" "s1-show-nodes"
 
 // disable an active node from 2 transactions, one aborts
-permutation "s1-add-node-1" "s1-begin" "s1-disable-node-1" "s2-disable-node-1" "s1-abort" "s1-show-nodes"
+permutation "s1-add-node-1" "s1-add-node-2" "s1-begin" "s1-disable-node-2" "s2-disable-node-2" "s1-abort" "s2-wait-metadata-sync" "s1-show-nodes"