Adds PG16Beta3 support (#6952)

DESCRIPTION: Adds PG16Beta3 support This is the final commit that adds PG16 compatibility with Citus's current features. You can use Citus community with PG16Beta3. This commit: - Enables PG16 in the configure script. - Adds PG16 tests to CI using test images that have 16beta3 - Skips wal2json cdc test since wal2json package is not available for PG16 yet - Fixes an isolation test Several PG16 Compatibility commits have been merged before this final one. All these subtasks are done https://github.com/citusdata/citus/issues/7017 See the list below: 1 - 42d956888d Resolve compilation issues 2 - 0d503dd5ac Ruleutils and successful CREATE EXTENSION 3 - 907d72e60d Some test outputs 4 - 7c6b4ce103 Outer join checks, subscription password, crash fixes 5 - 6056cb2c29 get_relation_info hook to avoid crash from adjusted partitioning 6 - b36c431abb Rework PlannedStmt and Query's Permission Info 7 - ee3153fe50 More test output fixes 8 - 2c50b5f7ff varnullingrels additions 9 - b2291374b4 More test output fixes 10- a2315fdc67 New options to vacuum and analyze 11- 9fa72545e2 Fix AM dependency and grant's admin option 12- 2d6cf8e79a One more outer join check Stay tuned for PG16 new features in Citus :)
2023-08-17 21:02:59 +03:00 · 2023-08-17 21:02:59 +03:00 · 682dca1f12
parent 2d6cf8e79a
commit 682dca1f12
10 changed files with 207 additions and 35 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -6,16 +6,19 @@ orbs:
 parameters:
  image_suffix:
    type: string
-    default: '-v1b94240'
+    default: '-v0c8d80c'
  pg14_version:
    type: string
    default: '14.9'
  pg15_version:
    type: string
    default: '15.4'
+  pg16_version:
+    type: string
+    default: '16beta3'
  upgrade_pg_versions:
    type: string
-    default: '14.9-15.4'
+    default: '14.9-15.4-16beta3'
  style_checker_tools_version:
    type: string
    default: '0.8.18'
@ -722,6 +725,10 @@ workflows:
          name: build-15
          pg_major: 15
          image_tag: '<< pipeline.parameters.pg15_version >>'
+      - build:
+          name: build-16
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'

      - check-style
      - check-sql-snapshots
@ -872,6 +879,79 @@ workflows:
          image: citus/failtester
          make: check-failure

+      - test-citus: &test-citus-16
+          name: 'test-16_check-split'
+          make: check-split
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise'
+          make: check-enterprise
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation'
+          make: check-enterprise-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-1'
+          make: check-enterprise-isolation-logicalrep-1
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-2'
+          make: check-enterprise-isolation-logicalrep-2
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-isolation-logicalrep-3'
+          make: check-enterprise-isolation-logicalrep-3
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-enterprise-failure'
+          image: citus/failtester
+          make: check-enterprise-failure
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-multi'
+          make: check-multi
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-multi-1'
+          make: check-multi-1
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-mx'
+          make: check-multi-mx
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-vanilla'
+          make: check-vanilla
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-isolation'
+          make: check-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-operations'
+          make: check-operations
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-follower-cluster'
+          make: check-follower-cluster
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-columnar'
+          make: check-columnar
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-columnar-isolation'
+          make: check-columnar-isolation
+      - test-citus:
+          <<:  *test-citus-16
+          name: 'test-16_check-failure'
+          image: citus/failtester
+          make: check-failure
+
      - test-pytest:
          name: 'test-14_pytest'
          pg_major: 14
@ -884,6 +964,12 @@ workflows:
          image_tag: '<< pipeline.parameters.pg15_version >>'
          requires: [build-15]

+      - test-pytest:
+          name: 'test-16_pytest'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
      - tap-test-citus:
          name: 'test-15_tap-cdc'
          suite: cdc
@ -891,6 +977,13 @@ workflows:
          image_tag: '<< pipeline.parameters.pg15_version >>'
          requires: [build-15]

+      - tap-test-citus:
+          name: 'test-16_tap-cdc'
+          suite: cdc
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
      - test-arbitrary-configs:
          name: 'test-14_check-arbitrary-configs'
          pg_major: 14
@ -903,6 +996,12 @@ workflows:
          image_tag: '<< pipeline.parameters.pg15_version >>'
          requires: [build-15]

+      - test-arbitrary-configs:
+          name: 'test-16_check-arbitrary-configs'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
      - test-query-generator:
          name: 'test-14_check-query-generator'
          pg_major: 14
@ -915,6 +1014,12 @@ workflows:
          image_tag: '<< pipeline.parameters.pg15_version >>'
          requires: [build-15]

+      - test-query-generator:
+          name: 'test-16_check-query-generator'
+          pg_major: 16
+          image_tag: '<< pipeline.parameters.pg16_version >>'
+          requires: [build-16]
+
      - test-pg-upgrade:
          name: 'test-14-15_check-pg-upgrade'
          old_pg_major: 14
@ -922,6 +1027,20 @@ workflows:
          image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
          requires: [build-14, build-15]

+      - test-pg-upgrade:
+          name: 'test-15-16_check-pg-upgrade'
+          old_pg_major: 15
+          new_pg_major: 16
+          image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
+          requires: [build-15, build-16]
+
+      - test-pg-upgrade:
+          name: 'test-14-16_check-pg-upgrade'
+          old_pg_major: 14
+          new_pg_major: 16
+          image_tag: '<< pipeline.parameters.upgrade_pg_versions >>'
+          requires: [build-14, build-16]
+
      - test-citus-upgrade:
          name: test-14_check-citus-upgrade
          pg_major: 14
@ -968,7 +1087,28 @@ workflows:
            - test-15_check-split
            - test-15_check-arbitrary-configs
            - test-15_check-query-generator
+            - test-16_check-multi
+            - test-16_check-multi-1
+            - test-16_check-mx
+            - test-16_check-vanilla
+            - test-16_check-isolation
+            - test-16_check-operations
+            - test-16_check-follower-cluster
+            - test-16_check-columnar
+            - test-16_check-columnar-isolation
+            - test-16_check-failure
+            - test-16_check-enterprise
+            - test-16_check-enterprise-isolation
+            - test-16_check-enterprise-isolation-logicalrep-1
+            - test-16_check-enterprise-isolation-logicalrep-2
+            - test-16_check-enterprise-isolation-logicalrep-3
+            - test-16_check-enterprise-failure
+            - test-16_check-split
+            - test-16_check-arbitrary-configs
+            - test-16_check-query-generator
            - test-14-15_check-pg-upgrade
+            - test-15-16_check-pg-upgrade
+            - test-14-16_check-pg-upgrade
            - test-14_check-citus-upgrade

      - ch_benchmark:
--- a/2
+++ b/2
@ -2588,7 +2588,7 @@ fi
 if test "$with_pg_version_check" = no; then
    { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5
 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;}
-elif test "$version_num" != '14' -a "$version_num" != '15'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
   as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5
 else
   { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5
--- a/configure.ac
+++ b/configure.ac
@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check)

 if test "$with_pg_version_check" = no; then
    AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)])
-elif test "$version_num" != '14' -a "$version_num" != '15'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
   AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.])
 else
   AC_MSG_NOTICE([building against PostgreSQL $version_num])
--- a/src/backend/distributed/commands/index.c
+++ b/src/backend/distributed/commands/index.c
@ -561,7 +561,7 @@ ReindexStmtFindRelationOid(ReindexStmt *reindexStmt, bool missingOk)
 	{
 		relationId = RangeVarGetRelidExtended(reindexStmt->relation, lockmode,
 											  (missingOk) ? RVR_MISSING_OK : 0,
-											  RANGE_VAR_TABLE_CALLBACK, NULL);
+											  RangeVarCallbackOwnsTable, NULL);
 	}

 	return relationId;
--- a/src/include/pg_version_compat.h
+++ b/src/include/pg_version_compat.h
@ -27,8 +27,6 @@

 #define float_abs(a) fabs(a)

-#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackMaintainsTable
-
 #define tuplesort_getdatum_compat(a, b, c, d, e, f) tuplesort_getdatum(a, b, c, d, e, f)

 static inline struct config_generic **
@ -57,8 +55,6 @@ get_guc_variables_compat(int *gucCount)
 #include "utils/guc.h"
 #include "utils/guc_tables.h"

-#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackOwnsTable
-
 #define pg_clean_ascii_compat(a, b) pg_clean_ascii(a)

 #define RelationPhysicalIdentifier_compat(a) ((a)->rd_node)
--- a/src/test/cdc/t/016_cdc_wal2json.pl
+++ b/src/test/cdc/t/016_cdc_wal2json.pl
@ -9,6 +9,13 @@ use cdctestlib;

 use threads;

+my $pg_major_version =  int($ENV{'pg_major_version'});
+print("working with PG major version : $pg_major_version\n");
+if ($pg_major_version >= 16) {
+	plan skip_all => 'wal2json is not available for PG16 yet';
+    exit 0;
+}
+
 # Initialize co-ordinator node
 my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;);
 my $result = 0;
--- a/src/test/regress/citus_tests/common.py
+++ b/src/test/regress/citus_tests/common.py
@ -92,6 +92,7 @@ PG_MAJOR_VERSION = get_pg_major_version()
 OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = {
    14: "10.2.0",
    15: "11.1.5",
+    16: "12.1devel",
 }

 OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION]
--- a/src/test/regress/expected/isolation_add_remove_node.out
+++ b/src/test/regress/expected/isolation_add_remove_node.out
@ -698,7 +698,7 @@ master_remove_node
 (1 row)


-starting permutation: s1-add-node-1 s1-begin s1-disable-node-1 s2-disable-node-1 s1-abort s1-show-nodes
+starting permutation: s1-add-node-1 s1-add-node-2 s1-begin s1-disable-node-2 s2-disable-node-2 s1-abort s2-wait-metadata-sync s1-show-nodes
 step s1-add-node-1:
 SELECT 1 FROM master_add_node('localhost', 57637);

@ -707,11 +707,19 @@ step s1-add-node-1:
       1
 (1 row)

+step s1-add-node-2:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
 step s1-begin:
 BEGIN;

-step s1-disable-node-1:
-	SELECT 1 FROM master_disable_node('localhost', 57637);
+step s1-disable-node-2:
+ SELECT 1 FROM master_disable_node('localhost', 57638);
 SELECT public.wait_until_metadata_sync();

 ?column?
@ -724,19 +732,21 @@ wait_until_metadata_sync

 (1 row)

-step s2-disable-node-1:
-	SELECT 1 FROM master_disable_node('localhost', 57637);
-	SELECT public.wait_until_metadata_sync();
+step s2-disable-node-2:
+ SELECT 1 FROM master_disable_node('localhost', 57638);
 <waiting ...>
 step s1-abort: 
 ABORT;

-step s2-disable-node-1: <... completed>
+step s2-disable-node-2: <... completed>
 ?column?
 ---------------------------------------------------------------------
       1
 (1 row)

+step s2-wait-metadata-sync:
+ SELECT public.wait_until_metadata_sync();
+
 wait_until_metadata_sync
 ---------------------------------------------------------------------

@ -747,11 +757,13 @@ step s1-show-nodes:

 nodename |nodeport|isactive
 ---------------------------------------------------------------------
-localhost|   57637|f
-(1 row)
+localhost|   57637|t
+localhost|   57638|f
+(2 rows)

 master_remove_node
 ---------------------------------------------------------------------

-(1 row)
+
+(2 rows)

--- a/src/test/regress/expected/isolation_update_node.out
+++ b/src/test/regress/expected/isolation_update_node.out
@ -3,8 +3,8 @@ Parsed test spec with 3 sessions
 starting permutation: s1-begin s1-update-node-1 s2-update-node-2 s1-commit s1-show-nodes s3-update-node-1-back s3-update-node-2-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    22|localhost|   57638
-    21|localhost|   57637
+    23|localhost|   57638
+    22|localhost|   57637
 (2 rows)

 step s1-begin:
@ -43,8 +43,8 @@ step s1-show-nodes:

 nodeid|nodename |nodeport|isactive
 ---------------------------------------------------------------------
-    21|localhost|   58637|t
-    22|localhost|   58638|t
+    22|localhost|   58637|t
+    23|localhost|   58638|t
 (2 rows)

 step s3-update-node-1-back:
@ -93,8 +93,8 @@ nodeid|nodename|nodeport
 starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes s3-update-node-1-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    24|localhost|   57638
-    23|localhost|   57637
+    25|localhost|   57638
+    24|localhost|   57637
 (2 rows)

 step s1-begin:
@ -139,8 +139,8 @@ step s1-show-nodes:

 nodeid|nodename |nodeport|isactive
 ---------------------------------------------------------------------
-    24|localhost|   57638|t
-    23|localhost|   58637|t
+    25|localhost|   57638|t
+    24|localhost|   58637|t
 (2 rows)

 step s3-update-node-1-back:
@ -178,8 +178,8 @@ nodeid|nodename|nodeport
 starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    26|localhost|   57638
-    25|localhost|   57637
+    27|localhost|   57638
+    26|localhost|   57637
 (2 rows)

 step s2-create-table:
--- a/src/test/regress/spec/isolation_add_remove_node.spec
+++ b/src/test/regress/spec/isolation_add_remove_node.spec
@ -45,6 +45,12 @@ step "s1-disable-node-1"
 	SELECT public.wait_until_metadata_sync();
 }

+step "s1-disable-node-2"
+{
+	SELECT 1 FROM master_disable_node('localhost', 57638);
+	SELECT public.wait_until_metadata_sync();
+}
+
 step "s1-remove-node-1"
 {
 	SELECT * FROM master_remove_node('localhost', 57637);
@ -88,6 +94,16 @@ step "s2-disable-node-1"
 	SELECT public.wait_until_metadata_sync();
 }

+step "s2-disable-node-2"
+{
+	SELECT 1 FROM master_disable_node('localhost', 57638);
+}
+
+step "s2-wait-metadata-sync"
+{
+	SELECT public.wait_until_metadata_sync();
+}
+
 step "s2-remove-node-1"
 {
 	SELECT * FROM master_remove_node('localhost', 57637);
@ -135,4 +151,4 @@ permutation "s1-add-inactive-1" "s1-begin" "s1-disable-node-1" "s2-activate-node
 permutation "s1-add-inactive-1" "s1-begin" "s1-activate-node-1" "s2-disable-node-1" "s1-commit" "s1-show-nodes"

 // disable an active node from 2 transactions, one aborts
-permutation "s1-add-node-1" "s1-begin" "s1-disable-node-1" "s2-disable-node-1" "s1-abort" "s1-show-nodes"
+permutation "s1-add-node-1" "s1-add-node-2" "s1-begin" "s1-disable-node-2" "s2-disable-node-2" "s1-abort" "s2-wait-metadata-sync" "s1-show-nodes"