From f8d36f79d731384a6d9e646b5ad2096f7d8bf839 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Wed, 17 Dec 2025 09:51:19 +0300 Subject: [PATCH] Adds Citus compatibility with PG18.1 (#8065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Citus compatibility with PostgreSQL 18.1 by wiring PG18 into the devcontainer, configure checks, and CI matrices. ### Summary - Build PostgreSQL 18.1 in the devcontainer image via `pgenv` and stage its artifacts alongside existing PG16–PG17 installs. - Switch the default devcontainer PostgreSQL version to 18.1 so local developer workflows exercise PG18 by default. - Extend `configure` / `configure.ac` version checks to treat PostgreSQL 18 as a supported server version. - Update GitHub Actions `build_and_test.yml` to: - Define a `pg18_version` entry (`major: 18`, `full: 18.1`). - Add PG18 to all relevant test matrices (check-split, check-multi, regress, enterprise, pytest, CDC installcheck, and query-generator). - Include upgrade paths to PG18 (16→18, 17→18) in the upgrade-test matrix. - Run the flakiness job against the PG18 fail-tester image. - Update `packaging-test-pipelines.yml` so the supported PG version list is derived more robustly from `build_and_test.yml` (no assumptions about version suffixes). - Extend `src/test/regress/citus_tests/common.py` with the OLDEST_SUPPORTED_CITUS_VERSION mapping for PG18 to keep upgrade tests working. ### Testing - Devcontainer image builds successfully and `pgenv switch 18.1` works inside the container. - GitHub Actions: - `build_and_test.yml` matrix includes the new PG18 jobs and they complete successfully. - `packaging-test-pipelines.yml` still discovers the expected set of PostgreSQL versions. --------- Co-authored-by: Colm Co-authored-by: naisila Co-authored-by: Naisila Puka <37271756+naisila@users.noreply.github.com> --- .devcontainer/Dockerfile | 15 +++++++- .github/workflows/build_and_test.yml | 36 ++++++++++++++++--- .../workflows/packaging-test-pipelines.yml | 6 +++- configure | 12 ++++--- configure.ac | 2 +- src/test/regress/citus_tests/common.py | 1 + 6 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c2467a330..2bca02c25 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -96,6 +96,18 @@ RUN mkdir .pgenv-staging/ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf +FROM base AS pg18 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.1 +RUN rm .pgenv/src/*.tar* +RUN make -C .pgenv/src/postgresql-*/ clean +RUN make -C .pgenv/src/postgresql-*/src/include install + +# Stage the pgenv artifacts for PG18 +RUN mkdir .pgenv-staging/ +RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ +RUN rm .pgenv-staging/config/default.conf + + FROM base AS uncrustify-builder RUN sudo apt update && sudo apt install -y cmake tree @@ -188,6 +200,7 @@ COPY --link --from=uncrustify-builder /uncrustify/usr/ /usr/ COPY --link --from=pg16 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ COPY --link --from=pg17 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ +COPY --link --from=pg18 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ COPY --link --from=pipenv /home/citus/.local/share/virtualenvs/ /home/citus/.local/share/virtualenvs/ @@ -203,7 +216,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 17.7 +RUN pgenv switch 18.1 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 555d548ca..f8f0b6c9f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -31,11 +31,12 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.33" - sql_snapshot_pg_version: "17.7" - image_suffix: "-vafdc5ac" + sql_snapshot_pg_version: "18.1" + image_suffix: "-v15cd08f" pg16_version: '{ "major": "16", "full": "16.11" }' pg17_version: '{ "major": "17", "full": "17.7" }' - upgrade_pg_versions: "16.11-17.7" + pg18_version: '{ "major": "18", "full": "18.1" }' + upgrade_pg_versions: "16.11-17.7-18.1" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters @@ -111,6 +112,7 @@ jobs: pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} + - ${{ needs.params.outputs.pg18_version }} runs-on: ubuntu-latest container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" @@ -141,6 +143,7 @@ jobs: pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} + - ${{ needs.params.outputs.pg18_version }} make: - check-split - check-multi @@ -167,6 +170,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg18_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress @@ -175,6 +182,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ needs.params.outputs.pg18_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress @@ -183,6 +194,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ needs.params.outputs.pg18_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: installcheck suite: cdc image_name: ${{ needs.params.outputs.test_image_name }} @@ -191,6 +206,10 @@ jobs: suite: cdc image_name: ${{ needs.params.outputs.test_image_name }} pg_version: ${{ needs.params.outputs.pg17_version }} + - make: installcheck + suite: cdc + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg18_version }} - make: check-query-generator pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress @@ -199,6 +218,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ needs.params.outputs.pg18_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} runs-on: ubuntu-latest container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" @@ -247,6 +270,7 @@ jobs: pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} + - ${{ needs.params.outputs.pg18_version }} parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs steps: - uses: actions/checkout@v4 @@ -293,6 +317,10 @@ jobs: include: - old_pg_major: 16 new_pg_major: 17 + - old_pg_major: 17 + new_pg_major: 18 + - old_pg_major: 16 + new_pg_major: 18 env: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} @@ -486,7 +514,7 @@ jobs: name: Test flakyness runs-on: ubuntu-latest container: - image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg17_version).full }}${{ needs.params.outputs.image_suffix }} + image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg18_version).full }}${{ needs.params.outputs.image_suffix }} options: --user root env: runs: 8 diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index db0fd08ef..9e418d77c 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -29,7 +29,7 @@ jobs: # Postgres versions are stored in .github/workflows/build_and_test.yml # file in json strings with major and full keys. # Below command extracts the versions and get the unique values. - pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[0-9.]+"' | sed -E 's/"major": "([0-9]+)", "full": "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',') + pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[^"]+"' | sed -E 's/.*"major": "([0-9]+)".*/\1/' | sort -n | uniq | tr '\n' ',') pg_versions_array="[ ${pg_versions} ]" echo "Supported PG Versions: ${pg_versions_array}" # Below line is needed to set the output variable to be used in the next job @@ -122,6 +122,10 @@ jobs: - ubuntu-jammy-all POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }} + exclude: + # PG18 is not supported on Ubuntu focal + - packaging_docker_image: ubuntu-focal-all + POSTGRES_VERSION: 18 container: image: citus/packaging:${{ matrix.packaging_docker_image }} diff --git a/configure b/configure index 26fd0d62f..d1c7ad75a 100755 --- a/configure +++ b/configure @@ -2549,11 +2549,13 @@ fi # check we're building against a supported version of PostgreSQL citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) -version_num=$(echo "$citusac_pg_config_version"| - $SED -e 's/^PostgreSQL \([0-9]*\)\(\.[0-9]*\)\{0,1\}\(.*\)$/\1\2/') +# Extract major[.minor] from "PostgreSQL X[.Y] ..." +version_num=$(printf '%s\n' "$citusac_pg_config_version" \ + | $SED -nE 's/^PostgreSQL[[:space:]]+([0-9]+(\.[0-9]+)?).*/\1/p') -# if PostgreSQL version starts with two digits, the major version is those digits -version_num=$(echo "$version_num"| $SED -e 's/^\([0-9]\{2\}\)\(.*\)$/\1/') +# If it starts with two digits (10+), keep only the major. +version_num=$(printf '%s\n' "$version_num" \ + | $SED -E 's/^([0-9]{2}).*$/\1/') if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 @@ -2588,7 +2590,7 @@ fi if test "$with_pg_version_check" = no; then { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;} -elif test "$version_num" != '16' -a "$version_num" != '17'; then +elif test "$version_num" != '16' -a "$version_num" != '17' -a "$version_num" != '18'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.ac b/configure.ac index 26db953f4..a77769c84 100644 --- a/configure.ac +++ b/configure.ac @@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check) if test "$with_pg_version_check" = no; then AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)]) -elif test "$version_num" != '16' -a "$version_num" != '17'; then +elif test "$version_num" != '16' -a "$version_num" != '17' -a "$version_num" != '18'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index c3ba14ac8..81eb5e544 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -94,6 +94,7 @@ OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 15: "11.1.5", 16: "12.1.5", 17: "13.0.1", + 18: "14.0devel", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION]