Fix CI on release branch 9.4 (#5942)

.circleci/config.yml

@@ -1,20 +1,37 @@
 version: 2.1
 orbs:
-  codecov: codecov/codecov@1.1.0
+  codecov: codecov/codecov@1.1.1
   azure-cli: circleci/azure-cli@1.0.0
 
 jobs:
+
   build:
+    description: Build the citus extension
+    parameters:
+      pg_major:
+        description: postgres major version building citus for
+        type: integer
+      image:
+        description: docker image to use for the build
+        type: string
+        default: citus/extbuilder
+      image_tag:
+        description: tag to use for the docker image
+        type: string
     docker:
-      - image: 'citus/extbuilder:latest'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     steps:
       - checkout
       - run:
           name: 'Configure, Build, and Install'
-          command: build-ext
+          command: |
+            ./ci/build-citus.sh
       - persist_to_workspace:
           root: .
-          paths: [.]
+          paths:
+            - build-<< parameters.pg_major >>/*
+            - install-<<parameters.pg_major >>.tar
+
   check-style:
     docker:
       - image: 'citus/stylechecker:latest'
@@ -53,6 +70,7 @@ jobs:
       - run:
           name: 'Check for banned C API usage'
           command: ci/banned.h.sh
+
   check-sql-snapshots:
     docker:
       - image: 'citus/extbuilder:latest'
@@ -60,199 +78,242 @@ jobs:
       - checkout
       - run:
           name: 'Check Snapshots'
-          command: "./configure && make check-sql-snapshots -C src/backend/distributed"
+          command: ci/check_sql_snapshots.sh
-  test-11_check-multi:
-    docker:
-      - image: 'citus/exttester-11:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-multi)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,multi'
 
-  test-11_check-tt-van-mx:
+  test-pg-upgrade:
+    description: Runs postgres upgrade tests
+    parameters:
+      old_pg_major:
+        description: 'postgres major version to use before the upgrade'
+        type: integer
+      new_pg_major:
+        description: 'postgres major version to upgrade to'
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/pgupgradetester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
+        default: latest
     docker:
-      - image: 'citus/exttester-11:latest'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     working_directory: /home/circleci/project
     steps:
+      - checkout
       - attach_workspace:
           at: .
       - run:
-          name: 'Install and Test (check-tt-van-mx)'
+          name: 'Install Extension'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'
+          command: |
-          no_output_timeout: 2m
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory /
-      - codecov/upload:
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory /
-          flags: 'test_11,tracker,vanilla,mx'
-  test-11_check-iso-work-fol:
-    docker:
-      - image: 'citus/exttester-11:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
       - run:
-          name: 'Install and Test (check-iso-work-fol)'
+          name: 'Configure'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation check-worker'
+          command: |
-          no_output_timeout: 2m
+            chown -R circleci .
-      - codecov/upload:
+            PG_CONFIG=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin/pg_config gosu circleci ./configure
-          flags: 'test_11,isolation,worker'
-  test-11_check-fol:
-    docker:
-      - image: 'citus/exttester-11:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
       - run:
           name: 'Enable core dumps'
-          command: 'ulimit -c unlimited'
-      - run:
-          name: 'Install and Test (fol)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster'
-          no_output_timeout: 2m
-      - run:
           command: |
-            mkdir -p /tmp/core_dumps
+            ulimit -c unlimited
-            cp core.* /tmp/core_dumps
-          when: on_fail
-      - codecov/upload:
-          flags: 'test_11,follower'
-      - store_artifacts:
-          path: '/tmp/core_dumps'
-  test-11_check-failure:
-    docker:
-      - image: 'citus/failtester-11:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-failure)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure'
-          no_output_timeout: 2m
-
-  test-11-12_check-pg-upgrade:
-    docker:
-      - image: 'citus/pgupgradetester:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
       - run:
           name: 'Install and test postgres upgrade'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 11 --new-pg-version 12'
+          command: |
-          no_output_timeout: 2m
+            gosu circleci \
-
+              make -C src/test/regress \
-  test-12_check-multi:
+                check-pg-upgrade \
-    docker:
+                old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \
-      - image: 'citus/exttester-12:latest'
+                new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-multi)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,multi'
-  test-12_check-tt-van-mx:
-    docker:
-      - image: 'citus/exttester-12:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-tt-van-mx)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,tracker,vanilla,mx'
-  test-12_check-iso-work-fol:
-    docker:
-      - image: 'citus/exttester-12:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-           at: .
-      - run:
-          name: 'Install and Test (check-iso-work-fol)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation check-worker'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,isolation,worker'
-  test-12_check-fol:
-    docker:
-      - image: 'citus/exttester-12:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Enable core dumps'
-          command: 'ulimit -c unlimited'
-      - run:
-          name: 'Install and Test (fol)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster'
           no_output_timeout: 2m
       - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
           command: |
             mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
           when: on_fail
       - codecov/upload:
-          flags: 'test_12,follower'
+          flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade'
-      - store_artifacts:
-          path: '/tmp/core_dumps'
 
-  test-12_check-failure:
+  test-citus-upgrade:
+    description: Runs citus upgrade tests
+    parameters:
+      pg_major:
+        description: "postgres major version"
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/citusupgradetester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
     docker:
-      - image: 'citus/failtester-12:latest'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     working_directory: /home/circleci/project
     steps:
+      - checkout
       - attach_workspace:
           at: .
       - run:
-          name: 'Install and Test (check-failure)'
+          name: 'Configure'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure'
+          command: |
-          no_output_timeout: 2m
+            chown -R circleci .
-
+            gosu circleci ./configure
-  test-11_check-citus-upgrade:
+      - run:
-    docker:
+          name: 'Enable core dumps'
-      - image: 'citus/citusupgradetester-11:latest'
+          command: |
-    working_directory: /home/circleci/project
+            ulimit -c unlimited
-    steps:
-      - {attach_workspace: {at: .}}
       - run:
           name: 'Install and test citus upgrade'
           command: |
-           chown -R circleci:circleci /home/circleci
+            # run make check-citus-upgrade for all citus versions
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.0.0.tar
+            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.1.0.tar
+            for citus_version in ${CITUS_VERSIONS}; do \
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.2.0.tar
+              gosu circleci \
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.3.0.tar
+                make -C src/test/regress \
+                  check-citus-upgrade \
+                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
+                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
+            done;
 
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.0.0.tar
+            # run make check-citus-upgrade-mixed for all citus versions
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.1.0.tar
+            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.2.0.tar
+            for citus_version in ${CITUS_VERSIONS}; do \
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.3.0.tar
+              gosu circleci \
+                make -C src/test/regress \
+                  check-citus-upgrade-mixed \
+                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
+                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
+            done;
           no_output_timeout: 2m
+      - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
+          command: |
+            mkdir -p /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
+          when: on_fail
+      - codecov/upload:
+          flags: 'test_<< parameters.pg_major >>,upgrade'
+
+  test-citus:
+    description: Runs the common tests of citus
+    parameters:
+      pg_major:
+        description: "postgres major version"
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/exttester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
+      make:
+        description: "make target"
+        type: string
+    docker:
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
+    working_directory: /home/circleci/project
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: 'Install Extension'
+          command: |
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
+      - run:
+          name: 'Configure'
+          command: |
+            chown -R circleci .
+            gosu circleci ./configure
+      - run:
+          name: 'Enable core dumps'
+          command: |
+            ulimit -c unlimited
+      - run:
+          name: 'Run Test'
+          command: |
+            gosu circleci make -C src/test/regress << parameters.make >>
+          no_output_timeout: 2m
+      - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
+          command: |
+            mkdir -p /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
+          when: on_fail
+      - codecov/upload:
+          flags: 'test_<< parameters.pg_major >>,<< parameters.make >>'
+          when: always
 
   check-merge-to-enterprise:
     docker:
-      - image: buildpack-deps:stretch
+      - image: citus/extbuilder:13.0
     working_directory: /home/circleci/project
     steps:
       - checkout
       - run:
           command: |
-            ./src/test/scripts/check_enterprise_merge.sh
+            ci/check_enterprise_merge.sh
+
   ch_benchmark:
     docker:
       - image: buildpack-deps:stretch
@@ -267,6 +328,7 @@ jobs:
             sh run_hammerdb.sh citusbot_ch_benchmark_rg
           name: install dependencies and run ch_benchmark tests
           no_output_timeout: 20m
+
   tpcc_benchmark:
     docker:
       - image: buildpack-deps:stretch
@@ -282,57 +344,150 @@ jobs:
           name: install dependencies and run ch_benchmark tests
           no_output_timeout: 20m
 
-
 workflows:
   version: 2
   build_and_test:
     jobs:
+
       - check-merge-to-enterprise:
           filters:
             branches:
               ignore:
                 - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.*
 
-      - build
+      - build:
+          name: build-11
+          pg_major: 11
+          image_tag: '11.9'
+      - build:
+          name: build-12
+          pg_major: 12
+          image_tag: '12.4'
+
       - check-style
       - check-sql-snapshots
 
-      - test-11_check-multi:
+      - test-citus:
-          requires: [build]
+          name: 'test-11_check-multi'
-      - test-11_check-tt-van-mx:
+          pg_major: 11
-          requires: [build]
+          image_tag: '11.9'
-      - test-11_check-iso-work-fol:
+          make: check-multi
-          requires: [build]
+          requires: [build-11]
-      - test-11_check-fol:
+      - test-citus:
-          requires: [build]
+          name: 'test-11_check-task-tracker'
-      - test-11_check-failure:
+          pg_major: 11
-          requires: [build]
+          image_tag: '11.9'
+          make: check-multi-task-tracker-extra
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-mx'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-multi-mx
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-vanilla'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-vanilla
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-isolation'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-isolation
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-worker'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-worker
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-follower-cluster'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-follower-cluster
+          requires: [build-11]
+      - test-citus:
+          name: 'test-11_check-failure'
+          pg_major: 11
+          image: citus/failtester
+          image_tag: '11.9'
+          make: check-failure
+          requires: [build-11]
 
-      - test-12_check-multi:
+      - test-citus:
-          requires: [build]
+          name: 'test-12_check-multi'
-      - test-12_check-tt-van-mx:
+          pg_major: 12
-          requires: [build]
+          image_tag: '12.4'
-      - test-12_check-iso-work-fol:
+          make: check-multi
-          requires: [build]
+          requires: [build-12]
-      - test-12_check-fol:
+      - test-citus:
-          requires: [build]
+          name: 'test-12_check-task-tracker'
-      - test-12_check-failure:
+          pg_major: 12
-          requires: [build]
+          image_tag: '12.4'
+          make: check-multi-task-tracker-extra
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-mx'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-multi-mx
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-vanilla'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-vanilla
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-isolation'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-isolation
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-worker'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-worker
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-follower-cluster'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-follower-cluster
+          requires: [build-12]
+      - test-citus:
+          name: 'test-12_check-failure'
+          pg_major: 12
+          image: citus/failtester
+          image_tag: '12.4'
+          make: check-failure
+          requires: [build-12]
 
-      - test-11-12_check-pg-upgrade:
+      - test-pg-upgrade:
-          requires: [build]
+          name: 'test-11-12_check-pg-upgrade'
+          old_pg_major: 11
+          new_pg_major: 12
+          image_tag: latest
+          requires: [build-11,build-12]
 
-      - test-11_check-citus-upgrade:
+      - test-citus-upgrade:
-          requires: [build]
+          name: test-11_check-citus-upgrade
+          pg_major: 11
+          image_tag: '11.9'
+          requires: [build-11]
 
       - ch_benchmark:
-          requires: [build]
+          requires: [build-12]
           filters:
             branches:
               only:
                 - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix
       - tpcc_benchmark:
-          requires: [build]
+          requires: [build-12]
           filters:
             branches:
               only:

ci/build-citus.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# make bash behave
+set -euo pipefail
+IFS=$'\n\t'
+
+# shellcheck disable=SC1091
+source ci/ci_helpers.sh
+
+# read pg major version, error if not provided
+PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
+
+# get codename from release file
+. /etc/os-release
+codename=${VERSION#*(}
+codename=${codename%)*}
+
+# get project from argument
+project="${CIRCLE_PROJECT_REPONAME}"
+
+# we'll do everything with absolute paths
+basedir="$(pwd)"
+
+# get the project and clear out the git repo (reduce workspace size
+rm -rf "${basedir}/.git"
+
+build_ext() {
+  pg_major="$1"
+
+  builddir="${basedir}/build-${pg_major}"
+  echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
+
+  # do everything in a subdirectory to avoid clutter in current directory
+  mkdir -p "${builddir}" && cd "${builddir}"
+
+  CFLAGS=-Werror "${basedir}/configure" PG_CONFIG="/usr/lib/postgresql/${pg_major}/bin/pg_config" --enable-coverage
+
+  installdir="${builddir}/install"
+  make -j$(nproc) && mkdir -p "${installdir}" && { make DESTDIR="${installdir}" install-all || make DESTDIR="${installdir}" install ; }
+
+  cd "${installdir}" && find . -type f -print > "${builddir}/files.lst"
+  tar cvf "${basedir}/install-${pg_major}.tar" `cat ${builddir}/files.lst`
+
+  cd "${builddir}" && rm -rf install files.lst && make clean
+}
+
+build_ext "${PG_MAJOR}"

ci/check_sql_snapshots.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source ci/ci_helpers.sh
+
+for udf_dir in src/backend/distributed/sql/udfs/*; do
+    # We want to find the last snapshotted sql file, to make sure it's the same
+    # as "latest.sql". This is done by:
+    # 1. Getting the filenames in the UDF directory (using find instead of ls, to keep shellcheck happy)
+    # 2. Filter out latest.sql
+    # 3. Sort using "version sort"
+    # 4. Get the last one using tail
+    latest_snapshot=$(\
+        find "$udf_dir" -iname "*.sql" -exec basename {} \; \
+        | { grep --invert-match latest.sql || true; } \
+        | sort --version-sort \
+        | tail --lines 1);
+    diff --unified --color=auto "$udf_dir/latest.sql" "$udf_dir/$latest_snapshot"; \
+done

ci/ci_helpers.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# For echo commands "set -x" would show the message effectively twice. Once as
+# part of the echo command shown by "set -x" and once because of the output of
+# the echo command. We do not want "set -x" to show the echo command. We only
+# want to see the actual message in the output of echo itself. This function is
+# a trick to do so. Read the StackOverflow post below to understand why this
+# works and what this works around.
+# Source: https://superuser.com/a/1141026/242593
+shopt -s expand_aliases
+alias echo='{ save_flags="$-"; set +x;} 2> /dev/null && echo_and_restore'
+echo_and_restore() {
+        builtin echo "$*"
+        #shellcheck disable=SC2154
+        case "$save_flags" in
+         (*x*)  set -x
+        esac
+}
+
+# Make sure that on a failing exit we show a useful message
+hint_on_fail() {
+    exit_code=$?
+    # Get filename of the currently running script
+    # Source: https://stackoverflow.com/a/192337/2570866
+    filename=$(basename "$0")
+    if [ $exit_code != 0 ]; then
+        echo "HINT: To solve this failure look here: https://github.com/citusdata/citus/blob/master/ci/README.md#$filename"
+    fi
+    exit $exit_code
+}
+trap hint_on_fail EXIT