rework ci

(cherry picked from commit a748729998)

.circleci/config.yml

@@ -5,47 +5,32 @@ orbs:
 
 jobs:
 
-  build-11:
+  build:
+    description: Build the citus extension
+    parameters:
+      pg_major:
+        description: postgres major version building citus for
+        type: integer
+      image:
+        description: docker image to use for the build
+        type: string
+        default: citus/extbuilder
+      image_tag:
+        description: tag to use for the docker image
+        type: string
     docker:
-      - image: 'citus/extbuilder:11.9'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     steps:
       - checkout
       - run:
           name: 'Configure, Build, and Install'
-          command: build-ext
+          command: |
+            ./ci/build-citus.sh
       - persist_to_workspace:
           root: .
           paths:
-            - build-11/*
-            - install-11.tar
-
-  build-12:
-    docker:
-      - image: 'citus/extbuilder:12.4'
-    steps:
-      - checkout
-      - run:
-          name: 'Configure, Build, and Install'
-          command: build-ext
-      - persist_to_workspace:
-          root: .
-          paths:
-            - build-12/*
-            - install-12.tar
-
-  build-13:
-    docker:
-      - image: 'citus/extbuilder:13.0'
-    steps:
-      - checkout
-      - run:
-          name: 'Configure, Build, and Install'
-          command: build-ext
-      - persist_to_workspace:
-          root: .
-          paths:
-            - build-13/*
-            - install-13.tar
+            - build-<< parameters.pg_major >>/*
+            - install-<<parameters.pg_major >>.tar
 
   check-style:
     docker:
@@ -91,6 +76,7 @@ jobs:
       - run:
           name: 'Check if all CI scripts are actually run'
           command: ci/check_all_ci_scripts_are_run.sh
+
   check-sql-snapshots:
     docker:
       - image: 'citus/extbuilder:latest'
@@ -99,392 +85,230 @@ jobs:
       - run:
           name: 'Check Snapshots'
           command: ci/check_sql_snapshots.sh
-  test-11_check-multi:
+
+  test-pg-upgrade:
+    description: Runs postgres upgrade tests
+    parameters:
+      old_pg_major:
+        description: 'postgres major version to use before the upgrade'
+        type: integer
+      new_pg_major:
+        description: 'postgres major version to upgrade to'
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/pgupgradetester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
+        default: latest
     docker:
-      - image: 'citus/exttester:11.9'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     working_directory: /home/circleci/project
     steps:
       - checkout
       - attach_workspace:
           at: .
       - run:
-          name: 'Install and Test (check-multi)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,multi'
-
-
-
-  test-11_check-vanilla:
-    docker:
-      - image: 'citus/exttester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
+          name: 'Install Extension'
+          command: |
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory /
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory /
       - run:
-          name: 'Install and Test (check-vanilla)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,vanilla'
-
-  test-11_check-mx:
-    docker:
-      - image: 'citus/exttester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-mx)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,mx'
-
-  test-11_check-worker:
-    docker:
-      - image: 'citus/exttester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-worker)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,worker'
-
-  test-11_check-isolation:
-    docker:
-      - image: 'citus/exttester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-isolation)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_11,isolation'
-  test-11_check-follower-cluster:
-    docker:
-      - image: 'citus/exttester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
+          name: 'Configure'
+          command: |
+            chown -R circleci .
+            gosu circleci ./configure
       - run:
           name: 'Enable core dumps'
-          command: 'ulimit -c unlimited'
+          command: |
+            ulimit -c unlimited
       - run:
-          name: 'Install and Test (follower-cluster)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster'
+          name: 'Install and test postgres upgrade'
+          command: |
+            gosu circleci \
+              make -C src/test/regress \
+                check-pg-upgrade \
+                old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \
+                new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin
           no_output_timeout: 2m
       - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
           command: |
             mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
           when: on_fail
       - codecov/upload:
-          flags: 'test_11,follower-cluster'
-      - store_artifacts:
-          path: '/tmp/core_dumps'
-  test-11_check-failure:
+          flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade'
+
+  test-citus-upgrade:
+    description: Runs citus upgrade tests
+    parameters:
+      pg_major:
+        description: "postgres major version"
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/citusupgradetester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
     docker:
-      - image: 'citus/failtester:11.9'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     working_directory: /home/circleci/project
     steps:
       - checkout
       - attach_workspace:
           at: .
       - run:
-          name: 'Install and Test (check-failure)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure'
-          no_output_timeout: 2m
-
-  test-11-12_check-pg-upgrade:
-    docker:
-      - image: 'citus/pgupgradetester:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and test postgres upgrade'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 11 --new-pg-version 12'
-          no_output_timeout: 2m
-
-  test-12-13_check-pg-upgrade:
-    docker:
-      - image: 'citus/pgupgradetester:latest'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and test postgres upgrade'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 12 --new-pg-version 13'
-          no_output_timeout: 2m
-
-  test-12_check-multi:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-multi)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,multi'
-  test-12_check-vanilla:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-vanilla)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,vanilla'
-
-  test-12_check-mx:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-mx)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,mx'
-
-  test-12_check-isolation:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-           at: .
-      - run:
-          name: 'Install and Test (check-isolation)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,isolation'
-
-  test-12_check-worker:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-           at: .
-      - run:
-          name: 'Install and Test (check-worker)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_12,worker'
-
-  test-12_check-follower-cluster:
-    docker:
-      - image: 'citus/exttester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
+          name: 'Configure'
+          command: |
+            chown -R circleci .
+            gosu circleci ./configure
       - run:
           name: 'Enable core dumps'
-          command: 'ulimit -c unlimited'
-      - run:
-          name: 'Install and Test (follower-cluster)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster'
-          no_output_timeout: 2m
-      - run:
           command: |
-            mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
-          when: on_fail
-      - codecov/upload:
-          flags: 'test_12,follower-cluster'
-      - store_artifacts:
-          path: '/tmp/core_dumps'
-
-  test-12_check-failure:
-    docker:
-      - image: 'citus/failtester:12.4'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-failure)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure'
-          no_output_timeout: 2m
-
-  test-11_check-citus-upgrade:
-    docker:
-      - image: 'citus/citusupgradetester:11.9'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
+            ulimit -c unlimited
       - run:
           name: 'Install and test citus upgrade'
           command: |
-           chown -R circleci:circleci /home/circleci
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.0.0.tar
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.1.0.tar
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.2.0.tar
-           install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.3.0.tar
+            # run make check-citus-upgrade for all citus versions
+            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+            for citus_version in ${CITUS_VERSIONS}; do \
+              gosu circleci \
+                make -C src/test/regress \
+                  check-citus-upgrade \
+                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
+                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
+            done;
 
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.0.0.tar
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.1.0.tar
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.2.0.tar
-           install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.3.0.tar
-          no_output_timeout: 2m
-
-  test-13_check-multi:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-multi)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_13,multi'
-
-  test-13_check-mx:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-mx)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_13,mx'
-
-  test-13_check-vanilla:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install and Test (check-vanilla)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_13,vanilla'
-
-  test-13_check-worker:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-           at: .
-      - run:
-          name: 'Install and Test (check-worker)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_13,worker'
-
-  test-13_check-isolation:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-           at: .
-      - run:
-          name: 'Install and Test (check-isolation)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation'
-          no_output_timeout: 2m
-      - codecov/upload:
-          flags: 'test_13,isolation'
-
-  test-13_check-follower-cluster:
-    docker:
-      - image: 'citus/exttester:13.0'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Enable core dumps'
-          command: 'ulimit -c unlimited'
-      - run:
-          name: 'Install and Test (follower-cluster)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster'
+            # run make check-citus-upgrade-mixed for all citus versions
+            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+            for citus_version in ${CITUS_VERSIONS}; do \
+              gosu circleci \
+                make -C src/test/regress \
+                  check-citus-upgrade-mixed \
+                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
+                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
+            done;
           no_output_timeout: 2m
       - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
           command: |
             mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
           when: on_fail
       - codecov/upload:
-          flags: 'test_13,follower-cluster'
-      - store_artifacts:
-          path: '/tmp/core_dumps'
+          flags: 'test_<< parameters.pg_major >>,upgrade'
 
-  test-13_check-failure:
+  test-citus:
+    description: Runs the common tests of citus
+    parameters:
+      pg_major:
+        description: "postgres major version"
+        type: integer
+      image:
+        description: 'docker image to use as for the tests'
+        type: string
+        default: citus/exttester
+      image_tag:
+        description: 'docker image tag to use'
+        type: string
+      make:
+        description: "make target"
+        type: string
     docker:
-      - image: 'citus/failtester:13.0'
+      - image: '<< parameters.image >>:<< parameters.image_tag >>'
     working_directory: /home/circleci/project
     steps:
       - checkout
       - attach_workspace:
           at: .
       - run:
-          name: 'Install and Test (check-failure)'
-          command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure'
+          name: 'Install Extension'
+          command: |
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
+      - run:
+          name: 'Configure'
+          command: |
+            chown -R circleci .
+            gosu circleci ./configure
+      - run:
+          name: 'Enable core dumps'
+          command: |
+            ulimit -c unlimited
+      - run:
+          name: 'Run Test'
+          command: |
+            gosu circleci make -C src/test/regress << parameters.make >>
           no_output_timeout: 2m
+      - run:
+          name: 'Regressions'
+          command: |
+            if [ -f "src/test/regress/regression.diffs" ]; then
+              cat src/test/regress/regression.diffs
+              exit 1
+            fi
+          when: on_fail
+      - run:
+          name: 'Copy coredumps'
+          command: |
+            mkdir -p /tmp/core_dumps
+            if ls core.* 1> /dev/null 2>&1; then
+                cp core.* /tmp/core_dumps
+            fi
+          when: on_fail
+      - store_artifacts:
+          name: 'Save regressions'
+          path: src/test/regress/regression.diffs
+          when: on_fail
+      - store_artifacts:
+          name: 'Save core dumps'
+          path: /tmp/core_dumps
+          when: on_fail
+      - codecov/upload:
+          flags: 'test_<< parameters.pg_major >>,<< parameters.make >>'
+          when: always
 
   check-merge-to-enterprise:
     docker:
@@ -495,6 +319,7 @@ jobs:
       - run:
           command: |
             ci/check_enterprise_merge.sh
+
   ch_benchmark:
     docker:
       - image: buildpack-deps:stretch
@@ -509,6 +334,7 @@ jobs:
             sh run_hammerdb.sh citusbot_ch_benchmark_rg
           name: install dependencies and run ch_benchmark tests
           no_output_timeout: 20m
+
   tpcc_benchmark:
     docker:
       - image: buildpack-deps:stretch
@@ -524,7 +350,6 @@ jobs:
           name: install dependencies and run ch_benchmark tests
           no_output_timeout: 20m
 
-
 workflows:
   version: 2
   build_and_test:
@@ -536,70 +361,173 @@ workflows:
               ignore:
                 - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.*
 
-
-      - build-11
-      - build-12
-      - build-13
+      - build:
+          name: build-11
+          pg_major: 11
+          image_tag: '11.9'
+      - build:
+          name: build-12
+          pg_major: 12
+          image_tag: '12.4'
+      - build:
+          name: build-13
+          pg_major: 13
+          image_tag: '13.0'
 
       - check-style
       - check-sql-snapshots
 
-      - test-11_check-multi:
+      - test-citus:
+          name: 'test-11_check-multi'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-multi
           requires: [build-11]
-      - test-11_check-vanilla:
+      - test-citus:
+          name: 'test-11_check-mx'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-multi-mx
           requires: [build-11]
-      - test-11_check-isolation:
+      - test-citus:
+          name: 'test-11_check-vanilla'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-vanilla
           requires: [build-11]
-      - test-11_check-mx:
+      - test-citus:
+          name: 'test-11_check-isolation'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-isolation
           requires: [build-11]
-      - test-11_check-worker:
+      - test-citus:
+          name: 'test-11_check-worker'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-worker
           requires: [build-11]
-      - test-11_check-follower-cluster:
+      - test-citus:
+          name: 'test-11_check-follower-cluster'
+          pg_major: 11
+          image_tag: '11.9'
+          make: check-follower-cluster
           requires: [build-11]
-      - test-11_check-failure:
+      - test-citus:
+          name: 'test-11_check-failure'
+          pg_major: 11
+          image: citus/failtester
+          image_tag: '11.9'
+          make: check-failure
           requires: [build-11]
 
-      - test-12_check-multi:
+      - test-citus:
+          name: 'test-12_check-multi'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-multi
           requires: [build-12]
-      - test-12_check-vanilla:
+      - test-citus:
+          name: 'test-12_check-mx'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-multi-mx
           requires: [build-12]
-      - test-12_check-isolation:
+      - test-citus:
+          name: 'test-12_check-vanilla'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-vanilla
           requires: [build-12]
-      - test-12_check-mx:
+      - test-citus:
+          name: 'test-12_check-isolation'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-isolation
           requires: [build-12]
-      - test-12_check-worker:
+      - test-citus:
+          name: 'test-12_check-worker'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-worker
           requires: [build-12]
-      - test-12_check-follower-cluster:
+      - test-citus:
+          name: 'test-12_check-follower-cluster'
+          pg_major: 12
+          image_tag: '12.4'
+          make: check-follower-cluster
           requires: [build-12]
-      - test-12_check-failure:
+      - test-citus:
+          name: 'test-12_check-failure'
+          pg_major: 12
+          image: citus/failtester
+          image_tag: '12.4'
+          make: check-failure
           requires: [build-12]
 
-      - test-13_check-multi:
+      - test-citus:
+          name: 'test-13_check-multi'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-multi
           requires: [build-13]
-      - test-13_check-vanilla:
+      - test-citus:
+          name: 'test-13_check-mx'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-multi-mx
           requires: [build-13]
-      - test-13_check-isolation:
+      - test-citus:
+          name: 'test-13_check-vanilla'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-vanilla
           requires: [build-13]
-      - test-13_check-mx:
+      - test-citus:
+          name: 'test-13_check-isolation'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-isolation
           requires: [build-13]
-      - test-13_check-worker:
+      - test-citus:
+          name: 'test-13_check-worker'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-worker
           requires: [build-13]
-      - test-13_check-follower-cluster:
-          requires: [build-13]
-      - test-13_check-failure:
+      - test-citus:
+          name: 'test-13_check-follower-cluster'
+          pg_major: 13
+          image_tag: '13.0'
+          make: check-follower-cluster
           requires: [build-13]
 
-      - test-11-12_check-pg-upgrade:
-          requires:
-            - build-11
-            - build-12
+      - test-citus:
+          name: 'test-13_check-failure'
+          pg_major: 13
+          image: citus/failtester
+          image_tag: '13.0'
+          make: check-failure
+          requires: [build-13]
 
-      - test-12-13_check-pg-upgrade:
-          requires:
-            - build-12
-            - build-13
+      - test-pg-upgrade:
+          name: 'test-11-12_check-pg-upgrade'
+          old_pg_major: 11
+          new_pg_major: 12
+          image_tag: latest
+          requires: [build-11,build-12]
 
-      - test-11_check-citus-upgrade:
+      - test-pg-upgrade:
+          name: 'test-12-13_check-pg-upgrade'
+          old_pg_major: 12
+          new_pg_major: 13
+          image_tag: latest
+          requires: [build-12,build-13]
+
+      - test-citus-upgrade:
+          name: test-11_check-citus-upgrade
+          pg_major: 11
+          image_tag: '11.9'
           requires: [build-11]
 
       - ch_benchmark:

ci/README.md

@@ -46,6 +46,16 @@ following:
    requires also adding a comment before explaining why this specific use of the
    function is safe.
 
+## `build-citus.sh`
+
+This is the script used during the build phase of the extension. Historically this script
+was embedded in the docker images. This made maintenance a hassle. Now it lives in tree
+with the rest of the source code.
+
+When this script fails you most likely have a build error on the postgres version it was
+building at the time of the failure. Fix the compile error and push a new version of your
+code to fix.
+
 ## `check_enterprise_merge.sh`
 
 This check exists to make sure that we can always merge the `master` branch of

ci/build-citus.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# make bash behave
+set -euo pipefail
+IFS=$'\n\t'
+
+# shellcheck disable=SC1091
+source ci/ci_helpers.sh
+
+# read pg major version, error if not provided
+PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
+
+# get codename from release file
+. /etc/os-release
+codename=${VERSION#*(}
+codename=${codename%)*}
+
+# get project from argument
+project="${CIRCLE_PROJECT_REPONAME}"
+
+# we'll do everything with absolute paths
+basedir="$(pwd)"
+
+# get the project and clear out the git repo (reduce workspace size
+rm -rf "${basedir}/.git"
+
+build_ext() {
+  pg_major="$1"
+
+  builddir="${basedir}/build-${pg_major}"
+  echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
+
+  # do everything in a subdirectory to avoid clutter in current directory
+  mkdir -p "${builddir}" && cd "${builddir}"
+
+  CFLAGS=-Werror "${basedir}/configure" PG_CONFIG="/usr/lib/postgresql/${pg_major}/bin/pg_config" --enable-coverage
+
+  installdir="${builddir}/install"
+  make -j$(nproc) && mkdir -p "${installdir}" && { make DESTDIR="${installdir}" install-all || make DESTDIR="${installdir}" install ; }
+
+  cd "${installdir}" && find . -type f -print > "${builddir}/files.lst"
+  tar cvf "${basedir}/install-${pg_major}.tar" `cat ${builddir}/files.lst`
+
+  cd "${builddir}" && rm -rf install files.lst && make clean
+}
+
+build_ext "${PG_MAJOR}"

src/test/regress/expected/follower_single_node.out

@@ -233,7 +233,7 @@ SELECT * FROM test WHERE x = 1;
 ERROR:  node group 0 does not have a secondary node
 -- add the the follower as secondary nodes and try again, the SELECT statement
 -- should work this time
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 SET search_path TO single_node;
 SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary');
  ?column?
@@ -350,7 +350,7 @@ SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
 
 RESET citus.task_assignment_policy;
 -- Cleanup
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 SET search_path TO single_node;
 SET client_min_messages TO WARNING;
 DROP SCHEMA single_node CASCADE;

src/test/regress/expected/multi_follower_dml.out

@@ -269,7 +269,7 @@ ERROR:  writing to worker nodes is not currently allowed
 DETAIL:  citus.use_secondary_nodes is set to 'always'
 SELECT * FROM citus_local_table ORDER BY a;
 ERROR:  there is a shard placement in node group 0 but there are no nodes in that group
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 DROP TABLE the_table;
 DROP TABLE reference_table;
 DROP TABLE citus_local_table;

src/test/regress/expected/multi_follower_select_statements.out

@@ -77,7 +77,7 @@ order by s_i_id;
 SELECT * FROM the_table;
 ERROR:  node group does not have a secondary node
 -- add the secondary nodes and try again, the SELECT statement should work this time
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 SELECT 1 FROM master_add_node('localhost', :follower_worker_1_port,
   groupid => (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_1_port),
   noderole => 'secondary');
@@ -149,7 +149,7 @@ order by s_i_id;
 ERROR:  there is a shard placement in node group but there are no nodes in that group
 -- now move the secondary nodes into the new cluster and see that the follower, finally
 -- correctly configured, can run select queries involving them
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 UPDATE pg_dist_node SET nodecluster = 'second-cluster' WHERE noderole = 'secondary';
 \c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always\ -c\ citus.cluster_name=second-cluster'"
 SELECT * FROM the_table;
@@ -160,6 +160,6 @@ SELECT * FROM the_table;
 (2 rows)
 
 -- clean up after ourselves
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 DROP TABLE the_table;
 DROP TABLE stock;

src/test/regress/multi_schedule

@@ -96,7 +96,9 @@ test: multi_explain hyperscale_tutorial partitioned_intermediate_results distrib
 test: multi_basic_queries cross_join multi_complex_expressions multi_subquery multi_subquery_complex_queries multi_subquery_behavioral_analytics
 test: multi_subquery_complex_reference_clause multi_subquery_window_functions multi_view multi_sql_function multi_prepare_sql
 test: sql_procedure multi_function_in_join row_types materialized_view undistribute_table
-test: multi_subquery_in_where_reference_clause join geqo adaptive_executor propagate_set_commands
+test: multi_subquery_in_where_reference_clause adaptive_executor propagate_set_commands geqo
+# this should be run alone as it gets too many clients
+test: join_pushdown
 test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc statement_cancel_error_message
 test: multi_agg_distinct multi_agg_approximate_distinct multi_limit_clause_approximate multi_outer_join_reference multi_single_relation_subquery multi_prepare_plsql set_role_in_transaction
 test: multi_reference_table multi_select_for_update relation_access_tracking pg13_with_ties

src/test/regress/sql/follower_single_node.sql

@@ -100,7 +100,7 @@ SELECT * FROM test WHERE x = 1;
 
 -- add the the follower as secondary nodes and try again, the SELECT statement
 -- should work this time
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 SET search_path TO single_node;
 
 SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary');
@@ -138,7 +138,7 @@ SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y);
 RESET citus.task_assignment_policy;
 
 -- Cleanup
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 SET search_path TO single_node;
 SET client_min_messages TO WARNING;
 DROP SCHEMA single_node CASCADE;

src/test/regress/sql/multi_follower_dml.sql

@@ -151,7 +151,7 @@ SELECT * FROM reference_table ORDER BY a;
 INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 3);
 SELECT * FROM citus_local_table ORDER BY a;
 
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 DROP TABLE the_table;
 DROP TABLE reference_table;
 DROP TABLE citus_local_table;

src/test/regress/sql/multi_follower_select_statements.sql

@@ -54,7 +54,7 @@ SELECT * FROM the_table;
 
 -- add the secondary nodes and try again, the SELECT statement should work this time
 
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 
 SELECT 1 FROM master_add_node('localhost', :follower_worker_1_port,
   groupid => (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_1_port),
@@ -106,12 +106,12 @@ order by s_i_id;
 -- now move the secondary nodes into the new cluster and see that the follower, finally
 -- correctly configured, can run select queries involving them
 
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 UPDATE pg_dist_node SET nodecluster = 'second-cluster' WHERE noderole = 'secondary';
 \c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always\ -c\ citus.cluster_name=second-cluster'"
 SELECT * FROM the_table;
 
 -- clean up after ourselves
-\c - - - :master_port
+\c -reuse-previous=off regression - - :master_port
 DROP TABLE the_table;
 DROP TABLE stock;