Backport GHA Migration to release-9.5 (#7282)

Co-authored-by: Jelte Fennema-Nio <jelte.fennema@microsoft.com>

.circleci/config.yml

@@ -1,544 +0,0 @@
-version: 2.1
-orbs:
-  codecov: codecov/codecov@1.1.1
-  azure-cli: circleci/azure-cli@1.0.0
-
-jobs:
-
-  build:
-    description: Build the citus extension
-    parameters:
-      pg_major:
-        description: postgres major version building citus for
-        type: integer
-      image:
-        description: docker image to use for the build
-        type: string
-        default: citus/extbuilder
-      image_tag:
-        description: tag to use for the docker image
-        type: string
-    docker:
-      - image: '<< parameters.image >>:<< parameters.image_tag >>'
-    steps:
-      - checkout
-      - run:
-          name: 'Configure, Build, and Install'
-          command: |
-            ./ci/build-citus.sh
-      - persist_to_workspace:
-          root: .
-          paths:
-            - build-<< parameters.pg_major >>/*
-            - install-<<parameters.pg_major >>.tar
-
-  check-style:
-    docker:
-      - image: 'citus/stylechecker:latest'
-    steps:
-      - checkout
-      - run:
-          name: 'Check Style'
-          command: citus_indent --check
-      - run:
-          name: 'Fix whitespace'
-          command: ci/editorconfig.sh
-      - run:
-          name: 'Check if whitespace fixing changed anything, install editorconfig if it did'
-          command: git diff --exit-code
-      - run:
-          name: 'Remove useless declarations'
-          command: ci/remove_useless_declarations.sh
-      - run:
-          name: 'Check if changed'
-          command: git diff --cached --exit-code
-      - run:
-          name: 'Normalize test output'
-          command: ci/normalize_expected.sh
-      - run:
-          name: 'Check if changed'
-          command: git diff --exit-code
-      - run:
-          name: 'Check for C-style comments in migration files'
-          command: ci/disallow_c_comments_in_migrations.sh
-      - run:
-          name: 'Check if changed'
-          command: git diff --exit-code
-      - run:
-          name: 'Check for lengths of changelog entries'
-          command: ci/disallow_long_changelog_entries.sh
-      - run:
-          name: 'Check for banned C API usage'
-          command: ci/banned.h.sh
-      - run:
-          name: 'Check for tests missing in schedules'
-          command: ci/check_all_tests_are_run.sh
-      - run:
-          name: 'Check if all CI scripts are actually run'
-          command: ci/check_all_ci_scripts_are_run.sh
-
-  check-sql-snapshots:
-    docker:
-      - image: 'citus/extbuilder:latest'
-    steps:
-      - checkout
-      - run:
-          name: 'Check Snapshots'
-          command: ci/check_sql_snapshots.sh
-
-  test-pg-upgrade:
-    description: Runs postgres upgrade tests
-    parameters:
-      old_pg_major:
-        description: 'postgres major version to use before the upgrade'
-        type: integer
-      new_pg_major:
-        description: 'postgres major version to upgrade to'
-        type: integer
-      image:
-        description: 'docker image to use as for the tests'
-        type: string
-        default: citus/pgupgradetester
-      image_tag:
-        description: 'docker image tag to use'
-        type: string
-        default: latest
-    docker:
-      - image: '<< parameters.image >>:<< parameters.image_tag >>'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install Extension'
-          command: |
-            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory /
-            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory /
-      - run:
-          name: 'Configure'
-          command: |
-            chown -R circleci .
-            gosu circleci ./configure
-      - run:
-          name: 'Enable core dumps'
-          command: |
-            ulimit -c unlimited
-      - run:
-          name: 'Install and test postgres upgrade'
-          command: |
-            gosu circleci \
-              make -C src/test/regress \
-                check-pg-upgrade \
-                old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \
-                new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin
-          no_output_timeout: 2m
-      - run:
-          name: 'Regressions'
-          command: |
-            if [ -f "src/test/regress/regression.diffs" ]; then
-              cat src/test/regress/regression.diffs
-              exit 1
-            fi
-          when: on_fail
-      - run:
-          name: 'Copy coredumps'
-          command: |
-            mkdir -p /tmp/core_dumps
-            if ls core.* 1> /dev/null 2>&1; then
-                cp core.* /tmp/core_dumps
-            fi
-          when: on_fail
-      - store_artifacts:
-          name: 'Save regressions'
-          path: src/test/regress/regression.diffs
-          when: on_fail
-      - store_artifacts:
-          name: 'Save core dumps'
-          path: /tmp/core_dumps
-          when: on_fail
-      - codecov/upload:
-          flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade'
-
-  test-citus-upgrade:
-    description: Runs citus upgrade tests
-    parameters:
-      pg_major:
-        description: "postgres major version"
-        type: integer
-      image:
-        description: 'docker image to use as for the tests'
-        type: string
-        default: citus/citusupgradetester
-      image_tag:
-        description: 'docker image tag to use'
-        type: string
-    docker:
-      - image: '<< parameters.image >>:<< parameters.image_tag >>'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Configure'
-          command: |
-            chown -R circleci .
-            gosu circleci ./configure
-      - run:
-          name: 'Enable core dumps'
-          command: |
-            ulimit -c unlimited
-      - run:
-          name: 'Install and test citus upgrade'
-          command: |
-            # run make check-citus-upgrade for all citus versions
-            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
-            for citus_version in ${CITUS_VERSIONS}; do \
-              gosu circleci \
-                make -C src/test/regress \
-                  check-citus-upgrade \
-                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
-                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
-                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
-            done;
-
-            # run make check-citus-upgrade-mixed for all citus versions
-            # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
-            for citus_version in ${CITUS_VERSIONS}; do \
-              gosu circleci \
-                make -C src/test/regress \
-                  check-citus-upgrade-mixed \
-                  bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
-                  citus-pre-tar=/install-pg11-citus${citus_version}.tar \
-                  citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
-            done;
-          no_output_timeout: 2m
-      - run:
-          name: 'Regressions'
-          command: |
-            if [ -f "src/test/regress/regression.diffs" ]; then
-              cat src/test/regress/regression.diffs
-              exit 1
-            fi
-          when: on_fail
-      - run:
-          name: 'Copy coredumps'
-          command: |
-            mkdir -p /tmp/core_dumps
-            if ls core.* 1> /dev/null 2>&1; then
-                cp core.* /tmp/core_dumps
-            fi
-          when: on_fail
-      - store_artifacts:
-          name: 'Save regressions'
-          path: src/test/regress/regression.diffs
-          when: on_fail
-      - store_artifacts:
-          name: 'Save core dumps'
-          path: /tmp/core_dumps
-          when: on_fail
-      - codecov/upload:
-          flags: 'test_<< parameters.pg_major >>,upgrade'
-
-  test-citus:
-    description: Runs the common tests of citus
-    parameters:
-      pg_major:
-        description: "postgres major version"
-        type: integer
-      image:
-        description: 'docker image to use as for the tests'
-        type: string
-        default: citus/exttester
-      image_tag:
-        description: 'docker image tag to use'
-        type: string
-      make:
-        description: "make target"
-        type: string
-    docker:
-      - image: '<< parameters.image >>:<< parameters.image_tag >>'
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - attach_workspace:
-          at: .
-      - run:
-          name: 'Install Extension'
-          command: |
-            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
-      - run:
-          name: 'Configure'
-          command: |
-            chown -R circleci .
-            gosu circleci ./configure
-      - run:
-          name: 'Enable core dumps'
-          command: |
-            ulimit -c unlimited
-      - run:
-          name: 'Run Test'
-          command: |
-            gosu circleci make -C src/test/regress << parameters.make >>
-          no_output_timeout: 2m
-      - run:
-          name: 'Regressions'
-          command: |
-            if [ -f "src/test/regress/regression.diffs" ]; then
-              cat src/test/regress/regression.diffs
-              exit 1
-            fi
-          when: on_fail
-      - run:
-          name: 'Copy coredumps'
-          command: |
-            mkdir -p /tmp/core_dumps
-            if ls core.* 1> /dev/null 2>&1; then
-                cp core.* /tmp/core_dumps
-            fi
-          when: on_fail
-      - store_artifacts:
-          name: 'Save regressions'
-          path: src/test/regress/regression.diffs
-          when: on_fail
-      - store_artifacts:
-          name: 'Save core dumps'
-          path: /tmp/core_dumps
-          when: on_fail
-      - codecov/upload:
-          flags: 'test_<< parameters.pg_major >>,<< parameters.make >>'
-          when: always
-
-  check-merge-to-enterprise:
-    docker:
-      - image: citus/extbuilder:13.0
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - run:
-          command: |
-            ci/check_enterprise_merge.sh
-
-  ch_benchmark:
-    docker:
-      - image: buildpack-deps:stretch
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - azure-cli/install
-      - azure-cli/login-with-service-principal
-      - run:
-          command: |
-            cd ./src/test/hammerdb
-            sh run_hammerdb.sh citusbot_ch_benchmark_rg
-          name: install dependencies and run ch_benchmark tests
-          no_output_timeout: 20m
-
-  tpcc_benchmark:
-    docker:
-      - image: buildpack-deps:stretch
-    working_directory: /home/circleci/project
-    steps:
-      - checkout
-      - azure-cli/install
-      - azure-cli/login-with-service-principal
-      - run:
-          command: |
-            cd ./src/test/hammerdb
-            sh run_hammerdb.sh citusbot_tpcc_benchmark_rg
-          name: install dependencies and run ch_benchmark tests
-          no_output_timeout: 20m
-
-workflows:
-  version: 2
-  build_and_test:
-    jobs:
-
-      - check-merge-to-enterprise:
-          filters:
-            branches:
-              ignore:
-                - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.*
-
-      - build:
-          name: build-11
-          pg_major: 11
-          image_tag: '11.9'
-      - build:
-          name: build-12
-          pg_major: 12
-          image_tag: '12.4'
-      - build:
-          name: build-13
-          pg_major: 13
-          image_tag: '13.0'
-
-      - check-style
-      - check-sql-snapshots
-
-      - test-citus:
-          name: 'test-11_check-multi'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-multi
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-mx'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-multi-mx
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-vanilla'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-vanilla
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-isolation'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-isolation
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-worker'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-worker
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-follower-cluster'
-          pg_major: 11
-          image_tag: '11.9'
-          make: check-follower-cluster
-          requires: [build-11]
-      - test-citus:
-          name: 'test-11_check-failure'
-          pg_major: 11
-          image: citus/failtester
-          image_tag: '11.9'
-          make: check-failure
-          requires: [build-11]
-
-      - test-citus:
-          name: 'test-12_check-multi'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-multi
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-mx'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-multi-mx
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-vanilla'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-vanilla
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-isolation'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-isolation
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-worker'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-worker
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-follower-cluster'
-          pg_major: 12
-          image_tag: '12.4'
-          make: check-follower-cluster
-          requires: [build-12]
-      - test-citus:
-          name: 'test-12_check-failure'
-          pg_major: 12
-          image: citus/failtester
-          image_tag: '12.4'
-          make: check-failure
-          requires: [build-12]
-
-      - test-citus:
-          name: 'test-13_check-multi'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-multi
-          requires: [build-13]
-      - test-citus:
-          name: 'test-13_check-mx'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-multi-mx
-          requires: [build-13]
-      - test-citus:
-          name: 'test-13_check-vanilla'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-vanilla
-          requires: [build-13]
-      - test-citus:
-          name: 'test-13_check-isolation'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-isolation
-          requires: [build-13]
-      - test-citus:
-          name: 'test-13_check-worker'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-worker
-          requires: [build-13]
-      - test-citus:
-          name: 'test-13_check-follower-cluster'
-          pg_major: 13
-          image_tag: '13.0'
-          make: check-follower-cluster
-          requires: [build-13]
-
-      - test-citus:
-          name: 'test-13_check-failure'
-          pg_major: 13
-          image: citus/failtester
-          image_tag: '13.0'
-          make: check-failure
-          requires: [build-13]
-
-      - test-pg-upgrade:
-          name: 'test-11-12_check-pg-upgrade'
-          old_pg_major: 11
-          new_pg_major: 12
-          image_tag: latest
-          requires: [build-11,build-12]
-
-      - test-pg-upgrade:
-          name: 'test-12-13_check-pg-upgrade'
-          old_pg_major: 12
-          new_pg_major: 13
-          image_tag: latest
-          requires: [build-12,build-13]
-
-      - test-citus-upgrade:
-          name: test-11_check-citus-upgrade
-          pg_major: 11
-          image_tag: '11.9'
-          requires: [build-11]
-
-      - ch_benchmark:
-          requires: [build-13]
-          filters:
-            branches:
-              only:
-                - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix
-      - tpcc_benchmark:
-          requires: [build-13]
-          filters:
-            branches:
-              only:
-                - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix

.github/actions/parallelization/action.yml

@@ -0,0 +1,23 @@
+name: 'Parallelization matrix'
+inputs:
+  count:
+    required: false
+    default: 32
+outputs:
+  json:
+    value: ${{ steps.generate_matrix.outputs.json }}
+runs:
+  using: "composite"
+  steps:
+    - name: Generate parallelization matrix
+      id: generate_matrix
+      shell: bash
+      run: |-
+        json_array="{\"include\": ["
+        for ((i = 1; i <= ${{ inputs.count }}; i++)); do
+            json_array+="{\"id\":\"$i\"},"
+        done
+        json_array=${json_array%,}
+        json_array+=" ]}"
+        echo "json=$json_array" >> "$GITHUB_OUTPUT"
+        echo "json=$json_array"

.github/actions/save_logs_and_results/action.yml

@@ -0,0 +1,38 @@
+name: save_logs_and_results
+inputs:
+  folder:
+    required: false
+    default: "log"
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.1
+    name: Upload logs
+    with:
+      name: ${{ inputs.folder }}
+      if-no-files-found: ignore
+      path: |
+        src/test/**/proxy.output
+        src/test/**/results/
+        src/test/**/tmp_check/master/log
+        src/test/**/tmp_check/worker.57638/log
+        src/test/**/tmp_check/worker.57637/log
+        src/test/**/*.diffs
+        src/test/**/out/ddls.sql
+        src/test/**/out/queries.sql
+        src/test/**/logfile_*
+        /tmp/pg_upgrade_newData_logs
+  - name: Publish regression.diffs
+    run: |-
+      diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)"
+      if ! [ -z "$diffs" ]; then
+        echo '```diff' >> $GITHUB_STEP_SUMMARY
+        echo -E "$diffs" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        echo -E $diffs
+      fi
+    shell: bash
+  - name: Print stack traces
+    run: "./ci/print_stack_trace.sh"
+    if: failure()
+    shell: bash

.github/actions/setup_extension/action.yml

@@ -0,0 +1,35 @@
+name: setup_extension
+inputs:
+  pg_major:
+    required: false
+  skip_installation:
+    required: false
+    default: false
+    type: boolean
+runs:
+  using: composite
+  steps:
+  - name: Expose $PG_MAJOR to Github Env
+    run: |-
+        if [ -z "${{ inputs.pg_major }}" ]; then
+          echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
+        else
+          echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV
+        fi
+    shell: bash
+  - uses: actions/download-artifact@v3.0.1
+    with:
+      name: build-${{ env.PG_MAJOR }}
+  - name: Install Extension
+    if: ${{ inputs.skip_installation == 'false' }}
+    run: tar xfv "install-$PG_MAJOR.tar" --directory /
+    shell: bash
+  - name: Configure
+    run: |-
+      chown -R circleci .
+      git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      gosu circleci ./configure --without-pg-version-check
+    shell: bash
+  - name: Enable core dumps
+    run: ulimit -c unlimited
+    shell: bash

.github/actions/upload_coverage/action.yml

@@ -0,0 +1,15 @@
+name: coverage
+inputs:
+  flags:
+    required: false
+  codecov_token:
+    required: true
+runs:
+  using: composite
+  steps:
+  - uses: codecov/codecov-action@v3
+    with:
+      flags: ${{ inputs.flags }}
+      token: ${{ inputs.codecov_token }}
+      verbose: true
+      gcov: true

.github/workflows/build_and_test.yml

@@ -0,0 +1,326 @@
+name: Build & Test
+run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+on:
+  workflow_dispatch:
+    inputs:
+      skip_test_flakyness:
+        required: false
+        default: false
+        type: boolean
+  pull_request:
+    types: [opened, reopened,synchronize]
+jobs:
+  # Since GHA does not interpolate env varibles in matrix context, we need to
+  # define them in a separate job and use them in other jobs.
+  params:
+    runs-on: ubuntu-latest
+    name: Initialize parameters
+    outputs:
+      build_image_name: "citus/extbuilder"
+      test_image_name: "citus/exttester"
+      citusupgrade_image_name: "citus/citusupgradetester"
+      fail_test_image_name: "citus/failtester"
+      pgupgrade_image_name: "citus/pgupgradetester"
+      style_checker_image_name: "citus/stylechecker"
+      style_checker_tools_version: "latest"
+      image_suffix: ""
+      pg11_version: '{ "major": "11", "full": "11.9" }'
+      pg12_version: '{ "major": "12", "full": "12.4" }'
+      pg13_version: '{ "major": "13", "full": "13.0" }'
+      upgrade_pg_versions: "latest"
+    steps:
+      # Since GHA jobs needs at least one step we use a noop step here.
+      - name: Set up parameters
+        run: echo 'noop'
+  check-sql-snapshots:
+    needs: params
+    runs-on: ubuntu-20.04
+    container:
+      image: ${{ needs.params.outputs.build_image_name }}:latest
+      options: --user root
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Check Snapshots
+      run: |
+        git config --global --add safe.directory ${GITHUB_WORKSPACE}
+        ci/check_sql_snapshots.sh
+  check-style:
+    needs: params
+    runs-on: ubuntu-20.04
+    container:
+      image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
+    steps:
+    - name: Check Snapshots
+      run: |
+        git config --global --add safe.directory ${GITHUB_WORKSPACE}
+    - uses: actions/checkout@v3.5.0
+      with:
+        fetch-depth: 0
+    - name: Check C Style
+      run: citus_indent --check
+    - name: Fix whitespace
+      run: ci/editorconfig.sh
+    - name: Check if whitespace fixing changed anything, install editorconfig if it did
+      run: git diff --exit-code
+    - name: Remove useless declarations
+      run: ci/remove_useless_declarations.sh
+    - name: Check if changed
+      run: git diff --cached --exit-code
+    - name: Normalize test output
+      run: ci/normalize_expected.sh
+    - name: Check if changed
+      run: git diff --exit-code
+    - name: Check for C-style comments in migration files
+      run: ci/disallow_c_comments_in_migrations.sh
+    - name: Check if changed
+      run: git diff --exit-code
+    - name: Check for lengths of changelog entries
+      run: ci/disallow_long_changelog_entries.sh
+    - name: Check for banned C API usage
+      run: ci/banned.h.sh
+    - name: Check for tests missing in schedules
+      run: ci/check_all_tests_are_run.sh
+    - name: Check if all CI scripts are actually run
+      run: ci/check_all_ci_scripts_are_run.sh
+  build:
+    needs: params
+    name: Build for PG${{ fromJson(matrix.pg_version).major }}
+    strategy:
+      fail-fast: false
+      matrix:
+        image_name:
+          - ${{ needs.params.outputs.build_image_name }}
+        image_suffix:
+          - ${{ needs.params.outputs.image_suffix}}
+        pg_version:
+          - ${{ needs.params.outputs.pg11_version }}
+          - ${{ needs.params.outputs.pg12_version }}
+          - ${{ needs.params.outputs.pg13_version }}
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
+      options: --user root
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Expose $PG_MAJOR to Github Env
+      run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
+      shell: bash
+    - name: Build
+      run: "./ci/build-citus.sh"
+      shell: bash
+    - uses: actions/upload-artifact@v3.1.1
+      with:
+        name: build-${{ env.PG_MAJOR }}
+        path: |-
+          ./build-${{ env.PG_MAJOR }}/*
+          ./install-${{ env.PG_MAJOR }}.tar
+  test-citus:
+    name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - regress
+        image_name:
+          - ${{ needs.params.outputs.test_image_name }}
+        pg_version:
+          - ${{ needs.params.outputs.pg11_version }}
+          - ${{ needs.params.outputs.pg12_version }}
+          - ${{ needs.params.outputs.pg13_version }}
+        make:
+          - check-multi
+          - check-multi-mx
+          - check-vanilla
+          - check-isolation
+          - check-worker
+          - check-follower-cluster
+        include:
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg11_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg12_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root --dns=8.8.8.8
+      # Due to Github creates a default network for each job, we need to use
+      # --dns= to have similar DNS settings as our other CI systems or local
+      # machines. Otherwise, we may see different results.
+    needs:
+    - params
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - name: Install dependencies
+      if: matrix.make == 'check-failure'
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update || true
+        apt install git -y
+    - uses: "./.github/actions/setup_extension"
+    - name: Run Test
+      run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
+      timeout-minutes: 20
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+      with:
+        folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  test-pg-upgrade:
+    name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root
+    needs:
+    - params
+    - build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - old_pg_major: 11
+            new_pg_major: 12
+          - old_pg_major: 12
+            new_pg_major: 13
+    env:
+      old_pg_major: ${{ matrix.old_pg_major }}
+      new_pg_major: ${{ matrix.new_pg_major }}
+    steps:
+    - name: Install dependencies
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update || true
+        apt install git -y
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+      with:
+        pg_major: "${{ env.old_pg_major }}"
+    - uses: "./.github/actions/setup_extension"
+      with:
+        pg_major: "${{ env.new_pg_major }}"
+    - name: Install and test postgres upgrade
+      run: |-
+        gosu circleci \
+          make -C src/test/regress \
+            check-pg-upgrade \
+            old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \
+            new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin
+    - name: Copy pg_upgrade logs for newData dir
+      run: |-
+        mkdir -p /tmp/pg_upgrade_newData_logs
+        if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
+            cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
+        fi
+      if: failure()
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  test-citus-upgrade:
+    name: PG${{ fromJson(needs.params.outputs.pg11_version).major }} - check-citus-upgrade
+    runs-on: ubuntu-20.04
+    container:
+      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg11_version).full }}${{ needs.params.outputs.image_suffix }}"
+      options: --user root
+    needs:
+    - params
+    - build
+    steps:
+    - name: Install dependencies
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update
+        apt install git -y
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/setup_extension"
+      with:
+        skip_installation: true
+    - name: Install and test citus upgrade
+      run: |-
+        # run make check-citus-upgrade for all citus versions
+        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+        for citus_version in ${CITUS_VERSIONS}; do \
+          gosu circleci \
+            make -C src/test/regress \
+              check-citus-upgrade \
+              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+              citus-old-version=${citus_version} \
+              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
+              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
+        done;
+        # run make check-citus-upgrade-mixed for all citus versions
+        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
+        for citus_version in ${CITUS_VERSIONS}; do \
+          gosu circleci \
+            make -C src/test/regress \
+              check-citus-upgrade-mixed \
+              citus-old-version=${citus_version} \
+              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
+              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
+              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
+        done;
+    - uses: "./.github/actions/save_logs_and_results"
+      if: always()
+    - uses: "./.github/actions/upload_coverage"
+      if: always()
+      with:
+        flags: ${{ env.pg_major }}_upgrade
+        codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  ch_benchmark:
+    name: CH Benchmark
+    if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
+    runs-on: ubuntu-20.04
+    needs:
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: install dependencies and run ch_benchmark tests
+      uses: azure/CLI@v1
+      with:
+        inlineScript: |
+          cd ./src/test/hammerdb
+          chmod +x run_hammerdb.sh
+          run_hammerdb.sh citusbot_ch_benchmark_rg
+  tpcc_benchmark:
+    name: TPCC Benchmark
+    if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/')
+    runs-on: ubuntu-20.04
+    needs:
+    - build
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: install dependencies and run tpcc_benchmark tests
+      uses: azure/CLI@v1
+      with:
+        inlineScript: |
+          cd ./src/test/hammerdb
+          chmod +x run_hammerdb.sh
+          run_hammerdb.sh citusbot_tpcc_benchmark_rg

ci/build-citus.sh

@@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
 codename=${VERSION#*(}
 codename=${codename%)*}
 
-# get project from argument
-project="${CIRCLE_PROJECT_REPONAME}"
-
 # we'll do everything with absolute paths
 basedir="$(pwd)"
 
@@ -28,7 +25,7 @@ build_ext() {
   pg_major="$1"
 
   builddir="${basedir}/build-${pg_major}"
-  echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
+  echo "Beginning build for PostgreSQL ${pg_major}..." >&2
 
   # do everything in a subdirectory to avoid clutter in current directory
   mkdir -p "${builddir}" && cd "${builddir}"

ci/check_all_ci_scripts_are_run.sh

@@ -11,11 +11,11 @@ source ci/ci_helpers.sh
 ci_scripts=$(
     find ci/ -iname "*.sh" |
     sed -E 's#^ci/##g' |
-    grep -v -E '^(ci_helpers.sh|fix_style.sh)$'
+    grep -v -E '^(ci_helpers.sh|fix_style.sh|check_enterprise_merge.sh)$'
 )
 for script in $ci_scripts; do
-    if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then
+    if ! grep "\\bci/$script\\b" -r .github > /dev/null; then
-        echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml"
+        echo "ERROR: CI script with name \"$script\" is not actually used in .github folder"
         exit 1
     fi
     if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then

ci/check_enterprise_merge.sh

@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# Testing this script locally requires you to set the following environment
-# variables:
-# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN
-
-# fail if trying to reference a variable that is not set.
-set -u
-# exit immediately if a command fails
-set -e
-# Fail on pipe failures
-set -o pipefail
-
-PR_BRANCH="${CIRCLE_BRANCH}"
-ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise"
-
-# shellcheck disable=SC1091
-source ci/ci_helpers.sh
-
-# List executed commands. This is done so debugging this script is easier when
-# it fails. It's explicitly done after git remote add so username and password
-# are not shown in CI output (even though it's also filtered out by CircleCI)
-set -x
-
-check_compile () {
-    echo "INFO: checking if merged code can be compiled"
-    ./configure --without-libcurl
-    make -j10
-}
-
-# Clone current git repo (which should be community) to a temporary working
-# directory and go there
-GIT_DIR_ROOT="$(git rev-parse --show-toplevel)"
-TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)"
-git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR"
-cd "$TMP_GIT_DIR"
-
-# Fails in CI without this
-git config user.email "citus-bot@microsoft.com"
-git config user.name "citus bot"
-
-# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords
-{ set +x ; } 2> /dev/null
-git remote add enterprise "$ENTERPRISE_REMOTE"
-set -x
-
-git remote set-url --push enterprise no-pushing
-
-# Fetch enterprise-master
-git fetch enterprise enterprise-master
-
-
-git checkout "enterprise/enterprise-master"
-
-if git merge --no-commit "origin/$PR_BRANCH"; then
-    echo "INFO: community PR branch could be merged into enterprise-master"
-    # check that we can compile after the merge
-    if check_compile; then
-        exit 0
-    fi
-
-    echo "WARN: Failed to compile after community PR branch was merged into enterprise"
-fi
-
-# undo partial merge
-git merge --abort
-
-if ! git fetch enterprise "$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master"
-    exit 1
-fi
-
-# Show the top commit of the enterprise PR branch to make debugging easier
-git log -n 1 "enterprise/$PR_BRANCH"
-
-# Check that this branch contains the top commit of the current community PR
-# branch. If it does not it means it's not up to date with the current PR, so
-# the enterprise branch should be updated.
-if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then
-    echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch"
-    exit 1
-fi
-
-# Now check if we can merge the enterprise PR into enterprise-master without
-# issues.
-git merge --no-commit "enterprise/$PR_BRANCH"
-# check that we can compile after the merge
-check_compile