From 2209e3785dec20ff748c2598867a4ab30b7fdd6d Mon Sep 17 00:00:00 2001
From: Gokhan Gulbiz <ggulbiz@gmail.com>
Date: Wed, 1 Nov 2023 10:12:04 +0300
Subject: [PATCH] Reflect v10.2 changes to GHA workflow

---
 .github/workflows/build_and_test.yml | 294 +++++++--------------------
 1 file changed, 76 insertions(+), 218 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 4e9c4601a..d87027d84 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -25,12 +25,12 @@ jobs:
       fail_test_image_name: "citus/failtester"
       pgupgrade_image_name: "citus/pgupgradetester"
       style_checker_image_name: "citus/stylechecker"
-      style_checker_tools_version: "0.8.18"
-      image_suffix: "-v9d71045"
-      pg14_version: '{ "major": "14", "full": "14.9" }'
-      pg15_version: '{ "major": "15", "full": "15.4" }'
-      pg16_version: '{ "major": "16", "full": "16.0" }'
-      upgrade_pg_versions: "14.9-15.4-16.0"
+      style_checker_tools_version: "latest"
+      image_suffix: ""
+      pg12_version: '{ "major": "12", "full": "12.8" }'
+      pg13_version: '{ "major": "13", "full": "13.4" }'
+      pg14_version: '{ "major": "14", "full": "14.0" }'
+      upgrade_pg_versions: "12.8-13.4-14.0"
     steps:
       # Since GHA jobs needs at least one step we use a noop step here.
       - name: Set up parameters
@@ -61,24 +61,26 @@ jobs:
         fetch-depth: 0
     - name: Check C Style
       run: citus_indent --check
-    - name: Check Python style
-      run: black --check .
-    - name: Check Python import order
-      run: isort --check .
-    - name: Check Python lints
-      run: flake8 .
     - name: Fix whitespace
-      run: ci/editorconfig.sh && git diff --exit-code
+      run: ci/editorconfig.sh
+    - name: Check if whitespace fixing changed anything, install editorconfig if it did
+      run: git diff --exit-code
     - name: Remove useless declarations
-      run: ci/remove_useless_declarations.sh && git diff --cached --exit-code
+      run: ci/remove_useless_declarations.sh
+    - name: Check if changed
+      run: git diff --cached --exit-code
     - name: Normalize test output
-      run: ci/normalize_expected.sh && git diff --exit-code
+      run: ci/normalize_expected.sh
+    - name: Check if changed
+      run: git diff --exit-code
     - name: Check for C-style comments in migration files
-      run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code
-    - name: 'Check for comment--cached ns that start with # character in spec files'
-      run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code
+      run: ci/disallow_c_comments_in_migrations.sh
+    - name: Check if changed
+      run: git diff --exit-code
     - name: Check for gitignore entries .for source files
-      run: ci/fix_gitignore.sh && git diff --exit-code
+      run: ci/fix_gitignore.sh
+    - name: Check if changed
+      run: git diff --exit-code
     - name: Check for lengths of changelog entries
       run: ci/disallow_long_changelog_entries.sh
     - name: Check for banned C API usage
@@ -87,10 +89,8 @@ jobs:
       run: ci/check_all_tests_are_run.sh
     - name: Check if all CI scripts are actually run
       run: ci/check_all_ci_scripts_are_run.sh
-    - name: Check if all GUCs are sorted alphabetically
+    - name: 'Check if all GUCs are sorted alphabetically'
       run: ci/check_gucs_are_alphabetically_sorted.sh
-    - name: Check for missing downgrade scripts
-      run: ci/check_migration_files.sh
   build:
     needs: params
     name: Build for PG${{ fromJson(matrix.pg_version).major }}
@@ -102,9 +102,9 @@ jobs:
         image_suffix:
           - ${{ needs.params.outputs.image_suffix}}
         pg_version:
+          - ${{ needs.params.outputs.pg12_version }}
+          - ${{ needs.params.outputs.pg13_version }}
           - ${{ needs.params.outputs.pg14_version }}
-          - ${{ needs.params.outputs.pg15_version }}
-          - ${{ needs.params.outputs.pg16_version }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
@@ -133,82 +133,57 @@ jobs:
         image_name:
           - ${{ needs.params.outputs.test_image_name }}
         pg_version:
+          - ${{ needs.params.outputs.pg12_version }}
+          - ${{ needs.params.outputs.pg13_version }}
           - ${{ needs.params.outputs.pg14_version }}
-          - ${{ needs.params.outputs.pg15_version }}
-          - ${{ needs.params.outputs.pg16_version }}
         make:
-          - check-split
           - check-multi
           - check-multi-1
           - check-multi-mx
           - check-vanilla
           - check-isolation
+          - check-worker
           - check-operations
           - check-follower-cluster
           - check-columnar
           - check-columnar-isolation
-          - check-enterprise
-          - check-enterprise-isolation
-          - check-enterprise-isolation-logicalrep-1
-          - check-enterprise-isolation-logicalrep-2
-          - check-enterprise-isolation-logicalrep-3
         include:
           - make: check-failure
-            pg_version: ${{ needs.params.outputs.pg14_version }}
+            pg_version: ${{ needs.params.outputs.pg12_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-failure
-            pg_version: ${{ needs.params.outputs.pg15_version }}
+            pg_version: ${{ needs.params.outputs.pg13_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-failure
-            pg_version: ${{ needs.params.outputs.pg16_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-enterprise-failure
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-enterprise-failure
-            pg_version: ${{ needs.params.outputs.pg15_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-enterprise-failure
-            pg_version: ${{ needs.params.outputs.pg16_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-pytest
-            pg_version: ${{ needs.params.outputs.pg14_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-pytest
-            pg_version: ${{ needs.params.outputs.pg15_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-pytest
-            pg_version: ${{ needs.params.outputs.pg16_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: installcheck
-            suite: cdc
+            suite: recovery
             image_name: ${{ needs.params.outputs.test_image_name }}
-            pg_version: ${{ needs.params.outputs.pg15_version }}
+            pg_version: ${{ needs.params.outputs.pg12_version }}
           - make: installcheck
-            suite: cdc
+            suite: recovery
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+          - make: installcheck
+            suite: recovery
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg14_version }}
+          - make: installcheck
+            suite: columnar_freezing
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg12_version }}
+          - make: installcheck
+            suite: columnar_freezing
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg13_version }}
+          - make: installcheck
+            suite: columnar_freezing
             image_name: ${{ needs.params.outputs.test_image_name }}
-            pg_version: ${{ needs.params.outputs.pg16_version }}
-          - make: check-query-generator
             pg_version: ${{ needs.params.outputs.pg14_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-query-generator
-            pg_version: ${{ needs.params.outputs.pg15_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
-          - make: check-query-generator
-            pg_version: ${{ needs.params.outputs.pg16_version }}
-            suite: regress
-            image_name: ${{ needs.params.outputs.fail_test_image_name }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
@@ -221,6 +196,13 @@ jobs:
     - build
     steps:
     - uses: actions/checkout@v3.5.0
+    - name: Install dependencies
+      if: matrix.make == 'check-failure'
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update || true
+        apt install git -y
     - uses: "./.github/actions/setup_extension"
     - name: Run Test
       run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
@@ -234,53 +216,6 @@ jobs:
       with:
         flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
-  test-arbitrary-configs:
-    name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }}
-    runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"]
-    container:
-      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
-      options: --user root
-    needs:
-      - params
-      - build
-    strategy:
-      fail-fast: false
-      matrix:
-        image_name:
-          - ${{ needs.params.outputs.fail_test_image_name }}
-        pg_version:
-          - ${{ needs.params.outputs.pg14_version }}
-          - ${{ needs.params.outputs.pg15_version }}
-          - ${{ needs.params.outputs.pg16_version }}
-        parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
-    steps:
-    - uses: actions/checkout@v3.5.0
-    - uses: "./.github/actions/setup_extension"
-    - name: Test arbitrary configs
-      run: |-
-        # we use parallel jobs to split the tests into 6 parts and run them in parallel
-        # the script below extracts the tests for the current job
-        N=6  # Total number of jobs (see matrix.parallel)
-        X=${{ matrix.parallel }}  # Current job number
-        TESTS=$(src/test/regress/citus_tests/print_test_names.py |
-          tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{
-            split("", parts)
-            for (i = 1; i <= NF; i++) {
-                parts[i % N] = parts[i % N] $i ","
-            }
-            print substr(parts[X], 1, length(parts[X])-1)
-        }')
-        echo $TESTS
-        gosu circleci \
-          make -C src/test/regress \
-            check-arbitrary-configs parallel=4 CONFIGS=$TESTS
-    - uses: "./.github/actions/save_logs_and_results"
-      if: always()
-    - uses: "./.github/actions/upload_coverage"
-      if: always()
-      with:
-        flags: ${{ env.pg_major }}_upgrade
-        codecov_token: ${{ secrets.CODECOV_TOKEN }}
   test-pg-upgrade:
     name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
     runs-on: ubuntu-20.04
@@ -294,16 +229,22 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - old_pg_major: 14
-            new_pg_major: 15
-          - old_pg_major: 15
-            new_pg_major: 16
-          - old_pg_major: 14
-            new_pg_major: 16
+          - old_pg_major: 12
+            new_pg_major: 13
+          - old_pg_major: 12
+            new_pg_major: 14
+          - old_pg_major: 13
+            new_pg_major: 14
     env:
       old_pg_major: ${{ matrix.old_pg_major }}
       new_pg_major: ${{ matrix.new_pg_major }}
     steps:
+    - name: Install dependencies
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update || true
+        apt install git -y
     - uses: actions/checkout@v3.5.0
     - uses: "./.github/actions/setup_extension"
       with:
@@ -333,15 +274,21 @@ jobs:
         flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
   test-citus-upgrade:
-    name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade
+    name: PG${{ fromJson(needs.params.outputs.pg12_version).major }} - check-citus-upgrade
     runs-on: ubuntu-20.04
     container:
-      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}"
+      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg12_version).full }}${{ needs.params.outputs.image_suffix }}"
       options: --user root
     needs:
     - params
     - build
     steps:
+    - name: Install dependencies
+      run: |-
+        # update stretch repositories
+        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
+        apt update || true
+        apt install git -y
     - uses: actions/checkout@v3.5.0
     - uses: "./.github/actions/setup_extension"
       with:
@@ -351,6 +298,7 @@ jobs:
         # run make check-citus-upgrade for all citus versions
         # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
         for citus_version in ${CITUS_VERSIONS}; do \
+          export upgrade_test_old_citus_version="$citus_version"; \
           gosu circleci \
             make -C src/test/regress \
               check-citus-upgrade \
@@ -377,28 +325,6 @@ jobs:
       with:
         flags: ${{ env.pg_major }}_upgrade
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
-  upload-coverage:
-    if: always()
-    env:
-      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-    runs-on: ubuntu-20.04
-    container:
-      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
-    needs:
-      - params
-      - test-citus
-      - test-arbitrary-configs
-      - test-citus-upgrade
-      - test-pg-upgrade
-    steps:
-      - uses: actions/download-artifact@v3.0.1
-        with:
-          name: "codeclimate"
-          path: "codeclimate"
-      - name: Upload coverage results to Code Climate
-        run: |-
-          cc-test-reporter sum-coverage codeclimate/*.json -o total.json
-          cc-test-reporter upload-coverage -i total.json
   ch_benchmark:
     name: CH Benchmark
     if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
@@ -435,71 +361,3 @@ jobs:
           cd ./src/test/hammerdb
           chmod +x run_hammerdb.sh
           run_hammerdb.sh citusbot_tpcc_benchmark_rg
-  prepare_parallelization_matrix_32:
-    name: Parallel 32
-    if: ${{ needs.test-flakyness-pre.outputs.tests != ''}}
-    needs: test-flakyness-pre
-    runs-on: ubuntu-20.04
-    outputs:
-      json: ${{ steps.parallelization.outputs.json }}
-    steps:
-      - uses: actions/checkout@v3.5.0
-      - uses: "./.github/actions/parallelization"
-        id: parallelization
-        with:
-          count: 32
-  test-flakyness-pre:
-    name: Detect regression tests need to be ran
-    if: ${{ !inputs.skip_test_flakyness }}}
-    runs-on: ubuntu-20.04
-    needs: build
-    outputs:
-      tests: ${{ steps.detect-regression-tests.outputs.tests }}
-    steps:
-    - uses: actions/checkout@v3.5.0
-      with:
-        fetch-depth: 0
-    - name: Detect regression tests need to be ran
-      id: detect-regression-tests
-      run: |-
-        detected_changes=$(git diff origin/main... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true))
-        tests=${detected_changes}
-        if [ -z "$tests" ]; then
-            echo "No test found."
-        else
-          echo "Detected tests " $tests
-        fi
-        echo tests="$tests" >> "$GITHUB_OUTPUT"
-  test-flakyness:
-    if: ${{ needs.test-flakyness-pre.outputs.tests != ''}}
-    name: Test flakyness
-    runs-on: ubuntu-20.04
-    container:
-      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
-      options: --user root
-    env:
-      runs: 8
-    needs:
-    - params
-    - build
-    - test-flakyness-pre
-    - prepare_parallelization_matrix_32
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }}
-    steps:
-    - uses: actions/checkout@v3.5.0
-    - uses: actions/download-artifact@v3.0.1
-    - uses: "./.github/actions/setup_extension"
-    - name: Run minimal tests
-      run: |-
-        tests="${{ needs.test-flakyness-pre.outputs.tests }}"
-        tests_array=($tests)
-        for test in "${tests_array[@]}"
-        do
-            test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/")
-            gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line
-        done
-      shell: bash
-    - uses: "./.github/actions/save_logs_and_results"
-      if: always()