citus/.github/workflows/build_and_test.yml

name: Build & Test
run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }}
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
on:
  workflow_dispatch:
    inputs:
      skip_test_flakyness:
        required: false
        default: false
        type: boolean
  pull_request:
    types: [opened, reopened,synchronize]
jobs:
  # Since GHA does not interpolate env varibles in matrix context, we need to
  # define them in a separate job and use them in other jobs.
  params:
    runs-on: ubuntu-latest
    name: Initialize parameters
    outputs:
      build_image_name: "citus/extbuilder"
      test_image_name: "citus/exttester"
      citusupgrade_image_name: "citus/citusupgradetester"
      fail_test_image_name: "citus/failtester"
      pgupgrade_image_name: "citus/pgupgradetester"
      style_checker_image_name: "citus/stylechecker"
      style_checker_tools_version: "latest"
      image_suffix: ""
      pg11_version: '{ "major": "11", "full": "11.9" }'
      pg12_version: '{ "major": "12", "full": "12.4" }'
      upgrade_pg_versions: "latest"
    steps:
      # Since GHA jobs needs at least one step we use a noop step here.
      - name: Set up parameters
        run: echo 'noop'
  check-sql-snapshots:
    needs: params
    runs-on: ubuntu-20.04
    container:
      image: ${{ needs.params.outputs.build_image_name }}:latest
      options: --user root
    steps:
    - uses: actions/checkout@v3.5.0
    - name: Check Snapshots
      run: |
        git config --global --add safe.directory ${GITHUB_WORKSPACE}
        ci/check_sql_snapshots.sh        
  check-style:
    needs: params
    runs-on: ubuntu-20.04
    container:
      image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
    steps:
    - name: Check Snapshots
      run: |
        git config --global --add safe.directory ${GITHUB_WORKSPACE}        
    - uses: actions/checkout@v3.5.0
      with:
        fetch-depth: 0
    - name: Check C Style
      run: citus_indent --check
    - name: Fix whitespace
      run: ci/editorconfig.sh
    - name: Check if whitespace fixing changed anything, install editorconfig if it did
      run: git diff --exit-code
    - name: Remove useless declarations
      run: ci/remove_useless_declarations.sh
    - name: Check if changed
      run: git diff --cached --exit-code
    - name: Normalize test output
      run: ci/normalize_expected.sh
    - name: Check if changed
      run: git diff --exit-code
    - name: Check for C-style comments in migration files
      run: ci/disallow_c_comments_in_migrations.sh
    - name: Check if changed
      run: git diff --exit-code
    - name: Check for lengths of changelog entries
      run: ci/disallow_long_changelog_entries.sh
    - name: Check for banned C API usage
      run: ci/banned.h.sh
  build:
    needs: params
    name: Build for PG${{ fromJson(matrix.pg_version).major }}
    strategy:
      fail-fast: false
      matrix:
        image_name:
          - ${{ needs.params.outputs.build_image_name }}
        image_suffix:
          - ${{ needs.params.outputs.image_suffix}}
        pg_version:
          - ${{ needs.params.outputs.pg11_version }}
          - ${{ needs.params.outputs.pg12_version }}
    runs-on: ubuntu-20.04
    container:
      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
      options: --user root
    steps:
    - uses: actions/checkout@v3.5.0
    - name: Expose $PG_MAJOR to Github Env
      run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
      shell: bash
    - name: Build
      run: "./ci/build-citus.sh"
      shell: bash
    - uses: actions/upload-artifact@v3.1.1
      with:
        name: build-${{ env.PG_MAJOR }}
        path: |-
          ./build-${{ env.PG_MAJOR }}/*
          ./install-${{ env.PG_MAJOR }}.tar          
  test-citus:
    name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
    strategy:
      fail-fast: false
      matrix:
        suite:
          - regress
        image_name:
          - ${{ needs.params.outputs.test_image_name }}
        pg_version:
          - ${{ needs.params.outputs.pg11_version }}
          - ${{ needs.params.outputs.pg12_version }}
        make:
          - check-multi
          - check-multi-mx
          - check-vanilla
          - check-isolation
          - check-worker
          - check-follower-cluster
          - check-multi-task-tracker-extra
        include:
          - make: check-failure
            pg_version: ${{ needs.params.outputs.pg11_version }}
            suite: regress
            image_name: ${{ needs.params.outputs.fail_test_image_name }}
          - make: check-failure
            pg_version: ${{ needs.params.outputs.pg12_version }}
            suite: regress
            image_name: ${{ needs.params.outputs.fail_test_image_name }}
    runs-on: ubuntu-20.04
    container:
      image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
      options: --user root --dns=8.8.8.8
      # Due to Github creates a default network for each job, we need to use
      # --dns= to have similar DNS settings as our other CI systems or local
      # machines. Otherwise, we may see different results.
    needs:
    - params
    - build
    steps:
    - uses: actions/checkout@v3.5.0
    - name: Install dependencies
      if: matrix.make == 'check-failure'
      run: |-
        # update stretch repositories
        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
        apt update || true
        apt install git -y        
    - uses: "./.github/actions/setup_extension"
    - name: Run Test
      run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
      timeout-minutes: 20
    - uses: "./.github/actions/save_logs_and_results"
      if: always()
      with:
        folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
    - uses: "./.github/actions/upload_coverage"
      if: always()
      with:
        flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
        codecov_token: ${{ secrets.CODECOV_TOKEN }}
  test-pg-upgrade:
    name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
    runs-on: ubuntu-20.04
    container:
      image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
      options: --user root
    needs:
    - params
    - build
    strategy:
      fail-fast: false
      matrix:
        include:
          - old_pg_major: 11
            new_pg_major: 12
    env:
      old_pg_major: ${{ matrix.old_pg_major }}
      new_pg_major: ${{ matrix.new_pg_major }}
    steps:
    - name: Install dependencies
      run: |-
        # update stretch repositories
        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
        apt update || true
        apt install git -y        
    - name: Fix pg_config
      run: PG_CONFIG=/usr/lib/postgresql/${{ env.new_pg_major }} >>/bin/pg_config
    - uses: actions/checkout@v3.5.0
    - uses: "./.github/actions/setup_extension"
      with:
        pg_major: "${{ env.old_pg_major }}"
    - uses: "./.github/actions/setup_extension"
      with:
        pg_major: "${{ env.new_pg_major }}"
    - name: Install and test postgres upgrade
      run: |-
        gosu circleci \
          make -C src/test/regress \
            check-pg-upgrade \
            old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \
            new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin        
    - name: Copy pg_upgrade logs for newData dir
      run: |-
        mkdir -p /tmp/pg_upgrade_newData_logs
        if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
            cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
        fi        
      if: failure()
    - uses: "./.github/actions/save_logs_and_results"
      if: always()
    - uses: "./.github/actions/upload_coverage"
      if: always()
      with:
        flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
        codecov_token: ${{ secrets.CODECOV_TOKEN }}
  test-citus-upgrade:
    name: PG${{ fromJson(needs.params.outputs.pg11_version).major }} - check-citus-upgrade
    runs-on: ubuntu-20.04
    container:
      image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg11_version).full }}${{ needs.params.outputs.image_suffix }}"
      options: --user root
    needs:
    - params
    - build
    steps:
    - name: Install dependencies
      run: |-
        # update stretch repositories
        sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
        apt update
        apt install git -y        
    - uses: actions/checkout@v3.5.0
    - uses: "./.github/actions/setup_extension"
      with:
        skip_installation: true
    - name: Install and test citus upgrade
      run: |-
        # run make check-citus-upgrade for all citus versions
        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
        for citus_version in ${CITUS_VERSIONS}; do \
          gosu circleci \
            make -C src/test/regress \
              check-citus-upgrade \
              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
              citus-old-version=${citus_version} \
              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
        done;
        # run make check-citus-upgrade-mixed for all citus versions
        # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
        for citus_version in ${CITUS_VERSIONS}; do \
          gosu circleci \
            make -C src/test/regress \
              check-citus-upgrade-mixed \
              citus-old-version=${citus_version} \
              bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
              citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
              citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
        done;        
    - uses: "./.github/actions/save_logs_and_results"
      if: always()
    - uses: "./.github/actions/upload_coverage"
      if: always()
      with:
        flags: ${{ env.pg_major }}_upgrade
        codecov_token: ${{ secrets.CODECOV_TOKEN }}
  ch_benchmark:
    name: CH Benchmark
    if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
    runs-on: ubuntu-20.04
    needs:
    - build
    steps:
    - uses: actions/checkout@v3.5.0
    - uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
    - name: install dependencies and run ch_benchmark tests
      uses: azure/CLI@v1
      with:
        inlineScript: |
          cd ./src/test/hammerdb
          chmod +x run_hammerdb.sh
          run_hammerdb.sh citusbot_ch_benchmark_rg          
  tpcc_benchmark:
    name: TPCC Benchmark
    if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/')
    runs-on: ubuntu-20.04
    needs:
    - build
    steps:
    - uses: actions/checkout@v3.5.0
    - uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
    - name: install dependencies and run tpcc_benchmark tests
      uses: azure/CLI@v1
      with:
        inlineScript: |
          cd ./src/test/hammerdb
          chmod +x run_hammerdb.sh
          run_hammerdb.sh citusbot_tpcc_benchmark_rg