mirror of https://github.com/citusdata/citus.git
364 lines
14 KiB
YAML
364 lines
14 KiB
YAML
name: Build & Test
|
|
run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }}
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
skip_test_flakyness:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
pull_request:
|
|
types: [opened, reopened,synchronize]
|
|
jobs:
|
|
# Since GHA does not interpolate env varibles in matrix context, we need to
|
|
# define them in a separate job and use them in other jobs.
|
|
params:
|
|
runs-on: ubuntu-latest
|
|
name: Initialize parameters
|
|
outputs:
|
|
build_image_name: "citus/extbuilder"
|
|
test_image_name: "citus/exttester"
|
|
citusupgrade_image_name: "citus/citusupgradetester"
|
|
fail_test_image_name: "citus/failtester"
|
|
pgupgrade_image_name: "citus/pgupgradetester"
|
|
style_checker_image_name: "citus/stylechecker"
|
|
style_checker_tools_version: "latest"
|
|
image_suffix: ""
|
|
pg12_version: '{ "major": "12", "full": "12.8" }'
|
|
pg13_version: '{ "major": "13", "full": "13.4" }'
|
|
pg14_version: '{ "major": "14", "full": "14.0" }'
|
|
upgrade_pg_versions: "12.8-13.4-14.0"
|
|
steps:
|
|
# Since GHA jobs needs at least one step we use a noop step here.
|
|
- name: Set up parameters
|
|
run: echo 'noop'
|
|
check-sql-snapshots:
|
|
needs: params
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: ${{ needs.params.outputs.build_image_name }}:latest
|
|
options: --user root
|
|
steps:
|
|
- uses: actions/checkout@v3.5.0
|
|
- name: Check Snapshots
|
|
run: |
|
|
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
|
ci/check_sql_snapshots.sh
|
|
check-style:
|
|
needs: params
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
|
|
steps:
|
|
- name: Check Snapshots
|
|
run: |
|
|
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
|
- uses: actions/checkout@v3.5.0
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Check C Style
|
|
run: citus_indent --check
|
|
- name: Fix whitespace
|
|
run: ci/editorconfig.sh
|
|
- name: Check if whitespace fixing changed anything, install editorconfig if it did
|
|
run: git diff --exit-code
|
|
- name: Remove useless declarations
|
|
run: ci/remove_useless_declarations.sh
|
|
- name: Check if changed
|
|
run: git diff --cached --exit-code
|
|
- name: Normalize test output
|
|
run: ci/normalize_expected.sh
|
|
- name: Check if changed
|
|
run: git diff --exit-code
|
|
- name: Check for C-style comments in migration files
|
|
run: ci/disallow_c_comments_in_migrations.sh
|
|
- name: Check if changed
|
|
run: git diff --exit-code
|
|
- name: Check for gitignore entries .for source files
|
|
run: ci/fix_gitignore.sh
|
|
- name: Check if changed
|
|
run: git diff --exit-code
|
|
- name: Check for lengths of changelog entries
|
|
run: ci/disallow_long_changelog_entries.sh
|
|
- name: Check for banned C API usage
|
|
run: ci/banned.h.sh
|
|
- name: Check for tests missing in schedules
|
|
run: ci/check_all_tests_are_run.sh
|
|
- name: Check if all CI scripts are actually run
|
|
run: ci/check_all_ci_scripts_are_run.sh
|
|
- name: 'Check if all GUCs are sorted alphabetically'
|
|
run: ci/check_gucs_are_alphabetically_sorted.sh
|
|
build:
|
|
needs: params
|
|
name: Build for PG${{ fromJson(matrix.pg_version).major }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
image_name:
|
|
- ${{ needs.params.outputs.build_image_name }}
|
|
image_suffix:
|
|
- ${{ needs.params.outputs.image_suffix}}
|
|
pg_version:
|
|
- ${{ needs.params.outputs.pg12_version }}
|
|
- ${{ needs.params.outputs.pg13_version }}
|
|
- ${{ needs.params.outputs.pg14_version }}
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
|
|
options: --user root
|
|
steps:
|
|
- uses: actions/checkout@v3.5.0
|
|
- name: Expose $PG_MAJOR to Github Env
|
|
run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
|
|
shell: bash
|
|
- name: Build
|
|
run: "./ci/build-citus.sh"
|
|
shell: bash
|
|
- uses: actions/upload-artifact@v3.1.1
|
|
with:
|
|
name: build-${{ env.PG_MAJOR }}
|
|
path: |-
|
|
./build-${{ env.PG_MAJOR }}/*
|
|
./install-${{ env.PG_MAJOR }}.tar
|
|
test-citus:
|
|
name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
suite:
|
|
- regress
|
|
image_name:
|
|
- ${{ needs.params.outputs.test_image_name }}
|
|
pg_version:
|
|
- ${{ needs.params.outputs.pg12_version }}
|
|
- ${{ needs.params.outputs.pg13_version }}
|
|
- ${{ needs.params.outputs.pg14_version }}
|
|
make:
|
|
- check-multi
|
|
- check-multi-1
|
|
- check-multi-mx
|
|
- check-vanilla
|
|
- check-isolation
|
|
- check-worker
|
|
- check-operations
|
|
- check-follower-cluster
|
|
- check-columnar
|
|
- check-columnar-isolation
|
|
include:
|
|
- make: check-failure
|
|
pg_version: ${{ needs.params.outputs.pg12_version }}
|
|
suite: regress
|
|
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
|
- make: check-failure
|
|
pg_version: ${{ needs.params.outputs.pg13_version }}
|
|
suite: regress
|
|
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
|
- make: check-failure
|
|
pg_version: ${{ needs.params.outputs.pg14_version }}
|
|
suite: regress
|
|
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
|
- make: installcheck
|
|
suite: recovery
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg12_version }}
|
|
- make: installcheck
|
|
suite: recovery
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg13_version }}
|
|
- make: installcheck
|
|
suite: recovery
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg14_version }}
|
|
- make: installcheck
|
|
suite: columnar_freezing
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg12_version }}
|
|
- make: installcheck
|
|
suite: columnar_freezing
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg13_version }}
|
|
- make: installcheck
|
|
suite: columnar_freezing
|
|
image_name: ${{ needs.params.outputs.test_image_name }}
|
|
pg_version: ${{ needs.params.outputs.pg14_version }}
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
|
|
options: --user root --dns=8.8.8.8
|
|
# Due to Github creates a default network for each job, we need to use
|
|
# --dns= to have similar DNS settings as our other CI systems or local
|
|
# machines. Otherwise, we may see different results.
|
|
needs:
|
|
- params
|
|
- build
|
|
steps:
|
|
- uses: actions/checkout@v3.5.0
|
|
- name: Install dependencies
|
|
if: matrix.make == 'check-failure'
|
|
run: |-
|
|
# update stretch repositories
|
|
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
|
apt update || true
|
|
apt install git -y
|
|
- uses: "./.github/actions/setup_extension"
|
|
- name: Run Test
|
|
run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
|
|
timeout-minutes: 20
|
|
- uses: "./.github/actions/save_logs_and_results"
|
|
if: always()
|
|
with:
|
|
folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
|
|
- uses: "./.github/actions/upload_coverage"
|
|
if: always()
|
|
with:
|
|
flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
|
|
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
|
test-pg-upgrade:
|
|
name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
|
|
options: --user root
|
|
needs:
|
|
- params
|
|
- build
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- old_pg_major: 12
|
|
new_pg_major: 13
|
|
- old_pg_major: 12
|
|
new_pg_major: 14
|
|
- old_pg_major: 13
|
|
new_pg_major: 14
|
|
env:
|
|
old_pg_major: ${{ matrix.old_pg_major }}
|
|
new_pg_major: ${{ matrix.new_pg_major }}
|
|
steps:
|
|
- name: Install dependencies
|
|
run: |-
|
|
# update stretch repositories
|
|
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
|
apt update || true
|
|
apt install git -y
|
|
- uses: actions/checkout@v3.5.0
|
|
- uses: "./.github/actions/setup_extension"
|
|
with:
|
|
pg_major: "${{ env.old_pg_major }}"
|
|
- uses: "./.github/actions/setup_extension"
|
|
with:
|
|
pg_major: "${{ env.new_pg_major }}"
|
|
- name: Install and test postgres upgrade
|
|
run: |-
|
|
gosu circleci \
|
|
make -C src/test/regress \
|
|
check-pg-upgrade \
|
|
old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \
|
|
new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin
|
|
- name: Copy pg_upgrade logs for newData dir
|
|
run: |-
|
|
mkdir -p /tmp/pg_upgrade_newData_logs
|
|
if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
|
|
cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
|
|
fi
|
|
if: failure()
|
|
- uses: "./.github/actions/save_logs_and_results"
|
|
if: always()
|
|
- uses: "./.github/actions/upload_coverage"
|
|
if: always()
|
|
with:
|
|
flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
|
|
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
|
test-citus-upgrade:
|
|
name: PG${{ fromJson(needs.params.outputs.pg12_version).major }} - check-citus-upgrade
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg12_version).full }}${{ needs.params.outputs.image_suffix }}"
|
|
options: --user root
|
|
needs:
|
|
- params
|
|
- build
|
|
steps:
|
|
- name: Install dependencies
|
|
run: |-
|
|
# update stretch repositories
|
|
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
|
apt update || true
|
|
apt install git -y
|
|
- uses: actions/checkout@v3.5.0
|
|
- uses: "./.github/actions/setup_extension"
|
|
with:
|
|
skip_installation: true
|
|
- name: Install and test citus upgrade
|
|
run: |-
|
|
# run make check-citus-upgrade for all citus versions
|
|
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
|
for citus_version in ${CITUS_VERSIONS}; do \
|
|
export upgrade_test_old_citus_version="$citus_version"; \
|
|
gosu circleci \
|
|
make -C src/test/regress \
|
|
check-citus-upgrade \
|
|
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
|
citus-old-version=${citus_version} \
|
|
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
|
|
citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
|
|
done;
|
|
# run make check-citus-upgrade-mixed for all citus versions
|
|
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
|
for citus_version in ${CITUS_VERSIONS}; do \
|
|
gosu circleci \
|
|
make -C src/test/regress \
|
|
check-citus-upgrade-mixed \
|
|
citus-old-version=${citus_version} \
|
|
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
|
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
|
|
citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
|
|
done;
|
|
- uses: "./.github/actions/save_logs_and_results"
|
|
if: always()
|
|
- uses: "./.github/actions/upload_coverage"
|
|
if: always()
|
|
with:
|
|
flags: ${{ env.pg_major }}_upgrade
|
|
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
|
ch_benchmark:
|
|
name: CH Benchmark
|
|
if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- build
|
|
steps:
|
|
- uses: actions/checkout@v3.5.0
|
|
- uses: azure/login@v1
|
|
with:
|
|
creds: ${{ secrets.AZURE_CREDENTIALS }}
|
|
- name: install dependencies and run ch_benchmark tests
|
|
uses: azure/CLI@v1
|
|
with:
|
|
inlineScript: |
|
|
cd ./src/test/hammerdb
|
|
chmod +x run_hammerdb.sh
|
|
run_hammerdb.sh citusbot_ch_benchmark_rg
|
|
tpcc_benchmark:
|
|
name: TPCC Benchmark
|
|
if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/')
|
|
runs-on: ubuntu-20.04
|
|
needs:
|
|
- build
|
|
steps:
|
|
- uses: actions/checkout@v3.5.0
|
|
- uses: azure/login@v1
|
|
with:
|
|
creds: ${{ secrets.AZURE_CREDENTIALS }}
|
|
- name: install dependencies and run tpcc_benchmark tests
|
|
uses: azure/CLI@v1
|
|
with:
|
|
inlineScript: |
|
|
cd ./src/test/hammerdb
|
|
chmod +x run_hammerdb.sh
|
|
run_hammerdb.sh citusbot_tpcc_benchmark_rg
|