mirror of https://github.com/citusdata/citus.git
Backport GHA Migration to release-9.5 (#7282)
Co-authored-by: Jelte Fennema-Nio <jelte.fennema@microsoft.com>release-9.5
Gokhan Gulbiz
GitHub
parent
00cd336e70
commit
d6f30e6eb4
|
|
@ -1,544 +0,0 @@
|
|||
version: 2.1
|
||||
orbs:
|
||||
codecov: codecov/codecov@1.1.1
|
||||
azure-cli: circleci/azure-cli@1.0.0
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
description: Build the citus extension
|
||||
parameters:
|
||||
pg_major:
|
||||
description: postgres major version building citus for
|
||||
type: integer
|
||||
image:
|
||||
description: docker image to use for the build
|
||||
type: string
|
||||
default: citus/extbuilder
|
||||
image_tag:
|
||||
description: tag to use for the docker image
|
||||
type: string
|
||||
docker:
|
||||
- image: '<< parameters.image >>:<< parameters.image_tag >>'
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: 'Configure, Build, and Install'
|
||||
command: |
|
||||
./ci/build-citus.sh
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- build-<< parameters.pg_major >>/*
|
||||
- install-<<parameters.pg_major >>.tar
|
||||
|
||||
check-style:
|
||||
docker:
|
||||
- image: 'citus/stylechecker:latest'
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: 'Check Style'
|
||||
command: citus_indent --check
|
||||
- run:
|
||||
name: 'Fix whitespace'
|
||||
command: ci/editorconfig.sh
|
||||
- run:
|
||||
name: 'Check if whitespace fixing changed anything, install editorconfig if it did'
|
||||
command: git diff --exit-code
|
||||
- run:
|
||||
name: 'Remove useless declarations'
|
||||
command: ci/remove_useless_declarations.sh
|
||||
- run:
|
||||
name: 'Check if changed'
|
||||
command: git diff --cached --exit-code
|
||||
- run:
|
||||
name: 'Normalize test output'
|
||||
command: ci/normalize_expected.sh
|
||||
- run:
|
||||
name: 'Check if changed'
|
||||
command: git diff --exit-code
|
||||
- run:
|
||||
name: 'Check for C-style comments in migration files'
|
||||
command: ci/disallow_c_comments_in_migrations.sh
|
||||
- run:
|
||||
name: 'Check if changed'
|
||||
command: git diff --exit-code
|
||||
- run:
|
||||
name: 'Check for lengths of changelog entries'
|
||||
command: ci/disallow_long_changelog_entries.sh
|
||||
- run:
|
||||
name: 'Check for banned C API usage'
|
||||
command: ci/banned.h.sh
|
||||
- run:
|
||||
name: 'Check for tests missing in schedules'
|
||||
command: ci/check_all_tests_are_run.sh
|
||||
- run:
|
||||
name: 'Check if all CI scripts are actually run'
|
||||
command: ci/check_all_ci_scripts_are_run.sh
|
||||
|
||||
check-sql-snapshots:
|
||||
docker:
|
||||
- image: 'citus/extbuilder:latest'
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: 'Check Snapshots'
|
||||
command: ci/check_sql_snapshots.sh
|
||||
|
||||
test-pg-upgrade:
|
||||
description: Runs postgres upgrade tests
|
||||
parameters:
|
||||
old_pg_major:
|
||||
description: 'postgres major version to use before the upgrade'
|
||||
type: integer
|
||||
new_pg_major:
|
||||
description: 'postgres major version to upgrade to'
|
||||
type: integer
|
||||
image:
|
||||
description: 'docker image to use as for the tests'
|
||||
type: string
|
||||
default: citus/pgupgradetester
|
||||
image_tag:
|
||||
description: 'docker image tag to use'
|
||||
type: string
|
||||
default: latest
|
||||
docker:
|
||||
- image: '<< parameters.image >>:<< parameters.image_tag >>'
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run:
|
||||
name: 'Install Extension'
|
||||
command: |
|
||||
tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory /
|
||||
tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory /
|
||||
- run:
|
||||
name: 'Configure'
|
||||
command: |
|
||||
chown -R circleci .
|
||||
gosu circleci ./configure
|
||||
- run:
|
||||
name: 'Enable core dumps'
|
||||
command: |
|
||||
ulimit -c unlimited
|
||||
- run:
|
||||
name: 'Install and test postgres upgrade'
|
||||
command: |
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-pg-upgrade \
|
||||
old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \
|
||||
new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin
|
||||
no_output_timeout: 2m
|
||||
- run:
|
||||
name: 'Regressions'
|
||||
command: |
|
||||
if [ -f "src/test/regress/regression.diffs" ]; then
|
||||
cat src/test/regress/regression.diffs
|
||||
exit 1
|
||||
fi
|
||||
when: on_fail
|
||||
- run:
|
||||
name: 'Copy coredumps'
|
||||
command: |
|
||||
mkdir -p /tmp/core_dumps
|
||||
if ls core.* 1> /dev/null 2>&1; then
|
||||
cp core.* /tmp/core_dumps
|
||||
fi
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save regressions'
|
||||
path: src/test/regress/regression.diffs
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save core dumps'
|
||||
path: /tmp/core_dumps
|
||||
when: on_fail
|
||||
- codecov/upload:
|
||||
flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade'
|
||||
|
||||
test-citus-upgrade:
|
||||
description: Runs citus upgrade tests
|
||||
parameters:
|
||||
pg_major:
|
||||
description: "postgres major version"
|
||||
type: integer
|
||||
image:
|
||||
description: 'docker image to use as for the tests'
|
||||
type: string
|
||||
default: citus/citusupgradetester
|
||||
image_tag:
|
||||
description: 'docker image tag to use'
|
||||
type: string
|
||||
docker:
|
||||
- image: '<< parameters.image >>:<< parameters.image_tag >>'
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run:
|
||||
name: 'Configure'
|
||||
command: |
|
||||
chown -R circleci .
|
||||
gosu circleci ./configure
|
||||
- run:
|
||||
name: 'Enable core dumps'
|
||||
command: |
|
||||
ulimit -c unlimited
|
||||
- run:
|
||||
name: 'Install and test citus upgrade'
|
||||
command: |
|
||||
# run make check-citus-upgrade for all citus versions
|
||||
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
||||
for citus_version in ${CITUS_VERSIONS}; do \
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-citus-upgrade \
|
||||
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
||||
citus-pre-tar=/install-pg11-citus${citus_version}.tar \
|
||||
citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
|
||||
done;
|
||||
|
||||
# run make check-citus-upgrade-mixed for all citus versions
|
||||
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
||||
for citus_version in ${CITUS_VERSIONS}; do \
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-citus-upgrade-mixed \
|
||||
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
||||
citus-pre-tar=/install-pg11-citus${citus_version}.tar \
|
||||
citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
|
||||
done;
|
||||
no_output_timeout: 2m
|
||||
- run:
|
||||
name: 'Regressions'
|
||||
command: |
|
||||
if [ -f "src/test/regress/regression.diffs" ]; then
|
||||
cat src/test/regress/regression.diffs
|
||||
exit 1
|
||||
fi
|
||||
when: on_fail
|
||||
- run:
|
||||
name: 'Copy coredumps'
|
||||
command: |
|
||||
mkdir -p /tmp/core_dumps
|
||||
if ls core.* 1> /dev/null 2>&1; then
|
||||
cp core.* /tmp/core_dumps
|
||||
fi
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save regressions'
|
||||
path: src/test/regress/regression.diffs
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save core dumps'
|
||||
path: /tmp/core_dumps
|
||||
when: on_fail
|
||||
- codecov/upload:
|
||||
flags: 'test_<< parameters.pg_major >>,upgrade'
|
||||
|
||||
test-citus:
|
||||
description: Runs the common tests of citus
|
||||
parameters:
|
||||
pg_major:
|
||||
description: "postgres major version"
|
||||
type: integer
|
||||
image:
|
||||
description: 'docker image to use as for the tests'
|
||||
type: string
|
||||
default: citus/exttester
|
||||
image_tag:
|
||||
description: 'docker image tag to use'
|
||||
type: string
|
||||
make:
|
||||
description: "make target"
|
||||
type: string
|
||||
docker:
|
||||
- image: '<< parameters.image >>:<< parameters.image_tag >>'
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run:
|
||||
name: 'Install Extension'
|
||||
command: |
|
||||
tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
|
||||
- run:
|
||||
name: 'Configure'
|
||||
command: |
|
||||
chown -R circleci .
|
||||
gosu circleci ./configure
|
||||
- run:
|
||||
name: 'Enable core dumps'
|
||||
command: |
|
||||
ulimit -c unlimited
|
||||
- run:
|
||||
name: 'Run Test'
|
||||
command: |
|
||||
gosu circleci make -C src/test/regress << parameters.make >>
|
||||
no_output_timeout: 2m
|
||||
- run:
|
||||
name: 'Regressions'
|
||||
command: |
|
||||
if [ -f "src/test/regress/regression.diffs" ]; then
|
||||
cat src/test/regress/regression.diffs
|
||||
exit 1
|
||||
fi
|
||||
when: on_fail
|
||||
- run:
|
||||
name: 'Copy coredumps'
|
||||
command: |
|
||||
mkdir -p /tmp/core_dumps
|
||||
if ls core.* 1> /dev/null 2>&1; then
|
||||
cp core.* /tmp/core_dumps
|
||||
fi
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save regressions'
|
||||
path: src/test/regress/regression.diffs
|
||||
when: on_fail
|
||||
- store_artifacts:
|
||||
name: 'Save core dumps'
|
||||
path: /tmp/core_dumps
|
||||
when: on_fail
|
||||
- codecov/upload:
|
||||
flags: 'test_<< parameters.pg_major >>,<< parameters.make >>'
|
||||
when: always
|
||||
|
||||
check-merge-to-enterprise:
|
||||
docker:
|
||||
- image: citus/extbuilder:13.0
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
command: |
|
||||
ci/check_enterprise_merge.sh
|
||||
|
||||
ch_benchmark:
|
||||
docker:
|
||||
- image: buildpack-deps:stretch
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- azure-cli/install
|
||||
- azure-cli/login-with-service-principal
|
||||
- run:
|
||||
command: |
|
||||
cd ./src/test/hammerdb
|
||||
sh run_hammerdb.sh citusbot_ch_benchmark_rg
|
||||
name: install dependencies and run ch_benchmark tests
|
||||
no_output_timeout: 20m
|
||||
|
||||
tpcc_benchmark:
|
||||
docker:
|
||||
- image: buildpack-deps:stretch
|
||||
working_directory: /home/circleci/project
|
||||
steps:
|
||||
- checkout
|
||||
- azure-cli/install
|
||||
- azure-cli/login-with-service-principal
|
||||
- run:
|
||||
command: |
|
||||
cd ./src/test/hammerdb
|
||||
sh run_hammerdb.sh citusbot_tpcc_benchmark_rg
|
||||
name: install dependencies and run ch_benchmark tests
|
||||
no_output_timeout: 20m
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build_and_test:
|
||||
jobs:
|
||||
|
||||
- check-merge-to-enterprise:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.*
|
||||
|
||||
- build:
|
||||
name: build-11
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
- build:
|
||||
name: build-12
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
- build:
|
||||
name: build-13
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
|
||||
- check-style
|
||||
- check-sql-snapshots
|
||||
|
||||
- test-citus:
|
||||
name: 'test-11_check-multi'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-multi
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-mx'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-multi-mx
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-vanilla'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-vanilla
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-isolation'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-isolation
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-worker'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-worker
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-follower-cluster'
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
make: check-follower-cluster
|
||||
requires: [build-11]
|
||||
- test-citus:
|
||||
name: 'test-11_check-failure'
|
||||
pg_major: 11
|
||||
image: citus/failtester
|
||||
image_tag: '11.9'
|
||||
make: check-failure
|
||||
requires: [build-11]
|
||||
|
||||
- test-citus:
|
||||
name: 'test-12_check-multi'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-multi
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-mx'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-multi-mx
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-vanilla'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-vanilla
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-isolation'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-isolation
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-worker'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-worker
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-follower-cluster'
|
||||
pg_major: 12
|
||||
image_tag: '12.4'
|
||||
make: check-follower-cluster
|
||||
requires: [build-12]
|
||||
- test-citus:
|
||||
name: 'test-12_check-failure'
|
||||
pg_major: 12
|
||||
image: citus/failtester
|
||||
image_tag: '12.4'
|
||||
make: check-failure
|
||||
requires: [build-12]
|
||||
|
||||
- test-citus:
|
||||
name: 'test-13_check-multi'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-multi
|
||||
requires: [build-13]
|
||||
- test-citus:
|
||||
name: 'test-13_check-mx'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-multi-mx
|
||||
requires: [build-13]
|
||||
- test-citus:
|
||||
name: 'test-13_check-vanilla'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-vanilla
|
||||
requires: [build-13]
|
||||
- test-citus:
|
||||
name: 'test-13_check-isolation'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-isolation
|
||||
requires: [build-13]
|
||||
- test-citus:
|
||||
name: 'test-13_check-worker'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-worker
|
||||
requires: [build-13]
|
||||
- test-citus:
|
||||
name: 'test-13_check-follower-cluster'
|
||||
pg_major: 13
|
||||
image_tag: '13.0'
|
||||
make: check-follower-cluster
|
||||
requires: [build-13]
|
||||
|
||||
- test-citus:
|
||||
name: 'test-13_check-failure'
|
||||
pg_major: 13
|
||||
image: citus/failtester
|
||||
image_tag: '13.0'
|
||||
make: check-failure
|
||||
requires: [build-13]
|
||||
|
||||
- test-pg-upgrade:
|
||||
name: 'test-11-12_check-pg-upgrade'
|
||||
old_pg_major: 11
|
||||
new_pg_major: 12
|
||||
image_tag: latest
|
||||
requires: [build-11,build-12]
|
||||
|
||||
- test-pg-upgrade:
|
||||
name: 'test-12-13_check-pg-upgrade'
|
||||
old_pg_major: 12
|
||||
new_pg_major: 13
|
||||
image_tag: latest
|
||||
requires: [build-12,build-13]
|
||||
|
||||
- test-citus-upgrade:
|
||||
name: test-11_check-citus-upgrade
|
||||
pg_major: 11
|
||||
image_tag: '11.9'
|
||||
requires: [build-11]
|
||||
|
||||
- ch_benchmark:
|
||||
requires: [build-13]
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /ch_benchmark\/.*/ # match with ch_benchmark/ prefix
|
||||
- tpcc_benchmark:
|
||||
requires: [build-13]
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
name: 'Parallelization matrix'
|
||||
inputs:
|
||||
count:
|
||||
required: false
|
||||
default: 32
|
||||
outputs:
|
||||
json:
|
||||
value: ${{ steps.generate_matrix.outputs.json }}
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Generate parallelization matrix
|
||||
id: generate_matrix
|
||||
shell: bash
|
||||
run: |-
|
||||
json_array="{\"include\": ["
|
||||
for ((i = 1; i <= ${{ inputs.count }}; i++)); do
|
||||
json_array+="{\"id\":\"$i\"},"
|
||||
done
|
||||
json_array=${json_array%,}
|
||||
json_array+=" ]}"
|
||||
echo "json=$json_array" >> "$GITHUB_OUTPUT"
|
||||
echo "json=$json_array"
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
name: save_logs_and_results
|
||||
inputs:
|
||||
folder:
|
||||
required: false
|
||||
default: "log"
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/upload-artifact@v3.1.1
|
||||
name: Upload logs
|
||||
with:
|
||||
name: ${{ inputs.folder }}
|
||||
if-no-files-found: ignore
|
||||
path: |
|
||||
src/test/**/proxy.output
|
||||
src/test/**/results/
|
||||
src/test/**/tmp_check/master/log
|
||||
src/test/**/tmp_check/worker.57638/log
|
||||
src/test/**/tmp_check/worker.57637/log
|
||||
src/test/**/*.diffs
|
||||
src/test/**/out/ddls.sql
|
||||
src/test/**/out/queries.sql
|
||||
src/test/**/logfile_*
|
||||
/tmp/pg_upgrade_newData_logs
|
||||
- name: Publish regression.diffs
|
||||
run: |-
|
||||
diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)"
|
||||
if ! [ -z "$diffs" ]; then
|
||||
echo '```diff' >> $GITHUB_STEP_SUMMARY
|
||||
echo -E "$diffs" >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
echo -E $diffs
|
||||
fi
|
||||
shell: bash
|
||||
- name: Print stack traces
|
||||
run: "./ci/print_stack_trace.sh"
|
||||
if: failure()
|
||||
shell: bash
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
name: setup_extension
|
||||
inputs:
|
||||
pg_major:
|
||||
required: false
|
||||
skip_installation:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Expose $PG_MAJOR to Github Env
|
||||
run: |-
|
||||
if [ -z "${{ inputs.pg_major }}" ]; then
|
||||
echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
|
||||
else
|
||||
echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV
|
||||
fi
|
||||
shell: bash
|
||||
- uses: actions/download-artifact@v3.0.1
|
||||
with:
|
||||
name: build-${{ env.PG_MAJOR }}
|
||||
- name: Install Extension
|
||||
if: ${{ inputs.skip_installation == 'false' }}
|
||||
run: tar xfv "install-$PG_MAJOR.tar" --directory /
|
||||
shell: bash
|
||||
- name: Configure
|
||||
run: |-
|
||||
chown -R circleci .
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
gosu circleci ./configure --without-pg-version-check
|
||||
shell: bash
|
||||
- name: Enable core dumps
|
||||
run: ulimit -c unlimited
|
||||
shell: bash
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
name: coverage
|
||||
inputs:
|
||||
flags:
|
||||
required: false
|
||||
codecov_token:
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: codecov/codecov-action@v3
|
||||
with:
|
||||
flags: ${{ inputs.flags }}
|
||||
token: ${{ inputs.codecov_token }}
|
||||
verbose: true
|
||||
gcov: true
|
||||
|
|
@ -0,0 +1,326 @@
|
|||
name: Build & Test
|
||||
run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }}
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
skip_test_flakyness:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
pull_request:
|
||||
types: [opened, reopened,synchronize]
|
||||
jobs:
|
||||
# Since GHA does not interpolate env varibles in matrix context, we need to
|
||||
# define them in a separate job and use them in other jobs.
|
||||
params:
|
||||
runs-on: ubuntu-latest
|
||||
name: Initialize parameters
|
||||
outputs:
|
||||
build_image_name: "citus/extbuilder"
|
||||
test_image_name: "citus/exttester"
|
||||
citusupgrade_image_name: "citus/citusupgradetester"
|
||||
fail_test_image_name: "citus/failtester"
|
||||
pgupgrade_image_name: "citus/pgupgradetester"
|
||||
style_checker_image_name: "citus/stylechecker"
|
||||
style_checker_tools_version: "latest"
|
||||
image_suffix: ""
|
||||
pg11_version: '{ "major": "11", "full": "11.9" }'
|
||||
pg12_version: '{ "major": "12", "full": "12.4" }'
|
||||
pg13_version: '{ "major": "13", "full": "13.0" }'
|
||||
upgrade_pg_versions: "latest"
|
||||
steps:
|
||||
# Since GHA jobs needs at least one step we use a noop step here.
|
||||
- name: Set up parameters
|
||||
run: echo 'noop'
|
||||
check-sql-snapshots:
|
||||
needs: params
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: ${{ needs.params.outputs.build_image_name }}:latest
|
||||
options: --user root
|
||||
steps:
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- name: Check Snapshots
|
||||
run: |
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
ci/check_sql_snapshots.sh
|
||||
check-style:
|
||||
needs: params
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
|
||||
steps:
|
||||
- name: Check Snapshots
|
||||
run: |
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
- uses: actions/checkout@v3.5.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Check C Style
|
||||
run: citus_indent --check
|
||||
- name: Fix whitespace
|
||||
run: ci/editorconfig.sh
|
||||
- name: Check if whitespace fixing changed anything, install editorconfig if it did
|
||||
run: git diff --exit-code
|
||||
- name: Remove useless declarations
|
||||
run: ci/remove_useless_declarations.sh
|
||||
- name: Check if changed
|
||||
run: git diff --cached --exit-code
|
||||
- name: Normalize test output
|
||||
run: ci/normalize_expected.sh
|
||||
- name: Check if changed
|
||||
run: git diff --exit-code
|
||||
- name: Check for C-style comments in migration files
|
||||
run: ci/disallow_c_comments_in_migrations.sh
|
||||
- name: Check if changed
|
||||
run: git diff --exit-code
|
||||
- name: Check for lengths of changelog entries
|
||||
run: ci/disallow_long_changelog_entries.sh
|
||||
- name: Check for banned C API usage
|
||||
run: ci/banned.h.sh
|
||||
- name: Check for tests missing in schedules
|
||||
run: ci/check_all_tests_are_run.sh
|
||||
- name: Check if all CI scripts are actually run
|
||||
run: ci/check_all_ci_scripts_are_run.sh
|
||||
build:
|
||||
needs: params
|
||||
name: Build for PG${{ fromJson(matrix.pg_version).major }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
image_name:
|
||||
- ${{ needs.params.outputs.build_image_name }}
|
||||
image_suffix:
|
||||
- ${{ needs.params.outputs.image_suffix}}
|
||||
pg_version:
|
||||
- ${{ needs.params.outputs.pg11_version }}
|
||||
- ${{ needs.params.outputs.pg12_version }}
|
||||
- ${{ needs.params.outputs.pg13_version }}
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
|
||||
options: --user root
|
||||
steps:
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- name: Expose $PG_MAJOR to Github Env
|
||||
run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
|
||||
shell: bash
|
||||
- name: Build
|
||||
run: "./ci/build-citus.sh"
|
||||
shell: bash
|
||||
- uses: actions/upload-artifact@v3.1.1
|
||||
with:
|
||||
name: build-${{ env.PG_MAJOR }}
|
||||
path: |-
|
||||
./build-${{ env.PG_MAJOR }}/*
|
||||
./install-${{ env.PG_MAJOR }}.tar
|
||||
test-citus:
|
||||
name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite:
|
||||
- regress
|
||||
image_name:
|
||||
- ${{ needs.params.outputs.test_image_name }}
|
||||
pg_version:
|
||||
- ${{ needs.params.outputs.pg11_version }}
|
||||
- ${{ needs.params.outputs.pg12_version }}
|
||||
- ${{ needs.params.outputs.pg13_version }}
|
||||
make:
|
||||
- check-multi
|
||||
- check-multi-mx
|
||||
- check-vanilla
|
||||
- check-isolation
|
||||
- check-worker
|
||||
- check-follower-cluster
|
||||
include:
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg11_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg12_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
- make: check-failure
|
||||
pg_version: ${{ needs.params.outputs.pg13_version }}
|
||||
suite: regress
|
||||
image_name: ${{ needs.params.outputs.fail_test_image_name }}
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
|
||||
options: --user root --dns=8.8.8.8
|
||||
# Due to Github creates a default network for each job, we need to use
|
||||
# --dns= to have similar DNS settings as our other CI systems or local
|
||||
# machines. Otherwise, we may see different results.
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
steps:
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- name: Install dependencies
|
||||
if: matrix.make == 'check-failure'
|
||||
run: |-
|
||||
# update stretch repositories
|
||||
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
||||
apt update || true
|
||||
apt install git -y
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
- name: Run Test
|
||||
run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
|
||||
timeout-minutes: 20
|
||||
- uses: "./.github/actions/save_logs_and_results"
|
||||
if: always()
|
||||
with:
|
||||
folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }}
|
||||
- uses: "./.github/actions/upload_coverage"
|
||||
if: always()
|
||||
with:
|
||||
flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }}
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
test-pg-upgrade:
|
||||
name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
|
||||
options: --user root
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- old_pg_major: 11
|
||||
new_pg_major: 12
|
||||
- old_pg_major: 12
|
||||
new_pg_major: 13
|
||||
env:
|
||||
old_pg_major: ${{ matrix.old_pg_major }}
|
||||
new_pg_major: ${{ matrix.new_pg_major }}
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |-
|
||||
# update stretch repositories
|
||||
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
||||
apt update || true
|
||||
apt install git -y
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
with:
|
||||
pg_major: "${{ env.old_pg_major }}"
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
with:
|
||||
pg_major: "${{ env.new_pg_major }}"
|
||||
- name: Install and test postgres upgrade
|
||||
run: |-
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-pg-upgrade \
|
||||
old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \
|
||||
new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin
|
||||
- name: Copy pg_upgrade logs for newData dir
|
||||
run: |-
|
||||
mkdir -p /tmp/pg_upgrade_newData_logs
|
||||
if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then
|
||||
cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs
|
||||
fi
|
||||
if: failure()
|
||||
- uses: "./.github/actions/save_logs_and_results"
|
||||
if: always()
|
||||
- uses: "./.github/actions/upload_coverage"
|
||||
if: always()
|
||||
with:
|
||||
flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
test-citus-upgrade:
|
||||
name: PG${{ fromJson(needs.params.outputs.pg11_version).major }} - check-citus-upgrade
|
||||
runs-on: ubuntu-20.04
|
||||
container:
|
||||
image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg11_version).full }}${{ needs.params.outputs.image_suffix }}"
|
||||
options: --user root
|
||||
needs:
|
||||
- params
|
||||
- build
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |-
|
||||
# update stretch repositories
|
||||
sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list
|
||||
apt update
|
||||
apt install git -y
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- uses: "./.github/actions/setup_extension"
|
||||
with:
|
||||
skip_installation: true
|
||||
- name: Install and test citus upgrade
|
||||
run: |-
|
||||
# run make check-citus-upgrade for all citus versions
|
||||
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
||||
for citus_version in ${CITUS_VERSIONS}; do \
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-citus-upgrade \
|
||||
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
||||
citus-old-version=${citus_version} \
|
||||
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
|
||||
citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
|
||||
done;
|
||||
# run make check-citus-upgrade-mixed for all citus versions
|
||||
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
|
||||
for citus_version in ${CITUS_VERSIONS}; do \
|
||||
gosu circleci \
|
||||
make -C src/test/regress \
|
||||
check-citus-upgrade-mixed \
|
||||
citus-old-version=${citus_version} \
|
||||
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
|
||||
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
|
||||
citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \
|
||||
done;
|
||||
- uses: "./.github/actions/save_logs_and_results"
|
||||
if: always()
|
||||
- uses: "./.github/actions/upload_coverage"
|
||||
if: always()
|
||||
with:
|
||||
flags: ${{ env.pg_major }}_upgrade
|
||||
codecov_token: ${{ secrets.CODECOV_TOKEN }}
|
||||
ch_benchmark:
|
||||
name: CH Benchmark
|
||||
if: startsWith(github.ref, 'refs/heads/ch_benchmark/')
|
||||
runs-on: ubuntu-20.04
|
||||
needs:
|
||||
- build
|
||||
steps:
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{ secrets.AZURE_CREDENTIALS }}
|
||||
- name: install dependencies and run ch_benchmark tests
|
||||
uses: azure/CLI@v1
|
||||
with:
|
||||
inlineScript: |
|
||||
cd ./src/test/hammerdb
|
||||
chmod +x run_hammerdb.sh
|
||||
run_hammerdb.sh citusbot_ch_benchmark_rg
|
||||
tpcc_benchmark:
|
||||
name: TPCC Benchmark
|
||||
if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/')
|
||||
runs-on: ubuntu-20.04
|
||||
needs:
|
||||
- build
|
||||
steps:
|
||||
- uses: actions/checkout@v3.5.0
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{ secrets.AZURE_CREDENTIALS }}
|
||||
- name: install dependencies and run tpcc_benchmark tests
|
||||
uses: azure/CLI@v1
|
||||
with:
|
||||
inlineScript: |
|
||||
cd ./src/test/hammerdb
|
||||
chmod +x run_hammerdb.sh
|
||||
run_hammerdb.sh citusbot_tpcc_benchmark_rg
|
||||
|
|
@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version}
|
|||
codename=${VERSION#*(}
|
||||
codename=${codename%)*}
|
||||
|
||||
# get project from argument
|
||||
project="${CIRCLE_PROJECT_REPONAME}"
|
||||
|
||||
# we'll do everything with absolute paths
|
||||
basedir="$(pwd)"
|
||||
|
||||
|
|
@ -28,7 +25,7 @@ build_ext() {
|
|||
pg_major="$1"
|
||||
|
||||
builddir="${basedir}/build-${pg_major}"
|
||||
echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2
|
||||
echo "Beginning build for PostgreSQL ${pg_major}..." >&2
|
||||
|
||||
# do everything in a subdirectory to avoid clutter in current directory
|
||||
mkdir -p "${builddir}" && cd "${builddir}"
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ source ci/ci_helpers.sh
|
|||
ci_scripts=$(
|
||||
find ci/ -iname "*.sh" |
|
||||
sed -E 's#^ci/##g' |
|
||||
grep -v -E '^(ci_helpers.sh|fix_style.sh)$'
|
||||
grep -v -E '^(ci_helpers.sh|fix_style.sh|check_enterprise_merge.sh)$'
|
||||
)
|
||||
for script in $ci_scripts; do
|
||||
if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then
|
||||
echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml"
|
||||
if ! grep "\\bci/$script\\b" -r .github > /dev/null; then
|
||||
echo "ERROR: CI script with name \"$script\" is not actually used in .github folder"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Testing this script locally requires you to set the following environment
|
||||
# variables:
|
||||
# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN
|
||||
|
||||
# fail if trying to reference a variable that is not set.
|
||||
set -u
|
||||
# exit immediately if a command fails
|
||||
set -e
|
||||
# Fail on pipe failures
|
||||
set -o pipefail
|
||||
|
||||
PR_BRANCH="${CIRCLE_BRANCH}"
|
||||
ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source ci/ci_helpers.sh
|
||||
|
||||
# List executed commands. This is done so debugging this script is easier when
|
||||
# it fails. It's explicitly done after git remote add so username and password
|
||||
# are not shown in CI output (even though it's also filtered out by CircleCI)
|
||||
set -x
|
||||
|
||||
check_compile () {
|
||||
echo "INFO: checking if merged code can be compiled"
|
||||
./configure --without-libcurl
|
||||
make -j10
|
||||
}
|
||||
|
||||
# Clone current git repo (which should be community) to a temporary working
|
||||
# directory and go there
|
||||
GIT_DIR_ROOT="$(git rev-parse --show-toplevel)"
|
||||
TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)"
|
||||
git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR"
|
||||
cd "$TMP_GIT_DIR"
|
||||
|
||||
# Fails in CI without this
|
||||
git config user.email "citus-bot@microsoft.com"
|
||||
git config user.name "citus bot"
|
||||
|
||||
# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords
|
||||
{ set +x ; } 2> /dev/null
|
||||
git remote add enterprise "$ENTERPRISE_REMOTE"
|
||||
set -x
|
||||
|
||||
git remote set-url --push enterprise no-pushing
|
||||
|
||||
# Fetch enterprise-master
|
||||
git fetch enterprise enterprise-master
|
||||
|
||||
|
||||
git checkout "enterprise/enterprise-master"
|
||||
|
||||
if git merge --no-commit "origin/$PR_BRANCH"; then
|
||||
echo "INFO: community PR branch could be merged into enterprise-master"
|
||||
# check that we can compile after the merge
|
||||
if check_compile; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "WARN: Failed to compile after community PR branch was merged into enterprise"
|
||||
fi
|
||||
|
||||
# undo partial merge
|
||||
git merge --abort
|
||||
|
||||
if ! git fetch enterprise "$PR_BRANCH" ; then
|
||||
echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Show the top commit of the enterprise PR branch to make debugging easier
|
||||
git log -n 1 "enterprise/$PR_BRANCH"
|
||||
|
||||
# Check that this branch contains the top commit of the current community PR
|
||||
# branch. If it does not it means it's not up to date with the current PR, so
|
||||
# the enterprise branch should be updated.
|
||||
if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then
|
||||
echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Now check if we can merge the enterprise PR into enterprise-master without
|
||||
# issues.
|
||||
git merge --no-commit "enterprise/$PR_BRANCH"
|
||||
# check that we can compile after the merge
|
||||
check_compile
|
||||
Loading…
Reference in New Issue