external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move GHA environment variables to workflow file (#7275) 

Since GHA does not interpolate env variables in a matrix context, This
PR defines them in a separate job and uses them in other jobs.
pull/7269/head^2
Gokhan Gulbiz 2023-10-26 14:54:58 +03:00 committed by GitHub
parent 10198b18e8
commit 2bf1472c8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
.github/workflows/build_and_test.yml vendored
View File

@ -13,10 +13,33 @@ on:
pull_request: pull_request:
types: [opened, reopened,synchronize] types: [opened, reopened,synchronize]
jobs: jobs:
# Since GHA does not interpolate env varibles in matrix context, we need to
# define them in a separate job and use them in other jobs.
params:
runs-on: ubuntu-latest
name: Initialize parameters
outputs:
build_image_name: "citus/extbuilder"
test_image_name: "citus/exttester"
citusupgrade_image_name: "citus/citusupgradetester"
fail_test_image_name: "citus/failtester"
pgupgrade_image_name: "citus/pgupgradetester"
style_checker_image_name: "citus/stylechecker"
style_checker_tools_version: "0.8.18"
image_suffix: "-v9d71045"
pg14_version: "14.9"
pg15_version: "15.4"
pg16_version: "16.0"
upgrade_pg_versions: "14.9-15.4-16.0"
steps:
# Since GHA jobs needs at least one step we use a noop step here.
- name: Set up parameters
run: echo 'noop'
check-sql-snapshots: check-sql-snapshots:
needs: params
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: ${{ vars.build_image_name }}:latest image: ${{ needs.params.outputs.build_image_name }}:latest
options: --user root options: --user root
steps: steps:
- uses: actions/checkout@v3.5.0 - uses: actions/checkout@v3.5.0
@ -25,9 +48,10 @@ jobs:
git config --global --add safe.directory ${GITHUB_WORKSPACE} git config --global --add safe.directory ${GITHUB_WORKSPACE}
ci/check_sql_snapshots.sh ci/check_sql_snapshots.sh
check-style: check-style:
needs: params
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: ${{ vars.style_checker_image_name }}:${{ vars.style_checker_tools_version }}${{ vars.image_suffix }} image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }}
steps: steps:
- name: Check Snapshots - name: Check Snapshots
run: | run: |
@ -68,18 +92,19 @@ jobs:
- name: Check for missing downgrade scripts - name: Check for missing downgrade scripts
run: ci/check_migration_files.sh run: ci/check_migration_files.sh
build: build:
needs: params
name: Build for PG ${{ matrix.pg_version}} name: Build for PG ${{ matrix.pg_version}}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
image_name: image_name:
- ${{ vars.build_image_name }} - ${{ needs.params.outputs.build_image_name }}
image_suffix: image_suffix:
- ${{ vars.image_suffix}} - ${{ needs.params.outputs.image_suffix}}
pg_version: pg_version:
- ${{ vars.pg14_version }} - ${{ needs.params.outputs.pg14_version }}
- ${{ vars.pg15_version }} - ${{ needs.params.outputs.pg15_version }}
- ${{ vars.pg16_version }} - ${{ needs.params.outputs.pg16_version }}
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ matrix.image_suffix }}" image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ matrix.image_suffix }}"
@ -106,11 +131,11 @@ jobs:
suite: suite:
- regress - regress
image_name: image_name:
- ${{ vars.test_image_name }} - ${{ needs.params.outputs.test_image_name }}
pg_version: pg_version:
- ${{ vars.pg14_version }} - ${{ needs.params.outputs.pg14_version }}
- ${{ vars.pg15_version }} - ${{ needs.params.outputs.pg15_version }}
- ${{ vars.pg16_version }} - ${{ needs.params.outputs.pg16_version }}
make: make:
- check-split - check-split
- check-multi - check-multi
@ -129,69 +154,70 @@ jobs:
- check-enterprise-isolation-logicalrep-3 - check-enterprise-isolation-logicalrep-3
include: include:
- make: check-failure - make: check-failure
pg_version: ${{ vars.pg14_version }} pg_version: ${{ needs.params.outputs.pg14_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-failure - make: check-failure
pg_version: ${{ vars.pg15_version }} pg_version: ${{ needs.params.outputs.pg15_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-failure - make: check-failure
pg_version: ${{ vars.pg16_version }} pg_version: ${{ needs.params.outputs.pg16_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-enterprise-failure - make: check-enterprise-failure
pg_version: ${{ vars.pg14_version }} pg_version: ${{ needs.params.outputs.pg14_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-enterprise-failure - make: check-enterprise-failure
pg_version: ${{ vars.pg15_version }} pg_version: ${{ needs.params.outputs.pg15_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-enterprise-failure - make: check-enterprise-failure
pg_version: ${{ vars.pg16_version }} pg_version: ${{ needs.params.outputs.pg16_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-pytest - make: check-pytest
pg_version: ${{ vars.pg14_version }} pg_version: ${{ needs.params.outputs.pg14_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-pytest - make: check-pytest
pg_version: ${{ vars.pg15_version }} pg_version: ${{ needs.params.outputs.pg15_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-pytest - make: check-pytest
pg_version: ${{ vars.pg16_version }} pg_version: ${{ needs.params.outputs.pg16_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: installcheck - make: installcheck
suite: cdc suite: cdc
image_name: ${{ vars.test_image_name }} image_name: ${{ needs.params.outputs.test_image_name }}
pg_version: ${{ vars.pg15_version }} pg_version: ${{ needs.params.outputs.pg15_version }}
- make: installcheck - make: installcheck
suite: cdc suite: cdc
image_name: ${{ vars.test_image_name }} image_name: ${{ needs.params.outputs.test_image_name }}
pg_version: ${{ vars.pg16_version }} pg_version: ${{ needs.params.outputs.pg16_version }}
- make: check-query-generator - make: check-query-generator
pg_version: ${{ vars.pg14_version }} pg_version: ${{ needs.params.outputs.pg14_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-query-generator - make: check-query-generator
pg_version: ${{ vars.pg15_version }} pg_version: ${{ needs.params.outputs.pg15_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
- make: check-query-generator - make: check-query-generator
pg_version: ${{ vars.pg16_version }} pg_version: ${{ needs.params.outputs.pg16_version }}
suite: regress suite: regress
image_name: ${{ vars.fail_test_image_name }} image_name: ${{ needs.params.outputs.fail_test_image_name }}
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}"
options: --user root --dns=8.8.8.8 options: --user root --dns=8.8.8.8
# Due to Github creates a default network for each job, we need to use # Due to Github creates a default network for each job, we need to use
# --dns= to have similar DNS settings as our other CI systems or local # --dns= to have similar DNS settings as our other CI systems or local
# machines. Otherwise, we may see different results. # machines. Otherwise, we may see different results.
needs: needs:
- params
- build - build
steps: steps:
- uses: actions/checkout@v3.5.0 - uses: actions/checkout@v3.5.0
@ -212,19 +238,20 @@ jobs:
name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }} name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }}
runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"]
container: container:
image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}"
options: --user root options: --user root
needs: needs:
- params
- build - build
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
image_name: image_name:
- ${{ vars.fail_test_image_name }} - ${{ needs.params.outputs.fail_test_image_name }}
pg_version: pg_version:
- ${{ vars.pg14_version }} - ${{ needs.params.outputs.pg14_version }}
- ${{ vars.pg15_version }} - ${{ needs.params.outputs.pg15_version }}
- ${{ vars.pg16_version }} - ${{ needs.params.outputs.pg16_version }}
parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
steps: steps:
- uses: actions/checkout@v3.5.0 - uses: actions/checkout@v3.5.0
@ -258,9 +285,10 @@ jobs:
name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: "${{ vars.pgupgrade_image_name }}:${{ vars.upgrade_pg_versions }}${{ vars.image_suffix }}" image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}"
options: --user root options: --user root
needs: needs:
- params
- build - build
strategy: strategy:
fail-fast: false fail-fast: false
@ -305,12 +333,13 @@ jobs:
flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
codecov_token: ${{ secrets.CODECOV_TOKEN }} codecov_token: ${{ secrets.CODECOV_TOKEN }}
test-citus-upgrade: test-citus-upgrade:
name: PG${{ vars.pg14_version }} - check-citus-upgrade name: PG${{ needs.params.outputs.pg14_version }} - check-citus-upgrade
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: "${{ vars.citusupgrade_image_name }}:${{ vars.pg14_version }}${{ vars.image_suffix }}" image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ needs.params.outputs.pg14_version }}${{ needs.params.outputs.image_suffix }}"
options: --user root options: --user root
needs: needs:
- params
- build - build
steps: steps:
- uses: actions/checkout@v3.5.0 - uses: actions/checkout@v3.5.0
@ -354,8 +383,9 @@ jobs:
CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: ${{ vars.test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} image: ${{ needs.params.outputs.test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
needs: needs:
- params
- test-citus - test-citus
- test-arbitrary-configs - test-arbitrary-configs
- test-citus-upgrade - test-citus-upgrade
@ -448,11 +478,12 @@ jobs:
name: Test flakyness name: Test flakyness
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: container:
image: ${{ vars.fail_test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }}
options: --user root options: --user root
env: env:
runs: 8 runs: 8
needs: needs:
- params
- build - build
- test-flakyness-pre - test-flakyness-pre
- prepare_parallelization_matrix_32 - prepare_parallelization_matrix_32

8
.github/workflows/packaging-test-pipelines.yml vendored
View File

@ -24,14 +24,14 @@ jobs:
- name: Get Postgres Versions - name: Get Postgres Versions
id: get-postgres-versions id: get-postgres-versions
run: | run: |
# Postgres versions are stored in .circleci/config.yml file in "build-[pg-version] format. Below command # Postgres versions are stored in .github/workflows/build_and_test.yml file in "pg[pg-version]_version"
# extracts the versions and get the unique values. # format. Below command extracts the versions and get the unique values.
pg_versions=`grep -Eo 'build-[[:digit:]]{2}' .circleci/config.yml|sed -e "s/^build-//"|sort|uniq|tr '\n' ','| head -c -1` pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE 'pg[0-9]+_version: "[0-9.]+"' | sed -E 's/pg([0-9]+)_version: "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',')
pg_versions_array="[ ${pg_versions} ]" pg_versions_array="[ ${pg_versions} ]"
echo "Supported PG Versions: ${pg_versions_array}" echo "Supported PG Versions: ${pg_versions_array}"
# Below line is needed to set the output variable to be used in the next job # Below line is needed to set the output variable to be used in the next job
echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT
shell: bash
rpm_build_tests: rpm_build_tests:
name: rpm_build_tests name: rpm_build_tests
needs: get_postgres_versions_from_file needs: get_postgres_versions_from_file