From d901fac1fca3c866744458c4127d4dbc1bcf000b Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Tue, 10 Oct 2023 16:58:50 +0300 Subject: [PATCH 01/12] CircleCI to GHA migration (#7154) Co-authored-by: Hanefi Onaldi (cherry picked from commit e0b0cdbb87de7aad7e00d2858b4a19ac496377d5) --- .circleci/config.yml | 494 ------------------ .github/actions/parallelization/action.yml | 23 + .../actions/save_logs_and_results/action.yml | 38 ++ .github/actions/setup_extension/action.yml | 35 ++ .github/actions/upload_coverage/action.yml | 27 + .github/workflows/build_and_test.yml | 474 +++++++++++++++++ .github/workflows/flaky_test_debugging.yml | 79 +++ ci/build-citus.sh | 5 +- 8 files changed, 677 insertions(+), 498 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/actions/parallelization/action.yml create mode 100644 .github/actions/save_logs_and_results/action.yml create mode 100644 .github/actions/setup_extension/action.yml create mode 100644 .github/actions/upload_coverage/action.yml create mode 100644 .github/workflows/build_and_test.yml create mode 100644 .github/workflows/flaky_test_debugging.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c6385ded7..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,494 +0,0 @@ -version: 2.1 -orbs: - codecov: codecov/codecov@1.1.1 - azure-cli: circleci/azure-cli@1.0.0 - -jobs: - - build: - description: Build the citus extension - parameters: - pg_major: - description: postgres major version building citus for - type: integer - image: - description: docker image to use for the build - type: string - default: citus/extbuilder - image_tag: - description: tag to use for the docker image - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - ./ci/build-citus.sh - - persist_to_workspace: - root: . - paths: - - build-<< parameters.pg_major >>/* - - install-<>.tar - - check-style: - docker: - - image: 'citus/stylechecker:latest' - steps: - - checkout - - run: - name: 'Check Style' - command: citus_indent --check - - run: - name: 'Fix whitespace' - command: ci/editorconfig.sh - - run: - name: 'Check if whitespace fixing changed anything, install editorconfig if it did' - command: git diff --exit-code - - run: - name: 'Remove useless declarations' - command: ci/remove_useless_declarations.sh - - run: - name: 'Check if changed' - command: git diff --cached --exit-code - - run: - name: 'Normalize test output' - command: ci/normalize_expected.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for C-style comments in migration files' - command: ci/disallow_c_comments_in_migrations.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for lengths of changelog entries' - command: ci/disallow_long_changelog_entries.sh - - run: - name: 'Check for banned C API usage' - command: ci/banned.h.sh - - check-sql-snapshots: - docker: - - image: 'citus/extbuilder:latest' - steps: - - checkout - - run: - name: 'Check Snapshots' - command: ci/check_sql_snapshots.sh - - test-pg-upgrade: - description: Runs postgres upgrade tests - parameters: - old_pg_major: - description: 'postgres major version to use before the upgrade' - type: integer - new_pg_major: - description: 'postgres major version to upgrade to' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/pgupgradetester - image_tag: - description: 'docker image tag to use' - type: string - default: latest - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory / - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - PG_CONFIG=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin/pg_config gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test postgres upgrade' - command: | - gosu circleci \ - make -C src/test/regress \ - check-pg-upgrade \ - old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ - new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - - test-citus-upgrade: - description: Runs citus upgrade tests - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/citusupgradetester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test citus upgrade' - command: | - # run make check-citus-upgrade for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - - # run make check-citus-upgrade-mixed for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade-mixed \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,upgrade' - - test-citus: - description: Runs the common tests of citus - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - make: - description: "make target" - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/regress << parameters.make >> - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' - when: always - - check-merge-to-enterprise: - docker: - - image: citus/extbuilder:13.0 - working_directory: /home/circleci/project - steps: - - checkout - - run: - command: | - ci/check_enterprise_merge.sh - - ch_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_ch_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - - tpcc_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_tpcc_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - -workflows: - version: 2 - build_and_test: - jobs: - - - check-merge-to-enterprise: - filters: - branches: - ignore: - - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.* - - - build: - name: build-11 - pg_major: 11 - image_tag: '11.9' - - build: - name: build-12 - pg_major: 12 - image_tag: '12.4' - - - check-style - - check-sql-snapshots - - - test-citus: - name: 'test-11_check-multi' - pg_major: 11 - image_tag: '11.9' - make: check-multi - requires: [build-11] - - test-citus: - name: 'test-11_check-task-tracker' - pg_major: 11 - image_tag: '11.9' - make: check-multi-task-tracker-extra - requires: [build-11] - - test-citus: - name: 'test-11_check-mx' - pg_major: 11 - image_tag: '11.9' - make: check-multi-mx - requires: [build-11] - - test-citus: - name: 'test-11_check-vanilla' - pg_major: 11 - image_tag: '11.9' - make: check-vanilla - requires: [build-11] - - test-citus: - name: 'test-11_check-isolation' - pg_major: 11 - image_tag: '11.9' - make: check-isolation - requires: [build-11] - - test-citus: - name: 'test-11_check-worker' - pg_major: 11 - image_tag: '11.9' - make: check-worker - requires: [build-11] - - test-citus: - name: 'test-11_check-follower-cluster' - pg_major: 11 - image_tag: '11.9' - make: check-follower-cluster - requires: [build-11] - - test-citus: - name: 'test-11_check-failure' - pg_major: 11 - image: citus/failtester - image_tag: '11.9' - make: check-failure - requires: [build-11] - - - test-citus: - name: 'test-12_check-multi' - pg_major: 12 - image_tag: '12.4' - make: check-multi - requires: [build-12] - - test-citus: - name: 'test-12_check-task-tracker' - pg_major: 12 - image_tag: '12.4' - make: check-multi-task-tracker-extra - requires: [build-12] - - test-citus: - name: 'test-12_check-mx' - pg_major: 12 - image_tag: '12.4' - make: check-multi-mx - requires: [build-12] - - test-citus: - name: 'test-12_check-vanilla' - pg_major: 12 - image_tag: '12.4' - make: check-vanilla - requires: [build-12] - - test-citus: - name: 'test-12_check-isolation' - pg_major: 12 - image_tag: '12.4' - make: check-isolation - requires: [build-12] - - test-citus: - name: 'test-12_check-worker' - pg_major: 12 - image_tag: '12.4' - make: check-worker - requires: [build-12] - - test-citus: - name: 'test-12_check-follower-cluster' - pg_major: 12 - image_tag: '12.4' - make: check-follower-cluster - requires: [build-12] - - test-citus: - name: 'test-12_check-failure' - pg_major: 12 - image: citus/failtester - image_tag: '12.4' - make: check-failure - requires: [build-12] - - - test-pg-upgrade: - name: 'test-11-12_check-pg-upgrade' - old_pg_major: 11 - new_pg_major: 12 - image_tag: latest - requires: [build-11,build-12] - - - test-citus-upgrade: - name: test-11_check-citus-upgrade - pg_major: 11 - image_tag: '11.9' - requires: [build-11] - - - ch_benchmark: - requires: [build-12] - filters: - branches: - only: - - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix - - tpcc_benchmark: - requires: [build-12] - filters: - branches: - only: - - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix diff --git a/.github/actions/parallelization/action.yml b/.github/actions/parallelization/action.yml new file mode 100644 index 000000000..1f7d00202 --- /dev/null +++ b/.github/actions/parallelization/action.yml @@ -0,0 +1,23 @@ +name: 'Parallelization matrix' +inputs: + count: + required: false + default: 32 +outputs: + json: + value: ${{ steps.generate_matrix.outputs.json }} +runs: + using: "composite" + steps: + - name: Generate parallelization matrix + id: generate_matrix + shell: bash + run: |- + json_array="{\"include\": [" + for ((i = 1; i <= ${{ inputs.count }}; i++)); do + json_array+="{\"id\":\"$i\"}," + done + json_array=${json_array%,} + json_array+=" ]}" + echo "json=$json_array" >> "$GITHUB_OUTPUT" + echo "json=$json_array" diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml new file mode 100644 index 000000000..0f238835d --- /dev/null +++ b/.github/actions/save_logs_and_results/action.yml @@ -0,0 +1,38 @@ +name: save_logs_and_results +inputs: + folder: + required: false + default: "log" +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.1 + name: Upload logs + with: + name: ${{ inputs.folder }} + if-no-files-found: ignore + path: | + src/test/**/proxy.output + src/test/**/results/ + src/test/**/tmp_check/master/log + src/test/**/tmp_check/worker.57638/log + src/test/**/tmp_check/worker.57637/log + src/test/**/*.diffs + src/test/**/out/ddls.sql + src/test/**/out/queries.sql + src/test/**/logfile_* + /tmp/pg_upgrade_newData_logs + - name: Publish regression.diffs + run: |- + diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)" + if ! [ -z "$diffs" ]; then + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo -E "$diffs" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo -E $diffs + fi + shell: bash + - name: Print stack traces + run: "./ci/print_stack_trace.sh" + if: failure() + shell: bash diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml new file mode 100644 index 000000000..96b408e7e --- /dev/null +++ b/.github/actions/setup_extension/action.yml @@ -0,0 +1,35 @@ +name: setup_extension +inputs: + pg_major: + required: false + skip_installation: + required: false + default: false + type: boolean +runs: + using: composite + steps: + - name: Expose $PG_MAJOR to Github Env + run: |- + if [ -z "${{ inputs.pg_major }}" ]; then + echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + else + echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/download-artifact@v3.0.1 + with: + name: build-${{ env.PG_MAJOR }} + - name: Install Extension + if: ${{ inputs.skip_installation == 'false' }} + run: tar xfv "install-$PG_MAJOR.tar" --directory / + shell: bash + - name: Configure + run: |- + chown -R circleci . + git config --global --add safe.directory ${GITHUB_WORKSPACE} + gosu circleci ./configure --without-pg-version-check + shell: bash + - name: Enable core dumps + run: ulimit -c unlimited + shell: bash diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml new file mode 100644 index 000000000..0b5f581a6 --- /dev/null +++ b/.github/actions/upload_coverage/action.yml @@ -0,0 +1,27 @@ +name: coverage +inputs: + flags: + required: false + codecov_token: + required: true +runs: + using: composite + steps: + - uses: codecov/codecov-action@v3 + with: + flags: ${{ inputs.flags }} + token: ${{ inputs.codecov_token }} + verbose: true + gcov: true + - name: Create codeclimate coverage + run: |- + lcov --directory . --capture --output-file lcov.info + lcov --remove lcov.info -o lcov.info '/usr/*' + sed "s=^SF:$PWD/=SF:=g" -i lcov.info # relative pats are required by codeclimate + mkdir -p /tmp/codeclimate + cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + path: "/tmp/codeclimate/*.json" + name: codeclimate diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 000000000..90a4b1432 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,474 @@ +name: Build & Test +run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +on: + workflow_dispatch: + inputs: + skip_test_flakyness: + required: false + default: false + type: boolean + pull_request: + types: [opened, reopened,synchronize] +jobs: + check-sql-snapshots: + runs-on: ubuntu-20.04 + container: + image: ${{ vars.build_image_name }}:latest + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + ci/check_sql_snapshots.sh + check-style: + runs-on: ubuntu-20.04 + container: + image: ${{ vars.style_checker_image_name }}:${{ vars.style_checker_tools_version }}${{ vars.image_suffix }} + steps: + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Check C Style + run: citus_indent --check + - name: Check Python style + run: black --check . + - name: Check Python import order + run: isort --check . + - name: Check Python lints + run: flake8 . + - name: Fix whitespace + run: ci/editorconfig.sh && git diff --exit-code + - name: Remove useless declarations + run: ci/remove_useless_declarations.sh && git diff --cached --exit-code + - name: Normalize test output + run: ci/normalize_expected.sh && git diff --exit-code + - name: Check for C-style comments in migration files + run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code + - name: 'Check for comment--cached ns that start with # character in spec files' + run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code + - name: Check for gitignore entries .for source files + run: ci/fix_gitignore.sh && git diff --exit-code + - name: Check for lengths of changelog entries + run: ci/disallow_long_changelog_entries.sh + - name: Check for banned C API usage + run: ci/banned.h.sh + - name: Check for tests missing in schedules + run: ci/check_all_tests_are_run.sh + - name: Check if all CI scripts are actually run + run: ci/check_all_ci_scripts_are_run.sh + - name: Check if all GUCs are sorted alphabetically + run: ci/check_gucs_are_alphabetically_sorted.sh + - name: Check for missing downgrade scripts + run: ci/check_migration_files.sh + build: + name: Build for PG ${{ matrix.pg_version}} + strategy: + fail-fast: false + matrix: + image_name: + - ${{ vars.build_image_name }} + image_suffix: + - ${{ vars.image_suffix}} + pg_version: + - ${{ vars.pg14_version }} + - ${{ vars.pg15_version }} + - ${{ vars.pg16_version }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ matrix.image_suffix }}" + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Expose $PG_MAJOR to Github Env + run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + shell: bash + - name: Build + run: "./ci/build-citus.sh" + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + test-citus: + name: PG${{ matrix.pg_version }} - ${{ matrix.make }} + strategy: + fail-fast: false + matrix: + suite: + - regress + image_name: + - ${{ vars.test_image_name }} + pg_version: + - ${{ vars.pg14_version }} + - ${{ vars.pg15_version }} + - ${{ vars.pg16_version }} + make: + - check-split + - check-multi + - check-multi-1 + - check-multi-mx + - check-vanilla + - check-isolation + - check-operations + - check-follower-cluster + - check-columnar + - check-columnar-isolation + - check-enterprise + - check-enterprise-isolation + - check-enterprise-isolation-logicalrep-1 + - check-enterprise-isolation-logicalrep-2 + - check-enterprise-isolation-logicalrep-3 + include: + - make: check-failure + pg_version: ${{ vars.pg14_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-failure + pg_version: ${{ vars.pg15_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-failure + pg_version: ${{ vars.pg16_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ vars.pg14_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ vars.pg15_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ vars.pg16_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ vars.pg14_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ vars.pg15_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ vars.pg16_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: installcheck + suite: cdc + image_name: ${{ vars.test_image_name }} + pg_version: ${{ vars.pg15_version }} + - make: installcheck + suite: cdc + image_name: ${{ vars.test_image_name }} + pg_version: ${{ vars.pg16_version }} + - make: check-query-generator + pg_version: ${{ vars.pg14_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ vars.pg15_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ vars.pg16_version }} + suite: regress + image_name: ${{ vars.fail_test_image_name }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" + options: --user root --dns=8.8.8.8 + # Due to Github creates a default network for each job, we need to use + # --dns= to have similar DNS settings as our other CI systems or local + # machines. Otherwise, we may see different results. + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Run Test + run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} + timeout-minutes: 20 + - uses: "./.github/actions/save_logs_and_results" + if: always() + with: + folder: ${{ matrix.pg_version }}_${{ matrix.make }} + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-arbitrary-configs: + name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }} + runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] + container: + image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" + options: --user root + needs: + - build + strategy: + fail-fast: false + matrix: + image_name: + - ${{ vars.fail_test_image_name }} + pg_version: + - ${{ vars.pg14_version }} + - ${{ vars.pg15_version }} + - ${{ vars.pg16_version }} + parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Test arbitrary configs + run: |- + # we use parallel jobs to split the tests into 6 parts and run them in parallel + # the script below extracts the tests for the current job + N=6 # Total number of jobs (see matrix.parallel) + X=${{ matrix.parallel }} # Current job number + TESTS=$(src/test/regress/citus_tests/print_test_names.py | + tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{ + split("", parts) + for (i = 1; i <= NF; i++) { + parts[i % N] = parts[i % N] $i "," + } + print substr(parts[X], 1, length(parts[X])-1) + }') + echo $TESTS + gosu circleci \ + make -C src/test/regress \ + check-arbitrary-configs parallel=4 CONFIGS=$TESTS + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-pg-upgrade: + name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ vars.pgupgrade_image_name }}:${{ vars.upgrade_pg_versions }}${{ vars.image_suffix }}" + options: --user root + needs: + - build + strategy: + fail-fast: false + matrix: + include: + - old_pg_major: 14 + new_pg_major: 15 + - old_pg_major: 15 + new_pg_major: 16 + - old_pg_major: 14 + new_pg_major: 16 + env: + old_pg_major: ${{ matrix.old_pg_major }} + new_pg_major: ${{ matrix.new_pg_major }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.old_pg_major }}" + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.new_pg_major }}" + - name: Install and test postgres upgrade + run: |- + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ + new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin + - name: Copy pg_upgrade logs for newData dir + run: |- + mkdir -p /tmp/pg_upgrade_newData_logs + if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then + cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs + fi + if: failure() + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-citus-upgrade: + name: PG${{ vars.pg14_version }} - check-citus-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ vars.citusupgrade_image_name }}:${{ vars.pg14_version }}${{ vars.image_suffix }}" + options: --user root + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + skip_installation: true + - name: Install and test citus upgrade + run: |- + # run make check-citus-upgrade for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-old-version=${citus_version} \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + # run make check-citus-upgrade-mixed for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade-mixed \ + citus-old-version=${citus_version} \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + upload-coverage: + if: always() + env: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + runs-on: ubuntu-20.04 + container: + image: ${{ vars.test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} + needs: + - test-citus + - test-arbitrary-configs + - test-citus-upgrade + - test-pg-upgrade + steps: + - uses: actions/download-artifact@v3.0.1 + with: + name: "codeclimate" + path: "codeclimate" + - name: Upload coverage results to Code Climate + run: |- + cc-test-reporter sum-coverage codeclimate/*.json -o total.json + cc-test-reporter upload-coverage -i total.json + ch_benchmark: + name: CH Benchmark + if: startsWith(github.ref, 'refs/heads/ch_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run ch_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_ch_benchmark_rg + tpcc_benchmark: + name: TPCC Benchmark + if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run tpcc_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_tpcc_benchmark_rg + prepare_parallelization_matrix_32: + name: Parallel 32 + if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} + needs: test-flakyness-pre + runs-on: ubuntu-20.04 + outputs: + json: ${{ steps.parallelization.outputs.json }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/parallelization" + id: parallelization + with: + count: 32 + test-flakyness-pre: + name: Detect regression tests need to be ran + if: ${{ !inputs.skip_test_flakyness }}} + runs-on: ubuntu-20.04 + needs: build + outputs: + tests: ${{ steps.detect-regression-tests.outputs.tests }} + steps: + - uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Detect regression tests need to be ran + id: detect-regression-tests + run: |- + detected_changes=$(git diff origin/main... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true)) + tests=${detected_changes} + if [ -z "$tests" ]; then + echo "No test found." + else + echo "Detected tests " $tests + fi + echo tests="$tests" >> "$GITHUB_OUTPUT" + test-flakyness: + if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} + name: Test flakyness + runs-on: ubuntu-20.04 + container: + image: ${{ vars.fail_test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} + options: --user root + env: + runs: 8 + needs: + - build + - test-flakyness-pre + - prepare_parallelization_matrix_32 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: actions/download-artifact@v3.0.1 + - uses: "./.github/actions/setup_extension" + - name: Run minimal tests + run: |- + tests="${{ needs.test-flakyness-pre.outputs.tests }}" + tests_array=($tests) + for test in "${tests_array[@]}" + do + test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/") + gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line + done + shell: bash + - uses: "./.github/actions/save_logs_and_results" + if: always() diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml new file mode 100644 index 000000000..a666c1cd5 --- /dev/null +++ b/.github/workflows/flaky_test_debugging.yml @@ -0,0 +1,79 @@ +name: Flaky test debugging +run-name: Flaky test debugging - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }}) +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +on: + workflow_dispatch: + inputs: + flaky_test: + required: true + type: string + description: Test to run + flaky_test_runs_per_job: + required: false + default: 8 + type: number + description: Number of times to run the test + flaky_test_parallel_jobs: + required: false + default: 32 + type: number + description: Number of parallel jobs to run +jobs: + build: + name: Build Citus + runs-on: ubuntu-latest + container: + image: ${{ vars.build_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Configure, Build, and Install + run: | + echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + ./ci/build-citus.sh + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + prepare_parallelization_matrix: + name: Prepare parallelization matrix + runs-on: ubuntu-latest + outputs: + json: ${{ steps.parallelization.outputs.json }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/parallelization" + id: parallelization + with: + count: ${{ inputs.flaky_test_parallel_jobs }} + test_flakyness: + name: Test flakyness + runs-on: ubuntu-latest + container: + image: ${{ vars.fail_test_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} + options: --user root + needs: + [build, prepare_parallelization_matrix] + env: + test: "${{ inputs.flaky_test }}" + runs: "${{ inputs.flaky_test_runs_per_job }}" + skip: false + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Run minimal tests + run: |- + gosu circleci src/test/regress/citus_tests/run_test.py ${{ env.test }} --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line + shell: bash + - uses: "./.github/actions/save_logs_and_results" + if: always() + with: + folder: ${{ matrix.id }} diff --git a/ci/build-citus.sh b/ci/build-citus.sh index 870ca4481..a185d8452 100755 --- a/ci/build-citus.sh +++ b/ci/build-citus.sh @@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} codename=${VERSION#*(} codename=${codename%)*} -# get project from argument -project="${CIRCLE_PROJECT_REPONAME}" - # we'll do everything with absolute paths basedir="$(pwd)" @@ -28,7 +25,7 @@ build_ext() { pg_major="$1" builddir="${basedir}/build-${pg_major}" - echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2 + echo "Beginning build for PostgreSQL ${pg_major}..." >&2 # do everything in a subdirectory to avoid clutter in current directory mkdir -p "${builddir}" && cd "${builddir}" From 2ca304bbaa677cec8589c58175783183d83e72cb Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 14:54:58 +0300 Subject: [PATCH 02/12] Move GHA environment variables to workflow file (#7275) Since GHA does not interpolate env variables in a matrix context, This PR defines them in a separate job and uses them in other jobs. (cherry picked from commit 2bf1472c8e56d13d88169ed6c1c38cede98c8c23) --- .github/workflows/build_and_test.yml | 131 +++++++++++++++++---------- 1 file changed, 81 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 90a4b1432..258c2cb7d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -13,10 +13,33 @@ on: pull_request: types: [opened, reopened,synchronize] jobs: + # Since GHA does not interpolate env varibles in matrix context, we need to + # define them in a separate job and use them in other jobs. + params: + runs-on: ubuntu-latest + name: Initialize parameters + outputs: + build_image_name: "citus/extbuilder" + test_image_name: "citus/exttester" + citusupgrade_image_name: "citus/citusupgradetester" + fail_test_image_name: "citus/failtester" + pgupgrade_image_name: "citus/pgupgradetester" + style_checker_image_name: "citus/stylechecker" + style_checker_tools_version: "0.8.18" + image_suffix: "-v9d71045" + pg14_version: "14.9" + pg15_version: "15.4" + pg16_version: "16.0" + upgrade_pg_versions: "14.9-15.4-16.0" + steps: + # Since GHA jobs needs at least one step we use a noop step here. + - name: Set up parameters + run: echo 'noop' check-sql-snapshots: + needs: params runs-on: ubuntu-20.04 container: - image: ${{ vars.build_image_name }}:latest + image: ${{ needs.params.outputs.build_image_name }}:latest options: --user root steps: - uses: actions/checkout@v3.5.0 @@ -25,9 +48,10 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} ci/check_sql_snapshots.sh check-style: + needs: params runs-on: ubuntu-20.04 container: - image: ${{ vars.style_checker_image_name }}:${{ vars.style_checker_tools_version }}${{ vars.image_suffix }} + image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }} steps: - name: Check Snapshots run: | @@ -68,18 +92,19 @@ jobs: - name: Check for missing downgrade scripts run: ci/check_migration_files.sh build: + needs: params name: Build for PG ${{ matrix.pg_version}} strategy: fail-fast: false matrix: image_name: - - ${{ vars.build_image_name }} + - ${{ needs.params.outputs.build_image_name }} image_suffix: - - ${{ vars.image_suffix}} + - ${{ needs.params.outputs.image_suffix}} pg_version: - - ${{ vars.pg14_version }} - - ${{ vars.pg15_version }} - - ${{ vars.pg16_version }} + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} runs-on: ubuntu-20.04 container: image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ matrix.image_suffix }}" @@ -106,11 +131,11 @@ jobs: suite: - regress image_name: - - ${{ vars.test_image_name }} + - ${{ needs.params.outputs.test_image_name }} pg_version: - - ${{ vars.pg14_version }} - - ${{ vars.pg15_version }} - - ${{ vars.pg16_version }} + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} make: - check-split - check-multi @@ -129,69 +154,70 @@ jobs: - check-enterprise-isolation-logicalrep-3 include: - make: check-failure - pg_version: ${{ vars.pg14_version }} + pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-failure - pg_version: ${{ vars.pg15_version }} + pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-failure - pg_version: ${{ vars.pg16_version }} + pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure - pg_version: ${{ vars.pg14_version }} + pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure - pg_version: ${{ vars.pg15_version }} + pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure - pg_version: ${{ vars.pg16_version }} + pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest - pg_version: ${{ vars.pg14_version }} + pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest - pg_version: ${{ vars.pg15_version }} + pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest - pg_version: ${{ vars.pg16_version }} + pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: installcheck suite: cdc - image_name: ${{ vars.test_image_name }} - pg_version: ${{ vars.pg15_version }} + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg15_version }} - make: installcheck suite: cdc - image_name: ${{ vars.test_image_name }} - pg_version: ${{ vars.pg16_version }} + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg16_version }} - make: check-query-generator - pg_version: ${{ vars.pg14_version }} + pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-query-generator - pg_version: ${{ vars.pg15_version }} + pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-query-generator - pg_version: ${{ vars.pg16_version }} + pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress - image_name: ${{ vars.fail_test_image_name }} + image_name: ${{ needs.params.outputs.fail_test_image_name }} runs-on: ubuntu-20.04 container: - image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" + image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}" options: --user root --dns=8.8.8.8 # Due to Github creates a default network for each job, we need to use # --dns= to have similar DNS settings as our other CI systems or local # machines. Otherwise, we may see different results. needs: + - params - build steps: - uses: actions/checkout@v3.5.0 @@ -212,19 +238,20 @@ jobs: name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }} runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] container: - image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ vars.image_suffix }}" + image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: + - params - build strategy: fail-fast: false matrix: image_name: - - ${{ vars.fail_test_image_name }} + - ${{ needs.params.outputs.fail_test_image_name }} pg_version: - - ${{ vars.pg14_version }} - - ${{ vars.pg15_version }} - - ${{ vars.pg16_version }} + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs steps: - uses: actions/checkout@v3.5.0 @@ -258,9 +285,10 @@ jobs: name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade runs-on: ubuntu-20.04 container: - image: "${{ vars.pgupgrade_image_name }}:${{ vars.upgrade_pg_versions }}${{ vars.image_suffix }}" + image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: + - params - build strategy: fail-fast: false @@ -305,12 +333,13 @@ jobs: flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-upgrade: - name: PG${{ vars.pg14_version }} - check-citus-upgrade + name: PG${{ needs.params.outputs.pg14_version }} - check-citus-upgrade runs-on: ubuntu-20.04 container: - image: "${{ vars.citusupgrade_image_name }}:${{ vars.pg14_version }}${{ vars.image_suffix }}" + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ needs.params.outputs.pg14_version }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: + - params - build steps: - uses: actions/checkout@v3.5.0 @@ -354,8 +383,9 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} runs-on: ubuntu-20.04 container: - image: ${{ vars.test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} + image: ${{ needs.params.outputs.test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} needs: + - params - test-citus - test-arbitrary-configs - test-citus-upgrade @@ -445,11 +475,12 @@ jobs: name: Test flakyness runs-on: ubuntu-20.04 container: - image: ${{ vars.fail_test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} + image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} options: --user root env: runs: 8 needs: + - params - build - test-flakyness-pre - prepare_parallelization_matrix_32 From c47309646761c23844ca0a1b6fd2284867ba91e5 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 31 Oct 2023 14:05:09 +0100 Subject: [PATCH 03/12] Only put major Postgres version in CI task name (#7289) Making tasks in CI required before merging to master is important and useful. The way this works is by saving the exact names of the required tasks in the admin interface of the repo. It has a search box to add them so it's not completely horrible, but doing so is quite a hassle since we have so many jobs. So limiting the amount of churn in this list of required jobs is quite useful. This changes the names of tasks to only include the major versions of Postgres, not the minor ones. Otherwise the next time we bump the minor versions we would have to remove and re-add each of the jobs. (cherry picked from commit 83e3fb817de8fc93c4819808b0dc8925c2bb6d38) --- .github/workflows/build_and_test.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 258c2cb7d..4e9c4601a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -27,9 +27,9 @@ jobs: style_checker_image_name: "citus/stylechecker" style_checker_tools_version: "0.8.18" image_suffix: "-v9d71045" - pg14_version: "14.9" - pg15_version: "15.4" - pg16_version: "16.0" + pg14_version: '{ "major": "14", "full": "14.9" }' + pg15_version: '{ "major": "15", "full": "15.4" }' + pg16_version: '{ "major": "16", "full": "16.0" }' upgrade_pg_versions: "14.9-15.4-16.0" steps: # Since GHA jobs needs at least one step we use a noop step here. @@ -93,7 +93,7 @@ jobs: run: ci/check_migration_files.sh build: needs: params - name: Build for PG ${{ matrix.pg_version}} + name: Build for PG${{ fromJson(matrix.pg_version).major }} strategy: fail-fast: false matrix: @@ -107,7 +107,7 @@ jobs: - ${{ needs.params.outputs.pg16_version }} runs-on: ubuntu-20.04 container: - image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ matrix.image_suffix }}" + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" options: --user root steps: - uses: actions/checkout@v3.5.0 @@ -124,7 +124,7 @@ jobs: ./build-${{ env.PG_MAJOR }}/* ./install-${{ env.PG_MAJOR }}.tar test-citus: - name: PG${{ matrix.pg_version }} - ${{ matrix.make }} + name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }} strategy: fail-fast: false matrix: @@ -211,7 +211,7 @@ jobs: image_name: ${{ needs.params.outputs.fail_test_image_name }} runs-on: ubuntu-20.04 container: - image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}" + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root --dns=8.8.8.8 # Due to Github creates a default network for each job, we need to use # --dns= to have similar DNS settings as our other CI systems or local @@ -228,17 +228,17 @@ jobs: - uses: "./.github/actions/save_logs_and_results" if: always() with: - folder: ${{ matrix.pg_version }}_${{ matrix.make }} + folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }} - uses: "./.github/actions/upload_coverage" if: always() with: flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} codecov_token: ${{ secrets.CODECOV_TOKEN }} test-arbitrary-configs: - name: PG${{ matrix.pg_version }} - check-arbitrary-configs-${{ matrix.parallel }} + name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }} runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] container: - image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ needs.params.outputs.image_suffix }}" + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params @@ -333,10 +333,10 @@ jobs: flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-upgrade: - name: PG${{ needs.params.outputs.pg14_version }} - check-citus-upgrade + name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade runs-on: ubuntu-20.04 container: - image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ needs.params.outputs.pg14_version }}${{ needs.params.outputs.image_suffix }}" + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params @@ -383,7 +383,7 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} runs-on: ubuntu-20.04 container: - image: ${{ needs.params.outputs.test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} + image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }} needs: - params - test-citus From b106e54bedc99e870404741c44d2376805747b03 Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Tue, 31 Oct 2023 18:00:10 +0300 Subject: [PATCH 04/12] Disable CircleCI (#7276) We are switching to Github Actions. In the test period it has worked well enough, so now we can stop using CircleCI. (cherry picked from commit ce58c043049ad4fdbc2971cfda8e168d0b3d4a55) --- src/test/scripts/check_enterprise_merge.sh | 100 --------------------- 1 file changed, 100 deletions(-) delete mode 100755 src/test/scripts/check_enterprise_merge.sh diff --git a/src/test/scripts/check_enterprise_merge.sh b/src/test/scripts/check_enterprise_merge.sh deleted file mode 100755 index 007c27d10..000000000 --- a/src/test/scripts/check_enterprise_merge.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Testing this script locally requires you to set the following environment -# variables: -# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN - -# fail if trying to reference a variable that is not set. -set -u -# exit immediately if a command fails -set -e -# Fail on pipe failures -set -o pipefail - -PR_BRANCH="${CIRCLE_BRANCH}" -ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise" - -# For echo commands "set -x" would show the message effectively twice. Once as -# part of the echo command shown by "set -x" and once because of the output of -# the echo command. We do not want "set -x" to show the echo command. We only -# want to see the actual message in the output of echo itself. This function is -# a trick to do so. Read the StackOverflow post below to understand why this -# works and what this works around. -# Source: https://superuser.com/a/1141026/242593 -shopt -s expand_aliases -alias echo='{ save_flags="$-"; set +x;} 2> /dev/null; echo_and_restore' -echo_and_restore() { - builtin echo "$*" - #shellcheck disable=SC2154 - case "$save_flags" in - (*x*) set -x - esac -} - -# Make sure that on a failing exit we show a useful message -hint_on_fail() { - exit_code=$? - if [ $exit_code != 0 ]; then - echo HINT: To solve this failure look here: https://github.com/citusdata/citus/blob/master/src/test/scripts/README.md#check-merge-to-enterprise-job - fi - exit $exit_code -} -trap hint_on_fail EXIT - - -# List executed commands. This is done so debugging this script is easier when -# it fails. It's explicitly done after git remote add so username and password -# are not shown in CI output (even though it's also filtered out by CircleCI) -set -x - -# Clone current git repo (which should be community) to a temporary working -# directory and go there -GIT_DIR_ROOT="$(git rev-parse --show-toplevel)" -TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)" -git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR" -cd "$TMP_GIT_DIR" - -# Fails in CI without this -git config user.email "citus-bot@microsoft.com" -git config user.name "citus bot" - -# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords -{ set +x ; } 2> /dev/null -git remote add enterprise "$ENTERPRISE_REMOTE" -set -x - -git remote set-url --push enterprise no-pushing - -# Fetch enterprise-master -git fetch enterprise enterprise-master - - -git checkout "enterprise/enterprise-master" - -if git merge --no-commit "origin/$PR_BRANCH"; then - echo "INFO: community PR branch could be merged into enterprise-master, so everything is good" - exit 0 -fi - -# undo partial merge -git merge --abort - -if ! git fetch enterprise "$PR_BRANCH" ; then - echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master" - exit 1 -fi - -# Show the top commit of the enterprise PR branch to make debugging easier -git log -n 1 "enterprise/$PR_BRANCH" - -# Check that this branch contains the top commit of the current community PR -# branch. If it does not it means it's not up to date with the current PR, so -# the enterprise branch should be updated. -if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then - echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch" - exit 1 -fi - -# Now check if we can merge the enterprise PR into enterprise-master without -# issues. -git merge --no-commit "enterprise/$PR_BRANCH" From a240c2c64575c53d1542c7bb815d6c0d6863a4b9 Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 15:31:42 +0300 Subject: [PATCH 05/12] Remove test-arbitrary-configs section (cherry picked from commit 344bc5b86e8a1bbac234bfbd91c36dda7d510895) --- .github/workflows/build_and_test.yml | 47 ---------------------------- 1 file changed, 47 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4e9c4601a..424b29d63 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -234,53 +234,6 @@ jobs: with: flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} codecov_token: ${{ secrets.CODECOV_TOKEN }} - test-arbitrary-configs: - name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }} - runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] - container: - image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" - options: --user root - needs: - - params - - build - strategy: - fail-fast: false - matrix: - image_name: - - ${{ needs.params.outputs.fail_test_image_name }} - pg_version: - - ${{ needs.params.outputs.pg14_version }} - - ${{ needs.params.outputs.pg15_version }} - - ${{ needs.params.outputs.pg16_version }} - parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs - steps: - - uses: actions/checkout@v3.5.0 - - uses: "./.github/actions/setup_extension" - - name: Test arbitrary configs - run: |- - # we use parallel jobs to split the tests into 6 parts and run them in parallel - # the script below extracts the tests for the current job - N=6 # Total number of jobs (see matrix.parallel) - X=${{ matrix.parallel }} # Current job number - TESTS=$(src/test/regress/citus_tests/print_test_names.py | - tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{ - split("", parts) - for (i = 1; i <= NF; i++) { - parts[i % N] = parts[i % N] $i "," - } - print substr(parts[X], 1, length(parts[X])-1) - }') - echo $TESTS - gosu circleci \ - make -C src/test/regress \ - check-arbitrary-configs parallel=4 CONFIGS=$TESTS - - uses: "./.github/actions/save_logs_and_results" - if: always() - - uses: "./.github/actions/upload_coverage" - if: always() - with: - flags: ${{ env.pg_major }}_upgrade - codecov_token: ${{ secrets.CODECOV_TOKEN }} test-pg-upgrade: name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade runs-on: ubuntu-20.04 From cb533fcb315ac91d5087384015593385ae671d6d Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 15:32:42 +0300 Subject: [PATCH 06/12] Remove upload-coverage section (cherry picked from commit ec531c4afbddace39628844975e655eaf8a24c45) --- .github/workflows/build_and_test.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 424b29d63..9be2a64f3 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -330,28 +330,6 @@ jobs: with: flags: ${{ env.pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} - upload-coverage: - if: always() - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - runs-on: ubuntu-20.04 - container: - image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }} - needs: - - params - - test-citus - - test-arbitrary-configs - - test-citus-upgrade - - test-pg-upgrade - steps: - - uses: actions/download-artifact@v3.0.1 - with: - name: "codeclimate" - path: "codeclimate" - - name: Upload coverage results to Code Climate - run: |- - cc-test-reporter sum-coverage codeclimate/*.json -o total.json - cc-test-reporter upload-coverage -i total.json ch_benchmark: name: CH Benchmark if: startsWith(github.ref, 'refs/heads/ch_benchmark/') From 0d8a3c0817676a43e9834eec648d5349b75a7e5d Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 15:34:08 +0300 Subject: [PATCH 07/12] Remove flaky test section (cherry picked from commit d69fe364f498fe2a0cf0d49af5bc715072bb3a7c) --- .github/workflows/build_and_test.yml | 68 ---------------------------- 1 file changed, 68 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9be2a64f3..a5bd96ff0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -366,71 +366,3 @@ jobs: cd ./src/test/hammerdb chmod +x run_hammerdb.sh run_hammerdb.sh citusbot_tpcc_benchmark_rg - prepare_parallelization_matrix_32: - name: Parallel 32 - if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} - needs: test-flakyness-pre - runs-on: ubuntu-20.04 - outputs: - json: ${{ steps.parallelization.outputs.json }} - steps: - - uses: actions/checkout@v3.5.0 - - uses: "./.github/actions/parallelization" - id: parallelization - with: - count: 32 - test-flakyness-pre: - name: Detect regression tests need to be ran - if: ${{ !inputs.skip_test_flakyness }}} - runs-on: ubuntu-20.04 - needs: build - outputs: - tests: ${{ steps.detect-regression-tests.outputs.tests }} - steps: - - uses: actions/checkout@v3.5.0 - with: - fetch-depth: 0 - - name: Detect regression tests need to be ran - id: detect-regression-tests - run: |- - detected_changes=$(git diff origin/main... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true)) - tests=${detected_changes} - if [ -z "$tests" ]; then - echo "No test found." - else - echo "Detected tests " $tests - fi - echo tests="$tests" >> "$GITHUB_OUTPUT" - test-flakyness: - if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} - name: Test flakyness - runs-on: ubuntu-20.04 - container: - image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} - options: --user root - env: - runs: 8 - needs: - - params - - build - - test-flakyness-pre - - prepare_parallelization_matrix_32 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} - steps: - - uses: actions/checkout@v3.5.0 - - uses: actions/download-artifact@v3.0.1 - - uses: "./.github/actions/setup_extension" - - name: Run minimal tests - run: |- - tests="${{ needs.test-flakyness-pre.outputs.tests }}" - tests_array=($tests) - for test in "${tests_array[@]}" - do - test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/") - gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line - done - shell: bash - - uses: "./.github/actions/save_logs_and_results" - if: always() From 70781c3aa1250ad72a2157ec520b255e94fa054a Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 16:01:13 +0300 Subject: [PATCH 08/12] Remove code climate coverage (cherry picked from commit 512d185cc3e78dc7b66d4ad92f6563caa1eb4d1b) --- .github/actions/upload_coverage/action.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml index 0b5f581a6..784cb2a76 100644 --- a/.github/actions/upload_coverage/action.yml +++ b/.github/actions/upload_coverage/action.yml @@ -13,15 +13,3 @@ runs: token: ${{ inputs.codecov_token }} verbose: true gcov: true - - name: Create codeclimate coverage - run: |- - lcov --directory . --capture --output-file lcov.info - lcov --remove lcov.info -o lcov.info '/usr/*' - sed "s=^SF:$PWD/=SF:=g" -i lcov.info # relative pats are required by codeclimate - mkdir -p /tmp/codeclimate - cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info - shell: bash - - uses: actions/upload-artifact@v3.1.1 - with: - path: "/tmp/codeclimate/*.json" - name: codeclimate From 3f8e0c414fe2bbdbb0a5945751b696c5abc5049a Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 26 Oct 2023 16:50:33 +0300 Subject: [PATCH 09/12] Adjust check-style (cherry picked from commit 5d7924e7c76397e5e3e18e87256e1dc61d4f59af) --- .github/workflows/build_and_test.yml | 30 +++++++++++----------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a5bd96ff0..6a0fd14b1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -61,24 +61,22 @@ jobs: fetch-depth: 0 - name: Check C Style run: citus_indent --check - - name: Check Python style - run: black --check . - - name: Check Python import order - run: isort --check . - - name: Check Python lints - run: flake8 . - name: Fix whitespace - run: ci/editorconfig.sh && git diff --exit-code + run: ci/editorconfig.sh + - name: Check if whitespace fixing changed anything, install editorconfig if it did + run: git diff --exit-code - name: Remove useless declarations - run: ci/remove_useless_declarations.sh && git diff --cached --exit-code + run: ci/remove_useless_declarations.sh + - name: Check if changed + run: git diff --cached --exit-code - name: Normalize test output - run: ci/normalize_expected.sh && git diff --exit-code + run: ci/normalize_expected.sh + - name: Check if changed + run: git diff --exit-code - name: Check for C-style comments in migration files - run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code - - name: 'Check for comment--cached ns that start with # character in spec files' - run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code - - name: Check for gitignore entries .for source files - run: ci/fix_gitignore.sh && git diff --exit-code + run: ci/disallow_c_comments_in_migrations.sh + - name: Check if changed + run: git diff --exit-code - name: Check for lengths of changelog entries run: ci/disallow_long_changelog_entries.sh - name: Check for banned C API usage @@ -87,10 +85,6 @@ jobs: run: ci/check_all_tests_are_run.sh - name: Check if all CI scripts are actually run run: ci/check_all_ci_scripts_are_run.sh - - name: Check if all GUCs are sorted alphabetically - run: ci/check_gucs_are_alphabetically_sorted.sh - - name: Check for missing downgrade scripts - run: ci/check_migration_files.sh build: needs: params name: Build for PG${{ fromJson(matrix.pg_version).major }} From c51ec26e0f16d8c618429ce1e7a36e63c45c5ab2 Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Fri, 27 Oct 2023 11:02:56 +0300 Subject: [PATCH 10/12] Update stretch repositories and install dependencies (git) (cherry picked from commit 3eb4f471f9950304e8f02d36468f80abaac5b75a) --- .github/workflows/build_and_test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6a0fd14b1..3e8847d3e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -215,6 +215,13 @@ jobs: - build steps: - uses: actions/checkout@v3.5.0 + - name: Install dependencies + if: matrix.make == 'check-failure' + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update || true + apt install git -y - uses: "./.github/actions/setup_extension" - name: Run Test run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} @@ -251,6 +258,12 @@ jobs: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} steps: + - name: Install dependencies + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update || true + apt install git -y - uses: actions/checkout@v3.5.0 - uses: "./.github/actions/setup_extension" with: @@ -289,6 +302,12 @@ jobs: - params - build steps: + - name: Install dependencies + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update + apt install git -y - uses: actions/checkout@v3.5.0 - uses: "./.github/actions/setup_extension" with: From 01e89390ec8e998877f1475f04e3a3e8a46c66f8 Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Wed, 1 Nov 2023 09:44:20 +0300 Subject: [PATCH 11/12] Remove flakiness workflow. (cherry picked from commit 9e0fee907ba9911bef6acdd590c327f51f982820) --- .github/workflows/flaky_test_debugging.yml | 79 ---------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/workflows/flaky_test_debugging.yml diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml deleted file mode 100644 index a666c1cd5..000000000 --- a/.github/workflows/flaky_test_debugging.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Flaky test debugging -run-name: Flaky test debugging - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }}) -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -on: - workflow_dispatch: - inputs: - flaky_test: - required: true - type: string - description: Test to run - flaky_test_runs_per_job: - required: false - default: 8 - type: number - description: Number of times to run the test - flaky_test_parallel_jobs: - required: false - default: 32 - type: number - description: Number of parallel jobs to run -jobs: - build: - name: Build Citus - runs-on: ubuntu-latest - container: - image: ${{ vars.build_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} - options: --user root - steps: - - uses: actions/checkout@v3.5.0 - - name: Configure, Build, and Install - run: | - echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV - ./ci/build-citus.sh - shell: bash - - uses: actions/upload-artifact@v3.1.1 - with: - name: build-${{ env.PG_MAJOR }} - path: |- - ./build-${{ env.PG_MAJOR }}/* - ./install-${{ env.PG_MAJOR }}.tar - prepare_parallelization_matrix: - name: Prepare parallelization matrix - runs-on: ubuntu-latest - outputs: - json: ${{ steps.parallelization.outputs.json }} - steps: - - uses: actions/checkout@v3.5.0 - - uses: "./.github/actions/parallelization" - id: parallelization - with: - count: ${{ inputs.flaky_test_parallel_jobs }} - test_flakyness: - name: Test flakyness - runs-on: ubuntu-latest - container: - image: ${{ vars.fail_test_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} - options: --user root - needs: - [build, prepare_parallelization_matrix] - env: - test: "${{ inputs.flaky_test }}" - runs: "${{ inputs.flaky_test_runs_per_job }}" - skip: false - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} - steps: - - uses: actions/checkout@v3.5.0 - - uses: "./.github/actions/setup_extension" - - name: Run minimal tests - run: |- - gosu circleci src/test/regress/citus_tests/run_test.py ${{ env.test }} --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line - shell: bash - - uses: "./.github/actions/save_logs_and_results" - if: always() - with: - folder: ${{ matrix.id }} From 404ca6150c97a706fccc84e33e6a93d9e7d99c9a Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Thu, 2 Nov 2023 10:49:56 +0300 Subject: [PATCH 12/12] Reflect v9.4 changes to GHA workflow --- .github/actions/setup_extension/action.yml | 1 + .github/workflows/build_and_test.yml | 103 ++++----------------- 2 files changed, 20 insertions(+), 84 deletions(-) diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml index 96b408e7e..80554e610 100644 --- a/.github/actions/setup_extension/action.yml +++ b/.github/actions/setup_extension/action.yml @@ -28,6 +28,7 @@ runs: run: |- chown -R circleci . git config --global --add safe.directory ${GITHUB_WORKSPACE} + PG_CONFIG=/usr/lib/postgresql/${{ inputs.pg_major }} >>/bin/pg_config gosu circleci ./configure --without-pg-version-check shell: bash - name: Enable core dumps diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3e8847d3e..cfc6b849e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -25,12 +25,11 @@ jobs: fail_test_image_name: "citus/failtester" pgupgrade_image_name: "citus/pgupgradetester" style_checker_image_name: "citus/stylechecker" - style_checker_tools_version: "0.8.18" - image_suffix: "-v9d71045" - pg14_version: '{ "major": "14", "full": "14.9" }' - pg15_version: '{ "major": "15", "full": "15.4" }' - pg16_version: '{ "major": "16", "full": "16.0" }' - upgrade_pg_versions: "14.9-15.4-16.0" + style_checker_tools_version: "latest" + image_suffix: "" + pg11_version: '{ "major": "11", "full": "11.9" }' + pg12_version: '{ "major": "12", "full": "12.4" }' + upgrade_pg_versions: "latest" steps: # Since GHA jobs needs at least one step we use a noop step here. - name: Set up parameters @@ -81,10 +80,6 @@ jobs: run: ci/disallow_long_changelog_entries.sh - name: Check for banned C API usage run: ci/banned.h.sh - - name: Check for tests missing in schedules - run: ci/check_all_tests_are_run.sh - - name: Check if all CI scripts are actually run - run: ci/check_all_ci_scripts_are_run.sh build: needs: params name: Build for PG${{ fromJson(matrix.pg_version).major }} @@ -96,9 +91,8 @@ jobs: image_suffix: - ${{ needs.params.outputs.image_suffix}} pg_version: - - ${{ needs.params.outputs.pg14_version }} - - ${{ needs.params.outputs.pg15_version }} - - ${{ needs.params.outputs.pg16_version }} + - ${{ needs.params.outputs.pg11_version }} + - ${{ needs.params.outputs.pg12_version }} runs-on: ubuntu-20.04 container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" @@ -127,80 +121,23 @@ jobs: image_name: - ${{ needs.params.outputs.test_image_name }} pg_version: - - ${{ needs.params.outputs.pg14_version }} - - ${{ needs.params.outputs.pg15_version }} - - ${{ needs.params.outputs.pg16_version }} + - ${{ needs.params.outputs.pg11_version }} + - ${{ needs.params.outputs.pg12_version }} make: - - check-split - check-multi - - check-multi-1 - check-multi-mx - check-vanilla - check-isolation - - check-operations + - check-worker - check-follower-cluster - - check-columnar - - check-columnar-isolation - - check-enterprise - - check-enterprise-isolation - - check-enterprise-isolation-logicalrep-1 - - check-enterprise-isolation-logicalrep-2 - - check-enterprise-isolation-logicalrep-3 + - check-multi-task-tracker-extra include: - make: check-failure - pg_version: ${{ needs.params.outputs.pg14_version }} + pg_version: ${{ needs.params.outputs.pg11_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-failure - pg_version: ${{ needs.params.outputs.pg15_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-failure - pg_version: ${{ needs.params.outputs.pg16_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-enterprise-failure - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-enterprise-failure - pg_version: ${{ needs.params.outputs.pg15_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-enterprise-failure - pg_version: ${{ needs.params.outputs.pg16_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-pytest - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-pytest - pg_version: ${{ needs.params.outputs.pg15_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-pytest - pg_version: ${{ needs.params.outputs.pg16_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: installcheck - suite: cdc - image_name: ${{ needs.params.outputs.test_image_name }} - pg_version: ${{ needs.params.outputs.pg15_version }} - - make: installcheck - suite: cdc - image_name: ${{ needs.params.outputs.test_image_name }} - pg_version: ${{ needs.params.outputs.pg16_version }} - - make: check-query-generator - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-query-generator - pg_version: ${{ needs.params.outputs.pg15_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-query-generator - pg_version: ${{ needs.params.outputs.pg16_version }} + pg_version: ${{ needs.params.outputs.pg12_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} runs-on: ubuntu-20.04 @@ -248,12 +185,8 @@ jobs: fail-fast: false matrix: include: - - old_pg_major: 14 - new_pg_major: 15 - - old_pg_major: 15 - new_pg_major: 16 - - old_pg_major: 14 - new_pg_major: 16 + - old_pg_major: 11 + new_pg_major: 12 env: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} @@ -264,6 +197,8 @@ jobs: sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list apt update || true apt install git -y + - name: Fix pg_config + run: PG_CONFIG=/usr/lib/postgresql/${{ env.new_pg_major }} >>/bin/pg_config - uses: actions/checkout@v3.5.0 - uses: "./.github/actions/setup_extension" with: @@ -293,10 +228,10 @@ jobs: flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-upgrade: - name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade + name: PG${{ fromJson(needs.params.outputs.pg11_version).major }} - check-citus-upgrade runs-on: ubuntu-20.04 container: - image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}" + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg11_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params