From b7e6eff7e07f42158a1c27107756a9389ed45e7f Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Fri, 18 Aug 2023 15:07:57 +0300 Subject: [PATCH] check-query-generator and pgupgrade --- .github/actions/install_extension/action.yml | 5 +- .../actions/save_logs_and_results/action.yml | 7 +- .github/workflows/build_and_test.yml | 223 +++++------------- 3 files changed, 65 insertions(+), 170 deletions(-) diff --git a/.github/actions/install_extension/action.yml b/.github/actions/install_extension/action.yml index f43e39b0f..010a30d10 100644 --- a/.github/actions/install_extension/action.yml +++ b/.github/actions/install_extension/action.yml @@ -1,9 +1,12 @@ name: install_extension +inputs: + pg_major: + required: false runs: using: composite steps: - name: Expose $PG_MAJOR to Github Env - run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + run: echo "PG_MAJOR=${{ inputs.pg_major || env.PG_MAJOR }}" >> $GITHUB_ENV shell: bash - uses: actions/download-artifact@v3.0.1 with: diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml index 7c7f5114c..76afb27e3 100644 --- a/.github/actions/save_logs_and_results/action.yml +++ b/.github/actions/save_logs_and_results/action.yml @@ -18,12 +18,15 @@ runs: src/test/**/tmp_check/worker.57638/log src/test/**/tmp_check/worker.57637/log src/test/**/*.diffs + src/test/**/out/ddls.sql + src/test/**/out/queries.sql src/test/**/logfile_* + /tmp/pg_upgrade_newData_logs - name: Publish regression.diffs run: |- - if [ -f "src/test/regress/regression.diffs" ]; then + if find src/test/regress -name "*.diffs" -type f -print -quit | grep -q .; then echo '```diff' >> $GITHUB_STEP_SUMMARY - cat src/test/regress/regression.diffs >> $GITHUB_STEP_SUMMARY + find src/test/regress -name "*.diffs" -exec cat {} + >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi shell: bash diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4ae768cb4..a22df87d2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,6 +26,9 @@ on: fail_test_image_name: required: false default: "citus/failtester" + pgupgrade_image_name: + required: false + default: "citus/pgupgradetester" # env: # AZURE_SP: xxxx # AZURE_SP_PASSWORD: xxxx @@ -91,8 +94,8 @@ jobs: - name: Check for missing downgrade scripts run: ci/check_migration_files.sh build: + if: ${{ false }} name: Build for PG ${{ matrix.pg_version}} - needs: [check-style, check-sql-snapshots] strategy: fail-fast: false matrix: @@ -176,8 +179,17 @@ jobs: image_name: ${{ inputs.fail_test_image_name }} - make: installcheck suite: cdc - image_name: "citus/exttester" + image_name: ${{ inputs.test_image_name }} pg_version: ${{ inputs.pg15_version }} + - make: check-query-generator + pg_version: ${{ inputs.pg14_version }} + suite: regress + image_name: ${{ inputs.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ inputs.pg15_version }} + suite: regress + image_name: ${{ inputs.fail_test_image_name }} + runs-on: ubuntu-latest container: image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ inputs.image_suffix }}" @@ -216,7 +228,8 @@ jobs: image: "${{ matrix.image_name }}:${{ matrix.pg_version }}${{ inputs.image_suffix }}" options: --user root needs: - - build + - build + - prepare_parallelization_matrix_6 strategy: fail-fast: false matrix: @@ -225,7 +238,7 @@ jobs: pg_version: - ${{ inputs.pg14_version }} - ${{ inputs.pg15_version }} - parallel: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} + parallel: ${{ fromJson(needs.prepare_parallelization_matrix_6.outputs.json) }} steps: - uses: "./.github/actions/install_extension" - name: Test arbitrary configs @@ -242,169 +255,45 @@ jobs: - uses: "./.github/actions/upload_coverage" with: flags: test_${{ env.pg_major }},upgrade - - # test-query-generator: - - # defaults: - # run: - # working-directory: "/home/circleci/project" - # runs-on: ubuntu-latest - # container: - # image: "${{ env.image }}" - # needs: - # - build-14 - # env: - # pg_major: 14 - # image: citus/failtester - # image_tag: "${{ inputs.pg14_version }}" - # steps: - # - uses: actions/checkout@v3.5.0 - # - uses: actions/download-artifact@v3.0.1 - # with: - # path: "." - # - uses: "./.github/actions/install_extension" - # with: - # pg_major: "${{ env.pg_major }}" - # - uses: "./.github/actions/configure" - # - uses: "./.github/actions/enable_core" - # - name: Run Test - # run: gosu circleci make -C src/test/regress check-query-generator - # - name: Show regressions - # run: |- - # find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" -exec cat {} + - # lines=$(find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" | wc -l) - # if [ $lines -ne 0 ]; then - # exit 1 - # fi - # if: failure() - # - name: Copy logfiles - # run: |- - # mkdir src/test/regress/tmp_citus_test/logfiles - # find src/test/regress/tmp_citus_test/ -name "logfile_*" -exec cp -t src/test/regress/tmp_citus_test/logfiles/ {} + - # if: failure() - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/tmp_citus_test/logfiles - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/ddls.sql - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/queries.sql - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/local_dist.diffs - # - uses: "./.github/actions/stack_trace" - # - uses: "./.github/actions/upload_coverage" - # with: - # flags: test_${{ env.pg_major }},querygen - # test-query-generator_1: - - # defaults: - # run: - # working-directory: "/home/circleci/project" - # runs-on: ubuntu-latest - # container: - # image: "${{ env.image }}" - # needs: - # - build-15 - # env: - # pg_major: 15 - # image: citus/failtester - # image_tag: "${{ inputs.pg15_version }}" - # steps: - # - uses: actions/checkout@v3.5.0 - # - uses: actions/download-artifact@v3.0.1 - # with: - # path: "." - # - uses: "./.github/actions/install_extension" - # with: - # pg_major: "${{ env.pg_major }}" - # - uses: "./.github/actions/configure" - # - uses: "./.github/actions/enable_core" - # - name: Run Test - # run: gosu circleci make -C src/test/regress check-query-generator - # - name: Show regressions - # run: |- - # find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" -exec cat {} + - # lines=$(find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" | wc -l) - # if [ $lines -ne 0 ]; then - # exit 1 - # fi - # if: failure() - # - name: Copy logfiles - # run: |- - # mkdir src/test/regress/tmp_citus_test/logfiles - # find src/test/regress/tmp_citus_test/ -name "logfile_*" -exec cp -t src/test/regress/tmp_citus_test/logfiles/ {} + - # if: failure() - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/tmp_citus_test/logfiles - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/ddls.sql - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/queries.sql - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: src/test/regress/citus_tests/query_generator/out/local_dist.diffs - # - uses: "./.github/actions/stack_trace" - # - uses: "./.github/actions/upload_coverage" - # with: - # flags: test_${{ env.pg_major }},querygen - # test-pg-upgrade: - - # defaults: - # run: - # working-directory: "/home/circleci/project" - # runs-on: ubuntu-latest - # container: - # image: "${{ env.image }}" - # needs: - # - build-14 - # - build-15 - # env: - # old_pg_major: 14 - # new_pg_major: 15 - # image: citus/pgupgradetester - # image_tag: "${{ inputs.upgrade_pg_versions }}" - # steps: - # - uses: actions/checkout@v3.5.0 - # - uses: actions/download-artifact@v3.0.1 - # with: - # path: "." - # - uses: "./.github/actions/install_extension" - # with: - # pg_major: "${{ env.old_pg_major }}" - # - uses: "./.github/actions/install_extension" - # with: - # pg_major: "${{ env.new_pg_major }}" - # - uses: "./.github/actions/configure" - # - uses: "./.github/actions/enable_core" - # - name: Install and test postgres upgrade - # run: |- - # gosu circleci \ - # make -C src/test/regress \ - # check-pg-upgrade \ - # old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ - # new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin - # - name: Copy pg_upgrade logs for newData dir - # run: |- - # mkdir -p /tmp/pg_upgrade_newData_logs - # if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then - # cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs - # fi - # if: failure() - # - uses: actions/upload-artifact@v3.1.1 - # with: - # path: "/tmp/pg_upgrade_newData_logs" - # - uses: "./.github/actions/save_logs_and_results" - # - uses: "./.github/actions/save_regressions" - # - uses: "./.github/actions/stack_trace" - # - uses: "./.github/actions/upload_coverage" - # with: - # flags: test_${{ env.old_pg_major }}_${{ env.new_pg_major }},upgrade + test-pg-upgrade: + runs-on: ubuntu-latest + container: + image: "${{ inputs.pgupgrade_image_name }}:${{ inputs.upgrade_pg_versions }}${{ inputs.image_suffix }}" + options: --user root + needs: + - build + env: + old_pg_major: 14 + new_pg_major: 15 + steps: + - uses: actions/checkout@v3.5.0 + - uses: actions/download-artifact@v3.0.1 + with: + path: "." + - uses: "./.github/actions/install_extension" + with: + pg_major: "${{ env.old_pg_major }}" + - uses: "./.github/actions/install_extension" + with: + pg_major: "${{ env.new_pg_major }}" + - name: Install and test postgres upgrade + run: |- + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ + new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin + - name: Copy pg_upgrade logs for newData dir + run: |- + mkdir -p /tmp/pg_upgrade_newData_logs + if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then + cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs + fi + if: failure() + - uses: "./.github/actions/save_logs_and_results" + - uses: "./.github/actions/upload_coverage" + with: + flags: test_${{ env.old_pg_major }}_${{ env.new_pg_major }},upgrade # test-citus-upgrade: # defaults: