diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 444b90c9c..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,674 +0,0 @@ -version: 2.1 -orbs: - codecov: codecov/codecov@1.1.1 - azure-cli: circleci/azure-cli@1.0.0 - -jobs: - - build: - description: Build the citus extension - parameters: - pg_major: - description: postgres major version building citus for - type: integer - image: - description: docker image to use for the build - type: string - default: citus/extbuilder - image_tag: - description: tag to use for the docker image - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - ./ci/build-citus.sh - - persist_to_workspace: - root: . - paths: - - build-<< parameters.pg_major >>/* - - install-<>.tar - - check-style: - docker: - - image: 'citus/stylechecker:latest' - steps: - - checkout - - run: - name: 'Check Style' - command: citus_indent --check - - run: - name: 'Fix whitespace' - command: ci/editorconfig.sh - - run: - name: 'Check if whitespace fixing changed anything, install editorconfig if it did' - command: git diff --exit-code - - run: - name: 'Remove useless declarations' - command: ci/remove_useless_declarations.sh - - run: - name: 'Check if changed' - command: git diff --cached --exit-code - - run: - name: 'Normalize test output' - command: ci/normalize_expected.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for C-style comments in migration files' - command: ci/disallow_c_comments_in_migrations.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for lengths of changelog entries' - command: ci/disallow_long_changelog_entries.sh - - run: - name: 'Check for banned C API usage' - command: ci/banned.h.sh - - run: - name: 'Check for tests missing in schedules' - command: ci/check_all_tests_are_run.sh - - run: - name: 'Check if all CI scripts are actually run' - command: ci/check_all_ci_scripts_are_run.sh - - check-sql-snapshots: - docker: - - image: 'citus/extbuilder:latest' - steps: - - checkout - - run: - name: 'Check Snapshots' - command: ci/check_sql_snapshots.sh - - test-pg-upgrade: - description: Runs postgres upgrade tests - parameters: - old_pg_major: - description: 'postgres major version to use before the upgrade' - type: integer - new_pg_major: - description: 'postgres major version to upgrade to' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/pgupgradetester - image_tag: - description: 'docker image tag to use' - type: string - default: 11-12-13 - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory / - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test postgres upgrade' - command: | - gosu circleci \ - make -C src/test/regress \ - check-pg-upgrade \ - old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ - new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - - test-citus-upgrade: - description: Runs citus upgrade tests - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/citusupgradetester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test citus upgrade' - command: | - # run make check-citus-upgrade for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - - # run make check-citus-upgrade-mixed for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade-mixed \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,upgrade' - - test-citus: - description: Runs the common tests of citus - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - make: - description: "make target" - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/regress << parameters.make >> - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' - when: always - - tap-test-citus: - description: Runs tap tests for citus - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - suite: - description: 'name of the tap test suite to run' - type: string - make: - description: "make target" - type: string - default: installcheck - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/<< parameters.suite >> << parameters.make >> - no_output_timeout: 2m - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save tap logs' - path: /home/circleci/project/src/test/<< parameters.suite >>/tmp_check/log - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,tap_<< parameters.suite >>_<< parameters.make >>' - when: always - - check-merge-to-enterprise: - docker: - - image: citus/extbuilder:13.0 - working_directory: /home/circleci/project - steps: - - checkout - - run: - command: | - ci/check_enterprise_merge.sh - - ch_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_ch_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - - tpcc_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_tpcc_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - -workflows: - version: 2 - build_and_test: - jobs: - - - check-merge-to-enterprise: - filters: - branches: - ignore: - - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.* - - - build: - name: build-11 - pg_major: 11 - image_tag: '11.9' - - build: - name: build-12 - pg_major: 12 - image_tag: '12.4' - - build: - name: build-13 - pg_major: 13 - image_tag: '13.0' - - - check-style - - check-sql-snapshots - - - test-citus: - name: 'test-11_check-multi' - pg_major: 11 - image_tag: '11.9' - make: check-multi - requires: [build-11] - - test-citus: - name: 'test-11_check-mx' - pg_major: 11 - image_tag: '11.9' - make: check-multi-mx - requires: [build-11] - - test-citus: - name: 'test-11_check-vanilla' - pg_major: 11 - image_tag: '11.9' - make: check-vanilla - requires: [build-11] - - test-citus: - name: 'test-11_check-isolation' - pg_major: 11 - image_tag: '11.9' - make: check-isolation - requires: [build-11] - - test-citus: - name: 'test-11_check-worker' - pg_major: 11 - image_tag: '11.9' - make: check-worker - requires: [build-11] - - test-citus: - name: 'test-11_check-operations' - pg_major: 11 - image_tag: '11.9' - make: check-operations - requires: [build-11] - - test-citus: - name: 'test-11_check-follower-cluster' - pg_major: 11 - image_tag: '11.9' - make: check-follower-cluster - requires: [build-11] - - test-citus: - name: 'test-11_check-failure' - pg_major: 11 - image: citus/failtester - image_tag: '11.9' - make: check-failure - requires: [build-11] - - - test-citus: - name: 'test-12_check-multi' - pg_major: 12 - image_tag: '12.4' - make: check-multi - requires: [build-12] - - test-citus: - name: 'test-12_check-mx' - pg_major: 12 - image_tag: '12.4' - make: check-multi-mx - requires: [build-12] - - test-citus: - name: 'test-12_check-vanilla' - pg_major: 12 - image_tag: '12.4' - make: check-vanilla - requires: [build-12] - - test-citus: - name: 'test-12_check-isolation' - pg_major: 12 - image_tag: '12.4' - make: check-isolation - requires: [build-12] - - test-citus: - name: 'test-12_check-worker' - pg_major: 12 - image_tag: '12.4' - make: check-worker - requires: [build-12] - - test-citus: - name: 'test-12_check-operations' - pg_major: 12 - image_tag: '12.4' - make: check-operations - requires: [build-12] - - test-citus: - name: 'test-12_check-follower-cluster' - pg_major: 12 - image_tag: '12.4' - make: check-follower-cluster - requires: [build-12] - - test-citus: - name: 'test-12_check-columnar' - pg_major: 12 - image_tag: '12.4' - make: check-columnar - requires: [build-12] - - test-citus: - name: 'test-12_check-columnar-isolation' - pg_major: 12 - image_tag: '12.4' - make: check-columnar-isolation - requires: [build-12] - - tap-test-citus: - name: 'test_12_tap-recovery' - pg_major: 12 - image_tag: '12.4' - suite: recovery - requires: [build-12] - - tap-test-citus: - name: 'test-12_tap-columnar-freezing' - pg_major: 12 - image_tag: '12.4' - suite: columnar_freezing - requires: [build-12] - - test-citus: - name: 'test-12_check-failure' - pg_major: 12 - image: citus/failtester - image_tag: '12.4' - make: check-failure - requires: [build-12] - - - test-citus: - name: 'test-13_check-multi' - pg_major: 13 - image_tag: '13.0' - make: check-multi - requires: [build-13] - - test-citus: - name: 'test-13_check-mx' - pg_major: 13 - image_tag: '13.0' - make: check-multi-mx - requires: [build-13] - - test-citus: - name: 'test-13_check-vanilla' - pg_major: 13 - image_tag: '13.0' - make: check-vanilla - requires: [build-13] - - test-citus: - name: 'test-13_check-isolation' - pg_major: 13 - image_tag: '13.0' - make: check-isolation - requires: [build-13] - - test-citus: - name: 'test-13_check-worker' - pg_major: 13 - image_tag: '13.0' - make: check-worker - requires: [build-13] - - test-citus: - name: 'test-13_check-operations' - pg_major: 13 - image_tag: '13.0' - make: check-operations - requires: [build-13] - - test-citus: - name: 'test-13_check-follower-cluster' - pg_major: 13 - image_tag: '13.0' - make: check-follower-cluster - requires: [build-13] - - test-citus: - name: 'test-13_check-columnar' - pg_major: 13 - image_tag: '13.0' - make: check-columnar - requires: [build-13] - - test-citus: - name: 'test-13_check-columnar-isolation' - pg_major: 13 - image_tag: '13.0' - make: check-columnar-isolation - requires: [build-13] - - tap-test-citus: - name: 'test_13_tap-recovery' - pg_major: 13 - image_tag: '13.0' - suite: recovery - requires: [build-13] - - tap-test-citus: - name: 'test-13_tap-columnar-freezing' - pg_major: 13 - image_tag: '13.0' - suite: columnar_freezing - requires: [build-13] - - test-citus: - name: 'test-13_check-failure' - pg_major: 13 - image: citus/failtester - image_tag: '13.0' - make: check-failure - requires: [build-13] - - - test-pg-upgrade: - name: 'test-11-12_check-pg-upgrade' - old_pg_major: 11 - new_pg_major: 12 - image_tag: 11-12-13 - requires: [build-11,build-12] - - - test-pg-upgrade: - name: 'test-12-13_check-pg-upgrade' - old_pg_major: 12 - new_pg_major: 13 - image_tag: 11-12-13 - requires: [build-12,build-13] - - - test-citus-upgrade: - name: test-11_check-citus-upgrade - pg_major: 11 - image_tag: '11.9' - requires: [build-11] - - - ch_benchmark: - requires: [build-13] - filters: - branches: - only: - - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix - - tpcc_benchmark: - requires: [build-13] - filters: - branches: - only: - - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix diff --git a/.github/actions/parallelization/action.yml b/.github/actions/parallelization/action.yml new file mode 100644 index 000000000..1f7d00202 --- /dev/null +++ b/.github/actions/parallelization/action.yml @@ -0,0 +1,23 @@ +name: 'Parallelization matrix' +inputs: + count: + required: false + default: 32 +outputs: + json: + value: ${{ steps.generate_matrix.outputs.json }} +runs: + using: "composite" + steps: + - name: Generate parallelization matrix + id: generate_matrix + shell: bash + run: |- + json_array="{\"include\": [" + for ((i = 1; i <= ${{ inputs.count }}; i++)); do + json_array+="{\"id\":\"$i\"}," + done + json_array=${json_array%,} + json_array+=" ]}" + echo "json=$json_array" >> "$GITHUB_OUTPUT" + echo "json=$json_array" diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml new file mode 100644 index 000000000..0f238835d --- /dev/null +++ b/.github/actions/save_logs_and_results/action.yml @@ -0,0 +1,38 @@ +name: save_logs_and_results +inputs: + folder: + required: false + default: "log" +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.1 + name: Upload logs + with: + name: ${{ inputs.folder }} + if-no-files-found: ignore + path: | + src/test/**/proxy.output + src/test/**/results/ + src/test/**/tmp_check/master/log + src/test/**/tmp_check/worker.57638/log + src/test/**/tmp_check/worker.57637/log + src/test/**/*.diffs + src/test/**/out/ddls.sql + src/test/**/out/queries.sql + src/test/**/logfile_* + /tmp/pg_upgrade_newData_logs + - name: Publish regression.diffs + run: |- + diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)" + if ! [ -z "$diffs" ]; then + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo -E "$diffs" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo -E $diffs + fi + shell: bash + - name: Print stack traces + run: "./ci/print_stack_trace.sh" + if: failure() + shell: bash diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml new file mode 100644 index 000000000..96b408e7e --- /dev/null +++ b/.github/actions/setup_extension/action.yml @@ -0,0 +1,35 @@ +name: setup_extension +inputs: + pg_major: + required: false + skip_installation: + required: false + default: false + type: boolean +runs: + using: composite + steps: + - name: Expose $PG_MAJOR to Github Env + run: |- + if [ -z "${{ inputs.pg_major }}" ]; then + echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + else + echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/download-artifact@v3.0.1 + with: + name: build-${{ env.PG_MAJOR }} + - name: Install Extension + if: ${{ inputs.skip_installation == 'false' }} + run: tar xfv "install-$PG_MAJOR.tar" --directory / + shell: bash + - name: Configure + run: |- + chown -R circleci . + git config --global --add safe.directory ${GITHUB_WORKSPACE} + gosu circleci ./configure --without-pg-version-check + shell: bash + - name: Enable core dumps + run: ulimit -c unlimited + shell: bash diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml new file mode 100644 index 000000000..784cb2a76 --- /dev/null +++ b/.github/actions/upload_coverage/action.yml @@ -0,0 +1,15 @@ +name: coverage +inputs: + flags: + required: false + codecov_token: + required: true +runs: + using: composite + steps: + - uses: codecov/codecov-action@v3 + with: + flags: ${{ inputs.flags }} + token: ${{ inputs.codecov_token }} + verbose: true + gcov: true diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 000000000..36ef779e1 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,350 @@ +name: Build & Test +run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +on: + workflow_dispatch: + inputs: + skip_test_flakyness: + required: false + default: false + type: boolean + pull_request: + types: [opened, reopened,synchronize] +jobs: + # Since GHA does not interpolate env varibles in matrix context, we need to + # define them in a separate job and use them in other jobs. + params: + runs-on: ubuntu-latest + name: Initialize parameters + outputs: + build_image_name: "citus/extbuilder" + test_image_name: "citus/exttester" + citusupgrade_image_name: "citus/citusupgradetester" + fail_test_image_name: "citus/failtester" + pgupgrade_image_name: "citus/pgupgradetester" + style_checker_image_name: "citus/stylechecker" + style_checker_tools_version: "latest" + image_suffix: "" + pg11_version: '{ "major": "11", "full": "11.9" }' + pg12_version: '{ "major": "12", "full": "12.4" }' + pg13_version: '{ "major": "13", "full": "13.0" }' + upgrade_pg_versions: "11-12-13" + steps: + # Since GHA jobs needs at least one step we use a noop step here. + - name: Set up parameters + run: echo 'noop' + check-sql-snapshots: + needs: params + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.build_image_name }}:latest + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + ci/check_sql_snapshots.sh + check-style: + needs: params + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }} + steps: + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Check C Style + run: citus_indent --check + - name: Fix whitespace + run: ci/editorconfig.sh + - name: Check if whitespace fixing changed anything, install editorconfig if it did + run: git diff --exit-code + - name: Remove useless declarations + run: ci/remove_useless_declarations.sh + - name: Check if changed + run: git diff --cached --exit-code + - name: Normalize test output + run: ci/normalize_expected.sh + - name: Check if changed + run: git diff --exit-code + - name: Check for C-style comments in migration files + run: ci/disallow_c_comments_in_migrations.sh + - name: Check if changed + run: git diff --exit-code + - name: Check for lengths of changelog entries + run: ci/disallow_long_changelog_entries.sh + - name: Check for banned C API usage + run: ci/banned.h.sh + - name: Check for tests missing in schedules + run: ci/check_all_tests_are_run.sh + - name: Check if all CI scripts are actually run + run: ci/check_all_ci_scripts_are_run.sh + build: + needs: params + name: Build for PG${{ fromJson(matrix.pg_version).major }} + strategy: + fail-fast: false + matrix: + image_name: + - ${{ needs.params.outputs.build_image_name }} + image_suffix: + - ${{ needs.params.outputs.image_suffix}} + pg_version: + - ${{ needs.params.outputs.pg11_version }} + - ${{ needs.params.outputs.pg12_version }} + - ${{ needs.params.outputs.pg13_version }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Expose $PG_MAJOR to Github Env + run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + shell: bash + - name: Build + run: "./ci/build-citus.sh" + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + test-citus: + name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }} + strategy: + fail-fast: false + matrix: + suite: + - regress + image_name: + - ${{ needs.params.outputs.test_image_name }} + pg_version: + - ${{ needs.params.outputs.pg11_version }} + - ${{ needs.params.outputs.pg12_version }} + - ${{ needs.params.outputs.pg13_version }} + make: + - check-multi + - check-multi-mx + - check-vanilla + - check-isolation + - check-worker + - check-operations + - check-follower-cluster + - check-columnar + - check-columnar-isolation + include: + - make: check-failure + pg_version: ${{ needs.params.outputs.pg11_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg12_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg13_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: installcheck + suite: recovery + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg12_version }} + - make: installcheck + suite: recovery + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg13_version }} + - make: installcheck + suite: columnar_freezing + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg12_version }} + - make: installcheck + suite: columnar_freezing + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg13_version }} + exclude: + - make: check-columnar + pg_version: ${{ needs.params.outputs.pg11_version }} + - make: check-columnar-isolation + pg_version: ${{ needs.params.outputs.pg11_version }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root --dns=8.8.8.8 + # Due to Github creates a default network for each job, we need to use + # --dns= to have similar DNS settings as our other CI systems or local + # machines. Otherwise, we may see different results. + needs: + - params + - build + steps: + - uses: actions/checkout@v3.5.0 + - name: Install dependencies + if: matrix.make == 'check-failure' + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update || true + apt install git -y + - uses: "./.github/actions/setup_extension" + - name: Run Test + run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} + timeout-minutes: 20 + - uses: "./.github/actions/save_logs_and_results" + if: always() + with: + folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }} + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-pg-upgrade: + name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}" + options: --user root + needs: + - params + - build + strategy: + fail-fast: false + matrix: + include: + - old_pg_major: 11 + new_pg_major: 12 + - old_pg_major: 12 + new_pg_major: 13 + env: + old_pg_major: ${{ matrix.old_pg_major }} + new_pg_major: ${{ matrix.new_pg_major }} + steps: + - name: Install dependencies + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update || true + apt install git -y + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.old_pg_major }}" + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.new_pg_major }}" + - name: Install and test postgres upgrade + run: |- + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ + new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin + - name: Copy pg_upgrade logs for newData dir + run: |- + mkdir -p /tmp/pg_upgrade_newData_logs + if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then + cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs + fi + if: failure() + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-citus-upgrade: + name: PG${{ fromJson(needs.params.outputs.pg11_version).major }} - check-citus-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg11_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root + needs: + - params + - build + steps: + - name: Install dependencies + run: |- + # update stretch repositories + sed -i -e 's/deb.debian.org/archive.debian.org/g' -e 's|security.debian.org|archive.debian.org/|g' -e '/stretch-updates/d' /etc/apt/sources.list + apt update + apt install git -y + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + skip_installation: true + - name: Install and test citus upgrade + run: |- + # run make check-citus-upgrade for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-old-version=${citus_version} \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + # run make check-citus-upgrade-mixed for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade-mixed \ + citus-old-version=${citus_version} \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + ch_benchmark: + name: CH Benchmark + if: startsWith(github.ref, 'refs/heads/ch_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run ch_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_ch_benchmark_rg + tpcc_benchmark: + name: TPCC Benchmark + if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run tpcc_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_tpcc_benchmark_rg diff --git a/ci/build-citus.sh b/ci/build-citus.sh index 870ca4481..a185d8452 100755 --- a/ci/build-citus.sh +++ b/ci/build-citus.sh @@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} codename=${VERSION#*(} codename=${codename%)*} -# get project from argument -project="${CIRCLE_PROJECT_REPONAME}" - # we'll do everything with absolute paths basedir="$(pwd)" @@ -28,7 +25,7 @@ build_ext() { pg_major="$1" builddir="${basedir}/build-${pg_major}" - echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2 + echo "Beginning build for PostgreSQL ${pg_major}..." >&2 # do everything in a subdirectory to avoid clutter in current directory mkdir -p "${builddir}" && cd "${builddir}" diff --git a/ci/check_all_ci_scripts_are_run.sh b/ci/check_all_ci_scripts_are_run.sh index 0b7abb3e3..a0b4484ee 100755 --- a/ci/check_all_ci_scripts_are_run.sh +++ b/ci/check_all_ci_scripts_are_run.sh @@ -11,11 +11,11 @@ source ci/ci_helpers.sh ci_scripts=$( find ci/ -iname "*.sh" | sed -E 's#^ci/##g' | - grep -v -E '^(ci_helpers.sh|fix_style.sh)$' + grep -v -E '^(ci_helpers.sh|fix_style.sh|check_enterprise_merge.sh)$' ) for script in $ci_scripts; do - if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then - echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml" + if ! grep "\\bci/$script\\b" -r .github > /dev/null; then + echo "ERROR: CI script with name \"$script\" is not actually used in .github folder" exit 1 fi if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then