diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 987657cc0..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,544 +0,0 @@ -version: 2.1 -orbs: - codecov: codecov/codecov@1.1.1 - azure-cli: circleci/azure-cli@1.0.0 - -jobs: - - build: - description: Build the citus extension - parameters: - pg_major: - description: postgres major version building citus for - type: integer - image: - description: docker image to use for the build - type: string - default: citus/extbuilder - image_tag: - description: tag to use for the docker image - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - ./ci/build-citus.sh - - persist_to_workspace: - root: . - paths: - - build-<< parameters.pg_major >>/* - - install-<>.tar - - check-style: - docker: - - image: 'citus/stylechecker:latest' - steps: - - checkout - - run: - name: 'Check Style' - command: citus_indent --check - - run: - name: 'Fix whitespace' - command: ci/editorconfig.sh - - run: - name: 'Check if whitespace fixing changed anything, install editorconfig if it did' - command: git diff --exit-code - - run: - name: 'Remove useless declarations' - command: ci/remove_useless_declarations.sh - - run: - name: 'Check if changed' - command: git diff --cached --exit-code - - run: - name: 'Normalize test output' - command: ci/normalize_expected.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for C-style comments in migration files' - command: ci/disallow_c_comments_in_migrations.sh - - run: - name: 'Check if changed' - command: git diff --exit-code - - run: - name: 'Check for lengths of changelog entries' - command: ci/disallow_long_changelog_entries.sh - - run: - name: 'Check for banned C API usage' - command: ci/banned.h.sh - - run: - name: 'Check for tests missing in schedules' - command: ci/check_all_tests_are_run.sh - - run: - name: 'Check if all CI scripts are actually run' - command: ci/check_all_ci_scripts_are_run.sh - - check-sql-snapshots: - docker: - - image: 'citus/extbuilder:latest' - steps: - - checkout - - run: - name: 'Check Snapshots' - command: ci/check_sql_snapshots.sh - - test-pg-upgrade: - description: Runs postgres upgrade tests - parameters: - old_pg_major: - description: 'postgres major version to use before the upgrade' - type: integer - new_pg_major: - description: 'postgres major version to upgrade to' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/pgupgradetester - image_tag: - description: 'docker image tag to use' - type: string - default: latest - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory / - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test postgres upgrade' - command: | - gosu circleci \ - make -C src/test/regress \ - check-pg-upgrade \ - old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ - new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - - test-citus-upgrade: - description: Runs citus upgrade tests - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/citusupgradetester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Install and test citus upgrade' - command: | - # run make check-citus-upgrade for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - - # run make check-citus-upgrade-mixed for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade-mixed \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg11-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,upgrade' - - test-citus: - description: Runs the common tests of citus - parameters: - pg_major: - description: "postgres major version" - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - make: - description: "make target" - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/regress << parameters.make >> - no_output_timeout: 2m - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - run: - name: 'Copy coredumps' - command: | - mkdir -p /tmp/core_dumps - if ls core.* 1> /dev/null 2>&1; then - cp core.* /tmp/core_dumps - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - when: on_fail - - store_artifacts: - name: 'Save core dumps' - path: /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' - when: always - - check-merge-to-enterprise: - docker: - - image: citus/extbuilder:13.0 - working_directory: /home/circleci/project - steps: - - checkout - - run: - command: | - ci/check_enterprise_merge.sh - - ch_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_ch_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - - tpcc_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_tpcc_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - -workflows: - version: 2 - build_and_test: - jobs: - - - check-merge-to-enterprise: - filters: - branches: - ignore: - - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.* - - - build: - name: build-11 - pg_major: 11 - image_tag: '11.9' - - build: - name: build-12 - pg_major: 12 - image_tag: '12.4' - - build: - name: build-13 - pg_major: 13 - image_tag: '13.0' - - - check-style - - check-sql-snapshots - - - test-citus: - name: 'test-11_check-multi' - pg_major: 11 - image_tag: '11.9' - make: check-multi - requires: [build-11] - - test-citus: - name: 'test-11_check-mx' - pg_major: 11 - image_tag: '11.9' - make: check-multi-mx - requires: [build-11] - - test-citus: - name: 'test-11_check-vanilla' - pg_major: 11 - image_tag: '11.9' - make: check-vanilla - requires: [build-11] - - test-citus: - name: 'test-11_check-isolation' - pg_major: 11 - image_tag: '11.9' - make: check-isolation - requires: [build-11] - - test-citus: - name: 'test-11_check-worker' - pg_major: 11 - image_tag: '11.9' - make: check-worker - requires: [build-11] - - test-citus: - name: 'test-11_check-follower-cluster' - pg_major: 11 - image_tag: '11.9' - make: check-follower-cluster - requires: [build-11] - - test-citus: - name: 'test-11_check-failure' - pg_major: 11 - image: citus/failtester - image_tag: '11.9' - make: check-failure - requires: [build-11] - - - test-citus: - name: 'test-12_check-multi' - pg_major: 12 - image_tag: '12.4' - make: check-multi - requires: [build-12] - - test-citus: - name: 'test-12_check-mx' - pg_major: 12 - image_tag: '12.4' - make: check-multi-mx - requires: [build-12] - - test-citus: - name: 'test-12_check-vanilla' - pg_major: 12 - image_tag: '12.4' - make: check-vanilla - requires: [build-12] - - test-citus: - name: 'test-12_check-isolation' - pg_major: 12 - image_tag: '12.4' - make: check-isolation - requires: [build-12] - - test-citus: - name: 'test-12_check-worker' - pg_major: 12 - image_tag: '12.4' - make: check-worker - requires: [build-12] - - test-citus: - name: 'test-12_check-follower-cluster' - pg_major: 12 - image_tag: '12.4' - make: check-follower-cluster - requires: [build-12] - - test-citus: - name: 'test-12_check-failure' - pg_major: 12 - image: citus/failtester - image_tag: '12.4' - make: check-failure - requires: [build-12] - - - test-citus: - name: 'test-13_check-multi' - pg_major: 13 - image_tag: '13.0' - make: check-multi - requires: [build-13] - - test-citus: - name: 'test-13_check-mx' - pg_major: 13 - image_tag: '13.0' - make: check-multi-mx - requires: [build-13] - - test-citus: - name: 'test-13_check-vanilla' - pg_major: 13 - image_tag: '13.0' - make: check-vanilla - requires: [build-13] - - test-citus: - name: 'test-13_check-isolation' - pg_major: 13 - image_tag: '13.0' - make: check-isolation - requires: [build-13] - - test-citus: - name: 'test-13_check-worker' - pg_major: 13 - image_tag: '13.0' - make: check-worker - requires: [build-13] - - test-citus: - name: 'test-13_check-follower-cluster' - pg_major: 13 - image_tag: '13.0' - make: check-follower-cluster - requires: [build-13] - - - test-citus: - name: 'test-13_check-failure' - pg_major: 13 - image: citus/failtester - image_tag: '13.0' - make: check-failure - requires: [build-13] - - - test-pg-upgrade: - name: 'test-11-12_check-pg-upgrade' - old_pg_major: 11 - new_pg_major: 12 - image_tag: latest - requires: [build-11,build-12] - - - test-pg-upgrade: - name: 'test-12-13_check-pg-upgrade' - old_pg_major: 12 - new_pg_major: 13 - image_tag: latest - requires: [build-12,build-13] - - - test-citus-upgrade: - name: test-11_check-citus-upgrade - pg_major: 11 - image_tag: '11.9' - requires: [build-11] - - - ch_benchmark: - requires: [build-13] - filters: - branches: - only: - - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix - - tpcc_benchmark: - requires: [build-13] - filters: - branches: - only: - - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix diff --git a/ci/check_all_ci_scripts_are_run.sh b/ci/check_all_ci_scripts_are_run.sh index 0b7abb3e3..a0b4484ee 100755 --- a/ci/check_all_ci_scripts_are_run.sh +++ b/ci/check_all_ci_scripts_are_run.sh @@ -11,11 +11,11 @@ source ci/ci_helpers.sh ci_scripts=$( find ci/ -iname "*.sh" | sed -E 's#^ci/##g' | - grep -v -E '^(ci_helpers.sh|fix_style.sh)$' + grep -v -E '^(ci_helpers.sh|fix_style.sh|check_enterprise_merge.sh)$' ) for script in $ci_scripts; do - if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then - echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml" + if ! grep "\\bci/$script\\b" -r .github > /dev/null; then + echo "ERROR: CI script with name \"$script\" is not actually used in .github folder" exit 1 fi if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then