From a748729998f631ebdec3fa92f638efed55a7683a Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 17 Dec 2020 20:14:28 +0100 Subject: [PATCH] rework ci --- .circleci/config.yml | 846 ++++++++---------- ci/README.md | 10 + ci/build-citus.sh | 47 + .../regress/expected/follower_single_node.out | 4 +- .../expected/{join.out => join_pushdown.out} | 0 .../regress/expected/multi_follower_dml.out | 2 +- .../multi_follower_select_statements.out | 6 +- src/test/regress/multi_schedule | 2 +- src/test/regress/sql/follower_single_node.sql | 4 +- .../sql/{join.sql => join_pushdown.sql} | 0 src/test/regress/sql/multi_follower_dml.sql | 2 +- .../sql/multi_follower_select_statements.sql | 6 +- 12 files changed, 431 insertions(+), 498 deletions(-) create mode 100755 ci/build-citus.sh rename src/test/regress/expected/{join.out => join_pushdown.out} (100%) rename src/test/regress/sql/{join.sql => join_pushdown.sql} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0784d90d..4f3d61059 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,47 +5,32 @@ orbs: jobs: - build-11: + build: + description: Build the citus extension + parameters: + pg_major: + description: postgres major version building citus for + type: integer + image: + description: docker image to use for the build + type: string + default: citus/extbuilder + image_tag: + description: tag to use for the docker image + type: string docker: - - image: 'citus/extbuilder:11.9' + - image: '<< parameters.image >>:<< parameters.image_tag >>' steps: - checkout - run: name: 'Configure, Build, and Install' - command: build-ext + command: | + ./ci/build-citus.sh - persist_to_workspace: root: . paths: - - build-11/* - - install-11.tar - - build-12: - docker: - - image: 'citus/extbuilder:12.4' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: build-ext - - persist_to_workspace: - root: . - paths: - - build-12/* - - install-12.tar - - build-13: - docker: - - image: 'citus/extbuilder:13.0' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: build-ext - - persist_to_workspace: - root: . - paths: - - build-13/* - - install-13.tar + - build-<< parameters.pg_major >>/* + - install-<>.tar check-style: docker: @@ -91,6 +76,7 @@ jobs: - run: name: 'Check if all CI scripts are actually run' command: ci/check_all_ci_scripts_are_run.sh + check-sql-snapshots: docker: - image: 'citus/extbuilder:latest' @@ -99,459 +85,230 @@ jobs: - run: name: 'Check Snapshots' command: ci/check_sql_snapshots.sh - test-11_check-multi: + + test-pg-upgrade: + description: Runs postgres upgrade tests + parameters: + old_pg_major: + description: 'postgres major version to use before the upgrade' + type: integer + new_pg_major: + description: 'postgres major version to upgrade to' + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/pgupgradetester + image_tag: + description: 'docker image tag to use' + type: string + default: latest docker: - - image: 'citus/exttester:11.9' + - image: '<< parameters.image >>:<< parameters.image_tag >>' working_directory: /home/circleci/project steps: - checkout - attach_workspace: at: . - run: - name: 'Install and Test (check-multi)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,multi' - - - - test-11_check-vanilla: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . + name: 'Install Extension' + command: | + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory / + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory / - run: - name: 'Install and Test (check-vanilla)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,vanilla' - - test-11_check-mx: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-mx)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,mx' - - test-11_check-worker: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-worker)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,worker' - - test-11_check-isolation: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-isolation)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,isolation' - test-11_check-follower-cluster: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . + name: 'Configure' + command: | + chown -R circleci . + gosu circleci ./configure - run: name: 'Enable core dumps' - command: 'ulimit -c unlimited' + command: | + ulimit -c unlimited - run: - name: 'Install and Test (follower-cluster)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' + name: 'Install and test postgres upgrade' + command: | + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ + new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin no_output_timeout: 2m - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' command: | mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail + - store_artifacts: + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps when: on_fail - codecov/upload: - flags: 'test_11,follower-cluster' - - store_artifacts: - path: '/tmp/core_dumps' + flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - test-11_check-failure: + test-citus-upgrade: + description: Runs citus upgrade tests + parameters: + pg_major: + description: "postgres major version" + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/citusupgradetester + image_tag: + description: 'docker image tag to use' + type: string docker: - - image: 'citus/failtester:11.9' + - image: '<< parameters.image >>:<< parameters.image_tag >>' working_directory: /home/circleci/project steps: - checkout - attach_workspace: at: . - run: - name: 'Install and Test (check-failure)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' - no_output_timeout: 2m - - test-11-12_check-pg-upgrade: - docker: - - image: 'citus/pgupgradetester:latest' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and test postgres upgrade' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 11 --new-pg-version 12' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11_12,upgrade' - - test-12-13_check-pg-upgrade: - docker: - - image: 'citus/pgupgradetester:latest' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and test postgres upgrade' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 12 --new-pg-version 13' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12_13,upgrade' - - test-12_check-multi: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-multi)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,multi' - test-12_check-vanilla: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-vanilla)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,vanilla' - - test-12_check-mx: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-mx)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,mx' - - test-12_check-isolation: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-isolation)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,isolation' - - test-12_check-worker: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-worker)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,worker' - - test-12_check-follower-cluster: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . + name: 'Configure' + command: | + chown -R circleci . + gosu circleci ./configure - run: name: 'Enable core dumps' - command: 'ulimit -c unlimited' - - run: - name: 'Install and Test (follower-cluster)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' - no_output_timeout: 2m - - run: command: | - mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_12,follower-cluster' - - store_artifacts: - path: '/tmp/core_dumps' - - test-12_check-columnar: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-columnar)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-columnar' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,columnar' - - test-12_check-columnar-isolation: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-columnar-isolation)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-columnar-isolation' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,columnar,isolation' - - test-12_check-failure: - docker: - - image: 'citus/failtester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-failure)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' - no_output_timeout: 2m - - test-11_check-citus-upgrade: - docker: - - image: 'citus/citusupgradetester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . + ulimit -c unlimited - run: name: 'Install and test citus upgrade' command: | - chown -R circleci:circleci /home/circleci - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.0.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.1.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.2.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.3.0.tar + # run make check-citus-upgrade for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg11-citus${citus_version}.tar \ + citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ + done; - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.0.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.1.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.2.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.3.0.tar - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,upgrade' - - test-13_check-multi: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-multi)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,multi' - - test-13_check-mx: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-mx)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-mx' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,mx' - - test-13_check-vanilla: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-vanilla)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,vanilla' - - test-13_check-worker: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-worker)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-worker' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,worker' - - test-13_check-isolation: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-isolation)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,isolation' - - test-13_check-follower-cluster: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Enable core dumps' - command: 'ulimit -c unlimited' - - run: - name: 'Install and Test (follower-cluster)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' + # run make check-citus-upgrade-mixed for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade-mixed \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg11-citus${citus_version}.tar \ + citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ + done; no_output_timeout: 2m - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' command: | mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail + - store_artifacts: + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps when: on_fail - codecov/upload: - flags: 'test_13,follower-cluster' + flags: 'test_<< parameters.pg_major >>,upgrade' + + test-citus: + description: Runs the common tests of citus + parameters: + pg_major: + description: "postgres major version" + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/exttester + image_tag: + description: 'docker image tag to use' + type: string + make: + description: "make target" + type: string + docker: + - image: '<< parameters.image >>:<< parameters.image_tag >>' + working_directory: /home/circleci/project + steps: + - checkout + - attach_workspace: + at: . + - run: + name: 'Install Extension' + command: | + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / + - run: + name: 'Configure' + command: | + chown -R circleci . + gosu circleci ./configure + - run: + name: 'Enable core dumps' + command: | + ulimit -c unlimited + - run: + name: 'Run Test' + command: | + gosu circleci make -C src/test/regress << parameters.make >> + no_output_timeout: 2m + - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' + command: | + mkdir -p /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail - store_artifacts: - path: '/tmp/core_dumps' - - test-13_check-columnar: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-columnar)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-columnar' - no_output_timeout: 2m + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps + when: on_fail - codecov/upload: - flags: 'test_13,columnar' - - test-13_check-columnar-isolation: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-columnar-isolation)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-columnar-isolation' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_13,columnar,isolation' - - test-13_check-failure: - docker: - - image: 'citus/failtester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-failure)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' - no_output_timeout: 2m + flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' + when: always check-merge-to-enterprise: docker: @@ -562,6 +319,7 @@ jobs: - run: command: | ci/check_enterprise_merge.sh + ch_benchmark: docker: - image: buildpack-deps:stretch @@ -576,6 +334,7 @@ jobs: sh run_hammerdb.sh citusbot_ch_benchmark_rg name: install dependencies and run ch_benchmark tests no_output_timeout: 20m + tpcc_benchmark: docker: - image: buildpack-deps:stretch @@ -591,7 +350,6 @@ jobs: name: install dependencies and run ch_benchmark tests no_output_timeout: 20m - workflows: version: 2 build_and_test: @@ -603,78 +361,196 @@ workflows: ignore: - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.* - - - build-11 - - build-12 - - build-13 + - build: + name: build-11 + pg_major: 11 + image_tag: '11.9' + - build: + name: build-12 + pg_major: 12 + image_tag: '12.4' + - build: + name: build-13 + pg_major: 13 + image_tag: '13.0' - check-style - check-sql-snapshots - - test-11_check-multi: + - test-citus: + name: 'test-11_check-multi' + pg_major: 11 + image_tag: '11.9' + make: check-multi requires: [build-11] - - test-11_check-vanilla: + - test-citus: + name: 'test-11_check-mx' + pg_major: 11 + image_tag: '11.9' + make: check-multi-mx requires: [build-11] - - test-11_check-isolation: + - test-citus: + name: 'test-11_check-vanilla' + pg_major: 11 + image_tag: '11.9' + make: check-vanilla requires: [build-11] - - test-11_check-mx: + - test-citus: + name: 'test-11_check-isolation' + pg_major: 11 + image_tag: '11.9' + make: check-isolation requires: [build-11] - - test-11_check-worker: + - test-citus: + name: 'test-11_check-worker' + pg_major: 11 + image_tag: '11.9' + make: check-worker requires: [build-11] - - test-11_check-follower-cluster: + - test-citus: + name: 'test-11_check-follower-cluster' + pg_major: 11 + image_tag: '11.9' + make: check-follower-cluster requires: [build-11] - - test-11_check-failure: + - test-citus: + name: 'test-11_check-failure' + pg_major: 11 + image: citus/failtester + image_tag: '11.9' + make: check-failure requires: [build-11] - - test-12_check-multi: + - test-citus: + name: 'test-12_check-multi' + pg_major: 12 + image_tag: '12.4' + make: check-multi requires: [build-12] - - test-12_check-vanilla: + - test-citus: + name: 'test-12_check-mx' + pg_major: 12 + image_tag: '12.4' + make: check-multi-mx requires: [build-12] - - test-12_check-isolation: + - test-citus: + name: 'test-12_check-vanilla' + pg_major: 12 + image_tag: '12.4' + make: check-vanilla requires: [build-12] - - test-12_check-mx: + - test-citus: + name: 'test-12_check-isolation' + pg_major: 12 + image_tag: '12.4' + make: check-isolation requires: [build-12] - - test-12_check-worker: + - test-citus: + name: 'test-12_check-worker' + pg_major: 12 + image_tag: '12.4' + make: check-worker requires: [build-12] - - test-12_check-follower-cluster: + - test-citus: + name: 'test-12_check-follower-cluster' + pg_major: 12 + image_tag: '12.4' + make: check-follower-cluster requires: [build-12] - - test-12_check-columnar: + - test-citus: + name: 'test-12_check-columnar' + pg_major: 12 + image_tag: '12.4' + make: check-columnar requires: [build-12] - - test-12_check-columnar-isolation: + - test-citus: + name: 'test-12_check-columnar-isolation' + pg_major: 12 + image_tag: '12.4' + make: check-columnar-isolation requires: [build-12] - - test-12_check-failure: + - test-citus: + name: 'test-12_check-failure' + pg_major: 12 + image: citus/failtester + image_tag: '12.4' + make: check-failure requires: [build-12] - - test-13_check-multi: + - test-citus: + name: 'test-13_check-multi' + pg_major: 13 + image_tag: '13.0' + make: check-multi requires: [build-13] - - test-13_check-vanilla: + - test-citus: + name: 'test-13_check-mx' + pg_major: 13 + image_tag: '13.0' + make: check-multi-mx requires: [build-13] - - test-13_check-isolation: + - test-citus: + name: 'test-13_check-vanilla' + pg_major: 13 + image_tag: '13.0' + make: check-vanilla requires: [build-13] - - test-13_check-mx: + - test-citus: + name: 'test-13_check-isolation' + pg_major: 13 + image_tag: '13.0' + make: check-isolation requires: [build-13] - - test-13_check-worker: + - test-citus: + name: 'test-13_check-worker' + pg_major: 13 + image_tag: '13.0' + make: check-worker requires: [build-13] - - test-13_check-follower-cluster: + - test-citus: + name: 'test-13_check-follower-cluster' + pg_major: 13 + image_tag: '13.0' + make: check-follower-cluster requires: [build-13] - - test-13_check-columnar: + - test-citus: + name: 'test-13_check-columnar' + pg_major: 13 + image_tag: '13.0' + make: check-columnar requires: [build-13] - - test-13_check-columnar-isolation: + - test-citus: + name: 'test-13_check-columnar-isolation' + pg_major: 13 + image_tag: '13.0' + make: check-columnar-isolation requires: [build-13] - - test-13_check-failure: + - test-citus: + name: 'test-13_check-failure' + pg_major: 13 + image: citus/failtester + image_tag: '13.0' + make: check-failure requires: [build-13] - - test-11-12_check-pg-upgrade: - requires: - - build-11 - - build-12 + - test-pg-upgrade: + name: 'test-11-12_check-pg-upgrade' + old_pg_major: 11 + new_pg_major: 12 + image_tag: latest + requires: [build-11,build-12] - - test-12-13_check-pg-upgrade: - requires: - - build-12 - - build-13 + - test-pg-upgrade: + name: 'test-12-13_check-pg-upgrade' + old_pg_major: 12 + new_pg_major: 13 + image_tag: latest + requires: [build-12,build-13] - - test-11_check-citus-upgrade: + - test-citus-upgrade: + name: test-11_check-citus-upgrade + pg_major: 11 + image_tag: '11.9' requires: [build-11] - ch_benchmark: diff --git a/ci/README.md b/ci/README.md index b382adb9a..0d1be65f2 100644 --- a/ci/README.md +++ b/ci/README.md @@ -46,6 +46,16 @@ following: requires also adding a comment before explaining why this specific use of the function is safe. +## `build-citus.sh` + +This is the script used during the build phase of the extension. Historically this script +was embedded in the docker images. This made maintenance a hassle. Now it lives in tree +with the rest of the source code. + +When this script fails you most likely have a build error on the postgres version it was +building at the time of the failure. Fix the compile error and push a new version of your +code to fix. + ## `check_enterprise_merge.sh` This check exists to make sure that we can always merge the `master` branch of diff --git a/ci/build-citus.sh b/ci/build-citus.sh new file mode 100755 index 000000000..870ca4481 --- /dev/null +++ b/ci/build-citus.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# make bash behave +set -euo pipefail +IFS=$'\n\t' + +# shellcheck disable=SC1091 +source ci/ci_helpers.sh + +# read pg major version, error if not provided +PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} + +# get codename from release file +. /etc/os-release +codename=${VERSION#*(} +codename=${codename%)*} + +# get project from argument +project="${CIRCLE_PROJECT_REPONAME}" + +# we'll do everything with absolute paths +basedir="$(pwd)" + +# get the project and clear out the git repo (reduce workspace size +rm -rf "${basedir}/.git" + +build_ext() { + pg_major="$1" + + builddir="${basedir}/build-${pg_major}" + echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2 + + # do everything in a subdirectory to avoid clutter in current directory + mkdir -p "${builddir}" && cd "${builddir}" + + CFLAGS=-Werror "${basedir}/configure" PG_CONFIG="/usr/lib/postgresql/${pg_major}/bin/pg_config" --enable-coverage + + installdir="${builddir}/install" + make -j$(nproc) && mkdir -p "${installdir}" && { make DESTDIR="${installdir}" install-all || make DESTDIR="${installdir}" install ; } + + cd "${installdir}" && find . -type f -print > "${builddir}/files.lst" + tar cvf "${basedir}/install-${pg_major}.tar" `cat ${builddir}/files.lst` + + cd "${builddir}" && rm -rf install files.lst && make clean +} + +build_ext "${PG_MAJOR}" diff --git a/src/test/regress/expected/follower_single_node.out b/src/test/regress/expected/follower_single_node.out index a4cd51561..d1da46dbd 100644 --- a/src/test/regress/expected/follower_single_node.out +++ b/src/test/regress/expected/follower_single_node.out @@ -233,7 +233,7 @@ SELECT * FROM test WHERE x = 1; ERROR: node group 0 does not have a secondary node -- add the the follower as secondary nodes and try again, the SELECT statement -- should work this time -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SET search_path TO single_node; SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary'); ?column? @@ -350,7 +350,7 @@ SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y); RESET citus.task_assignment_policy; -- Cleanup -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SET search_path TO single_node; SET client_min_messages TO WARNING; DROP SCHEMA single_node CASCADE; diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join_pushdown.out similarity index 100% rename from src/test/regress/expected/join.out rename to src/test/regress/expected/join_pushdown.out diff --git a/src/test/regress/expected/multi_follower_dml.out b/src/test/regress/expected/multi_follower_dml.out index 212312652..3cfce4acd 100644 --- a/src/test/regress/expected/multi_follower_dml.out +++ b/src/test/regress/expected/multi_follower_dml.out @@ -269,7 +269,7 @@ ERROR: writing to worker nodes is not currently allowed DETAIL: citus.use_secondary_nodes is set to 'always' SELECT * FROM citus_local_table ORDER BY a; ERROR: there is a shard placement in node group 0 but there are no nodes in that group -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port DROP TABLE the_table; DROP TABLE reference_table; DROP TABLE citus_local_table; diff --git a/src/test/regress/expected/multi_follower_select_statements.out b/src/test/regress/expected/multi_follower_select_statements.out index 73ce25895..13e09a6ca 100644 --- a/src/test/regress/expected/multi_follower_select_statements.out +++ b/src/test/regress/expected/multi_follower_select_statements.out @@ -77,7 +77,7 @@ order by s_i_id; SELECT * FROM the_table; ERROR: node group does not have a secondary node -- add the secondary nodes and try again, the SELECT statement should work this time -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SELECT 1 FROM master_add_node('localhost', :follower_worker_1_port, groupid => (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_1_port), noderole => 'secondary'); @@ -149,7 +149,7 @@ order by s_i_id; ERROR: there is a shard placement in node group but there are no nodes in that group -- now move the secondary nodes into the new cluster and see that the follower, finally -- correctly configured, can run select queries involving them -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port UPDATE pg_dist_node SET nodecluster = 'second-cluster' WHERE noderole = 'secondary'; \c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always\ -c\ citus.cluster_name=second-cluster'" SELECT * FROM the_table; @@ -160,6 +160,6 @@ SELECT * FROM the_table; (2 rows) -- clean up after ourselves -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port DROP TABLE the_table; DROP TABLE stock; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 16d60bfa5..1c030b3ec 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -106,7 +106,7 @@ test: multi_subquery_complex_reference_clause multi_subquery_window_functions mu test: sql_procedure multi_function_in_join row_types materialized_view undistribute_table test: multi_subquery_in_where_reference_clause adaptive_executor propagate_set_commands geqo # this should be run alone as it gets too many clients -test: join +test: join_pushdown test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc statement_cancel_error_message test: multi_agg_distinct multi_agg_approximate_distinct multi_limit_clause_approximate multi_outer_join_reference multi_single_relation_subquery multi_prepare_plsql set_role_in_transaction test: multi_reference_table multi_select_for_update relation_access_tracking pg13_with_ties diff --git a/src/test/regress/sql/follower_single_node.sql b/src/test/regress/sql/follower_single_node.sql index 9d0d38db1..125f45460 100644 --- a/src/test/regress/sql/follower_single_node.sql +++ b/src/test/regress/sql/follower_single_node.sql @@ -100,7 +100,7 @@ SELECT * FROM test WHERE x = 1; -- add the the follower as secondary nodes and try again, the SELECT statement -- should work this time -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SET search_path TO single_node; SELECT 1 FROM master_add_node('localhost', :follower_master_port, groupid => 0, noderole => 'secondary'); @@ -138,7 +138,7 @@ SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y); RESET citus.task_assignment_policy; -- Cleanup -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SET search_path TO single_node; SET client_min_messages TO WARNING; DROP SCHEMA single_node CASCADE; diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join_pushdown.sql similarity index 100% rename from src/test/regress/sql/join.sql rename to src/test/regress/sql/join_pushdown.sql diff --git a/src/test/regress/sql/multi_follower_dml.sql b/src/test/regress/sql/multi_follower_dml.sql index b24ff03bd..5fada2076 100644 --- a/src/test/regress/sql/multi_follower_dml.sql +++ b/src/test/regress/sql/multi_follower_dml.sql @@ -151,7 +151,7 @@ SELECT * FROM reference_table ORDER BY a; INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 3); SELECT * FROM citus_local_table ORDER BY a; -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port DROP TABLE the_table; DROP TABLE reference_table; DROP TABLE citus_local_table; diff --git a/src/test/regress/sql/multi_follower_select_statements.sql b/src/test/regress/sql/multi_follower_select_statements.sql index 436b08cda..f0e7bd404 100644 --- a/src/test/regress/sql/multi_follower_select_statements.sql +++ b/src/test/regress/sql/multi_follower_select_statements.sql @@ -54,7 +54,7 @@ SELECT * FROM the_table; -- add the secondary nodes and try again, the SELECT statement should work this time -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port SELECT 1 FROM master_add_node('localhost', :follower_worker_1_port, groupid => (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_1_port), @@ -106,12 +106,12 @@ order by s_i_id; -- now move the secondary nodes into the new cluster and see that the follower, finally -- correctly configured, can run select queries involving them -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port UPDATE pg_dist_node SET nodecluster = 'second-cluster' WHERE noderole = 'secondary'; \c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always\ -c\ citus.cluster_name=second-cluster'" SELECT * FROM the_table; -- clean up after ourselves -\c - - - :master_port +\c -reuse-previous=off regression - - :master_port DROP TABLE the_table; DROP TABLE stock;