diff --git a/.circleci/config.yml b/.circleci/config.yml index 54639aa80..c6385ded7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,20 +1,37 @@ version: 2.1 orbs: - codecov: codecov/codecov@1.1.0 + codecov: codecov/codecov@1.1.1 azure-cli: circleci/azure-cli@1.0.0 jobs: + build: + description: Build the citus extension + parameters: + pg_major: + description: postgres major version building citus for + type: integer + image: + description: docker image to use for the build + type: string + default: citus/extbuilder + image_tag: + description: tag to use for the docker image + type: string docker: - - image: 'citus/extbuilder:latest' + - image: '<< parameters.image >>:<< parameters.image_tag >>' steps: - checkout - run: name: 'Configure, Build, and Install' - command: build-ext + command: | + ./ci/build-citus.sh - persist_to_workspace: root: . - paths: [.] + paths: + - build-<< parameters.pg_major >>/* + - install-<>.tar + check-style: docker: - image: 'citus/stylechecker:latest' @@ -53,6 +70,7 @@ jobs: - run: name: 'Check for banned C API usage' command: ci/banned.h.sh + check-sql-snapshots: docker: - image: 'citus/extbuilder:latest' @@ -60,199 +78,242 @@ jobs: - checkout - run: name: 'Check Snapshots' - command: "./configure && make check-sql-snapshots -C src/backend/distributed" - test-11_check-multi: - docker: - - image: 'citus/exttester-11:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-multi)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,multi' + command: ci/check_sql_snapshots.sh - test-11_check-tt-van-mx: + test-pg-upgrade: + description: Runs postgres upgrade tests + parameters: + old_pg_major: + description: 'postgres major version to use before the upgrade' + type: integer + new_pg_major: + description: 'postgres major version to upgrade to' + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/pgupgradetester + image_tag: + description: 'docker image tag to use' + type: string + default: latest docker: - - image: 'citus/exttester-11:latest' + - image: '<< parameters.image >>:<< parameters.image_tag >>' working_directory: /home/circleci/project steps: + - checkout - attach_workspace: at: . - run: - name: 'Install and Test (check-tt-van-mx)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,tracker,vanilla,mx' - test-11_check-iso-work-fol: - docker: - - image: 'citus/exttester-11:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . + name: 'Install Extension' + command: | + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.old_pg_major >>.tar" --directory / + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.new_pg_major >>.tar" --directory / - run: - name: 'Install and Test (check-iso-work-fol)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation check-worker' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_11,isolation,worker' - test-11_check-fol: - docker: - - image: 'citus/exttester-11:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . + name: 'Configure' + command: | + chown -R circleci . + PG_CONFIG=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin/pg_config gosu circleci ./configure - run: name: 'Enable core dumps' - command: 'ulimit -c unlimited' - - run: - name: 'Install and Test (fol)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' - no_output_timeout: 2m - - run: command: | - mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps - when: on_fail - - codecov/upload: - flags: 'test_11,follower' - - store_artifacts: - path: '/tmp/core_dumps' - test-11_check-failure: - docker: - - image: 'citus/failtester-11:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-failure)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' - no_output_timeout: 2m - - test-11-12_check-pg-upgrade: - docker: - - image: 'citus/pgupgradetester:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . + ulimit -c unlimited - run: name: 'Install and test postgres upgrade' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 11 --new-pg-version 12' - no_output_timeout: 2m - - test-12_check-multi: - docker: - - image: 'citus/exttester-12:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-multi)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,multi' - test-12_check-tt-van-mx: - docker: - - image: 'citus/exttester-12:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-tt-van-mx)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,tracker,vanilla,mx' - test-12_check-iso-work-fol: - docker: - - image: 'citus/exttester-12:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Install and Test (check-iso-work-fol)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation check-worker' - no_output_timeout: 2m - - codecov/upload: - flags: 'test_12,isolation,worker' - test-12_check-fol: - docker: - - image: 'citus/exttester-12:latest' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: 'Enable core dumps' - command: 'ulimit -c unlimited' - - run: - name: 'Install and Test (fol)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' + command: | + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ + new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin no_output_timeout: 2m - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' command: | mkdir -p /tmp/core_dumps - cp core.* /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail + - store_artifacts: + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps when: on_fail - codecov/upload: - flags: 'test_12,follower' - - store_artifacts: - path: '/tmp/core_dumps' + flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - test-12_check-failure: + test-citus-upgrade: + description: Runs citus upgrade tests + parameters: + pg_major: + description: "postgres major version" + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/citusupgradetester + image_tag: + description: 'docker image tag to use' + type: string docker: - - image: 'citus/failtester-12:latest' + - image: '<< parameters.image >>:<< parameters.image_tag >>' working_directory: /home/circleci/project steps: + - checkout - attach_workspace: at: . - run: - name: 'Install and Test (check-failure)' - command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' - no_output_timeout: 2m - - test-11_check-citus-upgrade: - docker: - - image: 'citus/citusupgradetester-11:latest' - working_directory: /home/circleci/project - steps: - - {attach_workspace: {at: .}} + name: 'Configure' + command: | + chown -R circleci . + gosu circleci ./configure + - run: + name: 'Enable core dumps' + command: | + ulimit -c unlimited - run: name: 'Install and test citus upgrade' command: | - chown -R circleci:circleci /home/circleci - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.0.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.1.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.2.0.tar - install-and-test-ext --target check-citus-upgrade --citus-pre-tar /install-pg11-citusv8.3.0.tar + # run make check-citus-upgrade for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg11-citus${citus_version}.tar \ + citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ + done; - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.0.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.1.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.2.0.tar - install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.3.0.tar + # run make check-citus-upgrade-mixed for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade-mixed \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg11-citus${citus_version}.tar \ + citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ + done; no_output_timeout: 2m + - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' + command: | + mkdir -p /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail + - store_artifacts: + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps + when: on_fail + - codecov/upload: + flags: 'test_<< parameters.pg_major >>,upgrade' + + test-citus: + description: Runs the common tests of citus + parameters: + pg_major: + description: "postgres major version" + type: integer + image: + description: 'docker image to use as for the tests' + type: string + default: citus/exttester + image_tag: + description: 'docker image tag to use' + type: string + make: + description: "make target" + type: string + docker: + - image: '<< parameters.image >>:<< parameters.image_tag >>' + working_directory: /home/circleci/project + steps: + - checkout + - attach_workspace: + at: . + - run: + name: 'Install Extension' + command: | + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / + - run: + name: 'Configure' + command: | + chown -R circleci . + gosu circleci ./configure + - run: + name: 'Enable core dumps' + command: | + ulimit -c unlimited + - run: + name: 'Run Test' + command: | + gosu circleci make -C src/test/regress << parameters.make >> + no_output_timeout: 2m + - run: + name: 'Regressions' + command: | + if [ -f "src/test/regress/regression.diffs" ]; then + cat src/test/regress/regression.diffs + exit 1 + fi + when: on_fail + - run: + name: 'Copy coredumps' + command: | + mkdir -p /tmp/core_dumps + if ls core.* 1> /dev/null 2>&1; then + cp core.* /tmp/core_dumps + fi + when: on_fail + - store_artifacts: + name: 'Save regressions' + path: src/test/regress/regression.diffs + when: on_fail + - store_artifacts: + name: 'Save core dumps' + path: /tmp/core_dumps + when: on_fail + - codecov/upload: + flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' + when: always check-merge-to-enterprise: docker: - - image: buildpack-deps:stretch + - image: citus/extbuilder:13.0 working_directory: /home/circleci/project steps: - checkout - run: command: | - ./src/test/scripts/check_enterprise_merge.sh + ci/check_enterprise_merge.sh + ch_benchmark: docker: - image: buildpack-deps:stretch @@ -267,6 +328,7 @@ jobs: sh run_hammerdb.sh citusbot_ch_benchmark_rg name: install dependencies and run ch_benchmark tests no_output_timeout: 20m + tpcc_benchmark: docker: - image: buildpack-deps:stretch @@ -282,57 +344,150 @@ jobs: name: install dependencies and run ch_benchmark tests no_output_timeout: 20m - workflows: version: 2 build_and_test: jobs: + - check-merge-to-enterprise: filters: branches: ignore: - /release-[0-9]+\.[0-9]+.*/ # match with releaseX.Y.* - - build + - build: + name: build-11 + pg_major: 11 + image_tag: '11.9' + - build: + name: build-12 + pg_major: 12 + image_tag: '12.4' + - check-style - check-sql-snapshots - - test-11_check-multi: - requires: [build] - - test-11_check-tt-van-mx: - requires: [build] - - test-11_check-iso-work-fol: - requires: [build] - - test-11_check-fol: - requires: [build] - - test-11_check-failure: - requires: [build] + - test-citus: + name: 'test-11_check-multi' + pg_major: 11 + image_tag: '11.9' + make: check-multi + requires: [build-11] + - test-citus: + name: 'test-11_check-task-tracker' + pg_major: 11 + image_tag: '11.9' + make: check-multi-task-tracker-extra + requires: [build-11] + - test-citus: + name: 'test-11_check-mx' + pg_major: 11 + image_tag: '11.9' + make: check-multi-mx + requires: [build-11] + - test-citus: + name: 'test-11_check-vanilla' + pg_major: 11 + image_tag: '11.9' + make: check-vanilla + requires: [build-11] + - test-citus: + name: 'test-11_check-isolation' + pg_major: 11 + image_tag: '11.9' + make: check-isolation + requires: [build-11] + - test-citus: + name: 'test-11_check-worker' + pg_major: 11 + image_tag: '11.9' + make: check-worker + requires: [build-11] + - test-citus: + name: 'test-11_check-follower-cluster' + pg_major: 11 + image_tag: '11.9' + make: check-follower-cluster + requires: [build-11] + - test-citus: + name: 'test-11_check-failure' + pg_major: 11 + image: citus/failtester + image_tag: '11.9' + make: check-failure + requires: [build-11] - - test-12_check-multi: - requires: [build] - - test-12_check-tt-van-mx: - requires: [build] - - test-12_check-iso-work-fol: - requires: [build] - - test-12_check-fol: - requires: [build] - - test-12_check-failure: - requires: [build] + - test-citus: + name: 'test-12_check-multi' + pg_major: 12 + image_tag: '12.4' + make: check-multi + requires: [build-12] + - test-citus: + name: 'test-12_check-task-tracker' + pg_major: 12 + image_tag: '12.4' + make: check-multi-task-tracker-extra + requires: [build-12] + - test-citus: + name: 'test-12_check-mx' + pg_major: 12 + image_tag: '12.4' + make: check-multi-mx + requires: [build-12] + - test-citus: + name: 'test-12_check-vanilla' + pg_major: 12 + image_tag: '12.4' + make: check-vanilla + requires: [build-12] + - test-citus: + name: 'test-12_check-isolation' + pg_major: 12 + image_tag: '12.4' + make: check-isolation + requires: [build-12] + - test-citus: + name: 'test-12_check-worker' + pg_major: 12 + image_tag: '12.4' + make: check-worker + requires: [build-12] + - test-citus: + name: 'test-12_check-follower-cluster' + pg_major: 12 + image_tag: '12.4' + make: check-follower-cluster + requires: [build-12] + - test-citus: + name: 'test-12_check-failure' + pg_major: 12 + image: citus/failtester + image_tag: '12.4' + make: check-failure + requires: [build-12] - - test-11-12_check-pg-upgrade: - requires: [build] + - test-pg-upgrade: + name: 'test-11-12_check-pg-upgrade' + old_pg_major: 11 + new_pg_major: 12 + image_tag: latest + requires: [build-11,build-12] - - test-11_check-citus-upgrade: - requires: [build] + - test-citus-upgrade: + name: test-11_check-citus-upgrade + pg_major: 11 + image_tag: '11.9' + requires: [build-11] - ch_benchmark: - requires: [build] + requires: [build-12] filters: branches: only: - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix - tpcc_benchmark: - requires: [build] + requires: [build-12] filters: branches: only: diff --git a/ci/build-citus.sh b/ci/build-citus.sh new file mode 100755 index 000000000..870ca4481 --- /dev/null +++ b/ci/build-citus.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# make bash behave +set -euo pipefail +IFS=$'\n\t' + +# shellcheck disable=SC1091 +source ci/ci_helpers.sh + +# read pg major version, error if not provided +PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} + +# get codename from release file +. /etc/os-release +codename=${VERSION#*(} +codename=${codename%)*} + +# get project from argument +project="${CIRCLE_PROJECT_REPONAME}" + +# we'll do everything with absolute paths +basedir="$(pwd)" + +# get the project and clear out the git repo (reduce workspace size +rm -rf "${basedir}/.git" + +build_ext() { + pg_major="$1" + + builddir="${basedir}/build-${pg_major}" + echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2 + + # do everything in a subdirectory to avoid clutter in current directory + mkdir -p "${builddir}" && cd "${builddir}" + + CFLAGS=-Werror "${basedir}/configure" PG_CONFIG="/usr/lib/postgresql/${pg_major}/bin/pg_config" --enable-coverage + + installdir="${builddir}/install" + make -j$(nproc) && mkdir -p "${installdir}" && { make DESTDIR="${installdir}" install-all || make DESTDIR="${installdir}" install ; } + + cd "${installdir}" && find . -type f -print > "${builddir}/files.lst" + tar cvf "${basedir}/install-${pg_major}.tar" `cat ${builddir}/files.lst` + + cd "${builddir}" && rm -rf install files.lst && make clean +} + +build_ext "${PG_MAJOR}" diff --git a/ci/check_sql_snapshots.sh b/ci/check_sql_snapshots.sh new file mode 100755 index 000000000..3c871ab8b --- /dev/null +++ b/ci/check_sql_snapshots.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +# shellcheck disable=SC1091 +source ci/ci_helpers.sh + +for udf_dir in src/backend/distributed/sql/udfs/*; do + # We want to find the last snapshotted sql file, to make sure it's the same + # as "latest.sql". This is done by: + # 1. Getting the filenames in the UDF directory (using find instead of ls, to keep shellcheck happy) + # 2. Filter out latest.sql + # 3. Sort using "version sort" + # 4. Get the last one using tail + latest_snapshot=$(\ + find "$udf_dir" -iname "*.sql" -exec basename {} \; \ + | { grep --invert-match latest.sql || true; } \ + | sort --version-sort \ + | tail --lines 1); + diff --unified --color=auto "$udf_dir/latest.sql" "$udf_dir/$latest_snapshot"; \ +done diff --git a/ci/ci_helpers.sh b/ci/ci_helpers.sh new file mode 100755 index 000000000..2dc242361 --- /dev/null +++ b/ci/ci_helpers.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# For echo commands "set -x" would show the message effectively twice. Once as +# part of the echo command shown by "set -x" and once because of the output of +# the echo command. We do not want "set -x" to show the echo command. We only +# want to see the actual message in the output of echo itself. This function is +# a trick to do so. Read the StackOverflow post below to understand why this +# works and what this works around. +# Source: https://superuser.com/a/1141026/242593 +shopt -s expand_aliases +alias echo='{ save_flags="$-"; set +x;} 2> /dev/null && echo_and_restore' +echo_and_restore() { + builtin echo "$*" + #shellcheck disable=SC2154 + case "$save_flags" in + (*x*) set -x + esac +} + +# Make sure that on a failing exit we show a useful message +hint_on_fail() { + exit_code=$? + # Get filename of the currently running script + # Source: https://stackoverflow.com/a/192337/2570866 + filename=$(basename "$0") + if [ $exit_code != 0 ]; then + echo "HINT: To solve this failure look here: https://github.com/citusdata/citus/blob/master/ci/README.md#$filename" + fi + exit $exit_code +} +trap hint_on_fail EXIT