diff --git a/.circleci/config.yml b/.circleci/config.yml index 259841090..d117fd21e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: codecov: codecov/codecov@1.0.5 + azure-cli: circleci/azure-cli@1.0.0 jobs: build: @@ -243,7 +244,34 @@ jobs: - run: command: | ./src/test/scripts/check_enterprise_merge.sh - + ch_benchmark: + docker: + - image: buildpack-deps:stretch + working_directory: /home/circleci/project + steps: + - checkout + - azure-cli/install + - azure-cli/login-with-service-principal + - run: + command: | + cd ./src/test/hammerdb + sh run_hammerdb.sh citusbot_ch_benchmark_rg + name: install dependencies and run ch_benchmark tests + no_output_timeout: 20m + tpcc_benchmark: + docker: + - image: buildpack-deps:stretch + working_directory: /home/circleci/project + steps: + - checkout + - azure-cli/install + - azure-cli/login-with-service-principal + - run: + command: | + cd ./src/test/hammerdb + sh run_hammerdb.sh citusbot_tpcc_benchmark_rg + name: install dependencies and run ch_benchmark tests + no_output_timeout: 20m workflows: @@ -283,3 +311,15 @@ workflows: - test-11_check-citus-upgrade: requires: [build] + - ch_benchmark: + requires: [build] + filters: + branches: + only: + - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix + - tpcc_benchmark: + requires: [build] + filters: + branches: + only: + - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix diff --git a/src/test/hammerdb/README.md b/src/test/hammerdb/README.md new file mode 100644 index 000000000..be729207d --- /dev/null +++ b/src/test/hammerdb/README.md @@ -0,0 +1,26 @@ + +# How to trigger hammerdb benchmark jobs + +You can trigger two types of hammerdb benchmark jobs: +-ch_benchmark (analytical and transactional queries) +-tpcc_benchmark (only transactional queries) + +Your branch will be run against `master` branch. + +In order to trigger the jobs prepend `ch_benchmark/` or `tpcc_benchmark/` to your branch and push it. + +For example if you were running on a feature/improvement branch with name `improve/adaptive_executor`. In order to trigger a tpcc benchmark, you can do the following: + +```bash +git checkout improve/adaptive_executor +git checkout -b tpcc_benchmark/improve/adaptive_executor +git push origin tpcc_benchmark/improve/adaptive_executor # the tpcc benchmark job will be triggered. +``` + +You will see the results in a branch in [https://github.com/citusdata/release-test-results](https://github.com/citusdata/release-test-results). + +The branch name will be something like: `delete_me/citusbot_tpcc_benchmark_rg//`. + +On success, which is the vast majority of the cases, the resource groups on Azure which are automatically created for this purpose are deleted automatically. On failure, you need to manually drop the resource groups named: `citusbot_ch_benchmark_rg` and `citusbot_tpcc_benchmark_rg`. To check whether the job failed, go to Azure portal and search for these resource group names. If the resource group doesn't have any virtual machines or the virtual machines don't have any CPU/Memory load at all, it is highly possible that the job failed. Delete the resource groups and re-try with another branch. + +These jobs use the hammerdb tool under the hood, please see [https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark](https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark) for more details. diff --git a/src/test/hammerdb/run_hammerdb.sh b/src/test/hammerdb/run_hammerdb.sh new file mode 100644 index 000000000..9db1da2b6 --- /dev/null +++ b/src/test/hammerdb/run_hammerdb.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# fail if trying to reference a variable that is not set. +set -u +# exit immediately if a command fails +set -e +# echo commands +set -x + +rg=$1 + +export RESOURCE_GROUP_NAME="${rg}" +export AZURE_REGION=westus2 +# the branch name is stored in CIRCLE_BRANCH env variable in CI jobs. +export BRANCH="${CIRCLE_BRANCH}" + +# add github to known hosts +echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts + +set +x +git config --global credential.helper store +git clone https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/test-automation +set -x + +cd test-automation + +test_automation_dir=$(pwd) +# add the ssh keys +eval `ssh-agent -s` +ssh-add + +ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub + +now=$(date +"%m_%d_%Y_%s") +new_branch_name=delete_me/"${rg}"/"${now}" +git checkout -b "${new_branch_name}" + +cd ./fabfile/hammerdb_confs +branch_config=current_branch.ini +# create a config for this branch +cp master.ini "${branch_config}" + +# put the branch name to the config file. +sed -i "s@master@${BRANCH}@g" "${branch_config}" + +cd "${test_automation_dir}" +cd ./hammerdb + +git config --global user.email "citus-bot@microsoft.com" +git config --global user.name "citus bot" +git add -A +git commit -m "test hammerdb: ${rg} vs master" +git push origin "${new_branch_name}" + +cd "${test_automation_dir}" +cd ./hammerdb + +if [ "$rg" = "citusbot_ch_benchmark_rg" ]; then + export IS_CH=true + export IS_TPCC=true +fi + +if [ "$rg" = "citusbot_tpcc_benchmark_rg" ]; then + export IS_CH=false + export IS_TPCC=true +fi + +# create cluster and run the hammerd benchmark +./create-run.sh + +