From 41fceb78493c318f791c989ed055148103c56f29 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 14 May 2020 13:04:12 +0300 Subject: [PATCH] Add optional ch_benchmark and tpcc_benchmark job With this commit: You can trigger two types of hammerdb benchmark jobs: -ch_benchmark (analytical and transactional queries) -tpcc_benchmark (only transactional queries) Your branch will be run against `master` branch. In order to trigger the jobs prepend `ch_benchmark/` or `tpcc_benchmark/` to your branch and push it. For example if you were running on a feature/improvement branch with name `improve/adaptive_executor`. In order to trigger a tpcc benchmark, you can do the following: ```bash git checkout improve/adaptive_executor git checkout -b tpcc_benchmark/improve/adaptive_executor git push origin tpcc_benchmark/improve/adaptive_executor # the tpcc benchmark job will be triggered. ``` You will see the results in a branch in [https://github.com/citusdata/release-test-results](https://github.com/citusdata/release-test-results). The branch name will be something like: `delete_me/citusbot_tpcc_benchmark_rg//`. The resource groups will be deleted automatically but if the benchmark fails, they won't be deleted(If you don't see the results after a reasonable time, it might mean it failed, you can check the resource usage from portal, if it is almost 0 and you didn't see the results, it means it probably failed). In that case, you will need to delete the resource groups manually from portal, the resource groups are `citusbot_ch_benchmark_rg` and `citusbot_tpcc_benchmark_rg`. --- .circleci/config.yml | 42 +++++++++++++++++- src/test/hammerdb/README.md | 26 +++++++++++ src/test/hammerdb/run_hammerdb.sh | 71 +++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 src/test/hammerdb/README.md create mode 100644 src/test/hammerdb/run_hammerdb.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 259841090..d117fd21e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: codecov: codecov/codecov@1.0.5 + azure-cli: circleci/azure-cli@1.0.0 jobs: build: @@ -243,7 +244,34 @@ jobs: - run: command: | ./src/test/scripts/check_enterprise_merge.sh - + ch_benchmark: + docker: + - image: buildpack-deps:stretch + working_directory: /home/circleci/project + steps: + - checkout + - azure-cli/install + - azure-cli/login-with-service-principal + - run: + command: | + cd ./src/test/hammerdb + sh run_hammerdb.sh citusbot_ch_benchmark_rg + name: install dependencies and run ch_benchmark tests + no_output_timeout: 20m + tpcc_benchmark: + docker: + - image: buildpack-deps:stretch + working_directory: /home/circleci/project + steps: + - checkout + - azure-cli/install + - azure-cli/login-with-service-principal + - run: + command: | + cd ./src/test/hammerdb + sh run_hammerdb.sh citusbot_tpcc_benchmark_rg + name: install dependencies and run ch_benchmark tests + no_output_timeout: 20m workflows: @@ -283,3 +311,15 @@ workflows: - test-11_check-citus-upgrade: requires: [build] + - ch_benchmark: + requires: [build] + filters: + branches: + only: + - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix + - tpcc_benchmark: + requires: [build] + filters: + branches: + only: + - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix diff --git a/src/test/hammerdb/README.md b/src/test/hammerdb/README.md new file mode 100644 index 000000000..be729207d --- /dev/null +++ b/src/test/hammerdb/README.md @@ -0,0 +1,26 @@ + +# How to trigger hammerdb benchmark jobs + +You can trigger two types of hammerdb benchmark jobs: +-ch_benchmark (analytical and transactional queries) +-tpcc_benchmark (only transactional queries) + +Your branch will be run against `master` branch. + +In order to trigger the jobs prepend `ch_benchmark/` or `tpcc_benchmark/` to your branch and push it. + +For example if you were running on a feature/improvement branch with name `improve/adaptive_executor`. In order to trigger a tpcc benchmark, you can do the following: + +```bash +git checkout improve/adaptive_executor +git checkout -b tpcc_benchmark/improve/adaptive_executor +git push origin tpcc_benchmark/improve/adaptive_executor # the tpcc benchmark job will be triggered. +``` + +You will see the results in a branch in [https://github.com/citusdata/release-test-results](https://github.com/citusdata/release-test-results). + +The branch name will be something like: `delete_me/citusbot_tpcc_benchmark_rg//`. + +On success, which is the vast majority of the cases, the resource groups on Azure which are automatically created for this purpose are deleted automatically. On failure, you need to manually drop the resource groups named: `citusbot_ch_benchmark_rg` and `citusbot_tpcc_benchmark_rg`. To check whether the job failed, go to Azure portal and search for these resource group names. If the resource group doesn't have any virtual machines or the virtual machines don't have any CPU/Memory load at all, it is highly possible that the job failed. Delete the resource groups and re-try with another branch. + +These jobs use the hammerdb tool under the hood, please see [https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark](https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark) for more details. diff --git a/src/test/hammerdb/run_hammerdb.sh b/src/test/hammerdb/run_hammerdb.sh new file mode 100644 index 000000000..9db1da2b6 --- /dev/null +++ b/src/test/hammerdb/run_hammerdb.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# fail if trying to reference a variable that is not set. +set -u +# exit immediately if a command fails +set -e +# echo commands +set -x + +rg=$1 + +export RESOURCE_GROUP_NAME="${rg}" +export AZURE_REGION=westus2 +# the branch name is stored in CIRCLE_BRANCH env variable in CI jobs. +export BRANCH="${CIRCLE_BRANCH}" + +# add github to known hosts +echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts + +set +x +git config --global credential.helper store +git clone https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/test-automation +set -x + +cd test-automation + +test_automation_dir=$(pwd) +# add the ssh keys +eval `ssh-agent -s` +ssh-add + +ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub + +now=$(date +"%m_%d_%Y_%s") +new_branch_name=delete_me/"${rg}"/"${now}" +git checkout -b "${new_branch_name}" + +cd ./fabfile/hammerdb_confs +branch_config=current_branch.ini +# create a config for this branch +cp master.ini "${branch_config}" + +# put the branch name to the config file. +sed -i "s@master@${BRANCH}@g" "${branch_config}" + +cd "${test_automation_dir}" +cd ./hammerdb + +git config --global user.email "citus-bot@microsoft.com" +git config --global user.name "citus bot" +git add -A +git commit -m "test hammerdb: ${rg} vs master" +git push origin "${new_branch_name}" + +cd "${test_automation_dir}" +cd ./hammerdb + +if [ "$rg" = "citusbot_ch_benchmark_rg" ]; then + export IS_CH=true + export IS_TPCC=true +fi + +if [ "$rg" = "citusbot_tpcc_benchmark_rg" ]; then + export IS_CH=false + export IS_TPCC=true +fi + +# create cluster and run the hammerd benchmark +./create-run.sh + +