add support to run citus upgrade tests locally (#3083)

* add support to run citus upgrade tests locally

* dont build tars if they already exist

* use current code instead of master for upgrade

* always build the current code

* copy the current citus code to have isolated citus upgrade tests

* fix configure and simplify copy

src/test/regress/.gitignore

@@ -5,6 +5,7 @@
 /tmp_check/
 /tmp_upgrade/
 /tmp_citus_upgrade/
+/tmp_citus_tarballs/
 /results/
 /log/
 

src/test/regress/Makefile

@@ -161,6 +161,19 @@ check-citus-upgrade-mixed:
 		--citus-post-tar=$(citus-post-tar) \
 		--mixed
 
+check-citus-upgrade-local:
+	$(citus_upgrade_check) \
+		--bindir=$(bindir) \
+		--pgxsdir=$(pgxsdir) \
+		--citus-old-version=$(citus-old-version)	
+
+check-citus-upgrade-mixed-local:
+	$(citus_upgrade_check) \
+		--bindir=$(bindir) \
+		--pgxsdir=$(pgxsdir) \
+		--citus-old-version=$(citus-old-version) \
+		--mixed			
+
 clean distclean maintainer-clean:
 	rm -f $(output_files) $(input_files)
 	rm -rf tmp_check/

src/test/regress/upgrade/README.md

@@ -98,3 +98,15 @@ To see full command list:
 ```bash
     pipenv run upgrade/citus_upgrade_test.py -help
 ```
+
+In order to run citus upgrade tests locally you can use:
+
+```bash
+    pipenv run make check-citus-upgrade-local citus-old-version=v8.0.0
+```
+
+For mixed mode:
+
+```bash
+    pipenv run make check-citus-upgrade-mixed-local citus-old-version=v8.0.0
+```

src/test/regress/upgrade/citus_upgrade_test.py

@@ -2,14 +2,15 @@
 
 """citus_upgrade_test
 Usage:
-    citus_upgrade_test [options] --bindir=<bindir> --pgxsdir=<pgxsdir>
+    citus_upgrade_test [options] --bindir=<bindir> --pgxsdir=<pgxsdir> [--citus-old-version=<citus-old-version>]
 
 Options:
-    --bindir=<bindir>                      The PostgreSQL executable directory(ex: '~/.pgenv/pgsql-11.3/bin')
-    --citus-pre-tar=<citus-pre-tar>        Tarball with the citus artifacts to use as the base version to upgrade from
-    --citus-post-tar=<citus-post-tar>      Tarball with the citus artifacts to use as the new version to upgrade to
-    --pgxsdir=<pgxsdir>           	       Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3)
-    --mixed                                Run the verification phase with one node not upgraded.
+    --bindir=<bindir>                       The PostgreSQL executable directory(ex: '~/.pgenv/pgsql-11.3/bin')
+    --citus-pre-tar=<citus-pre-tar>         Tarball with the citus artifacts to use as the base version to upgrade from
+    --citus-post-tar=<citus-post-tar>       Tarball with the citus artifacts to use as the new version to upgrade to
+    --pgxsdir=<pgxsdir>           	        Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3)
+    --citus-old-version=<citus-old-version> Citus old version for local execution(ex v8.0.0)
+    --mixed                                 Run the verification phase with one node not upgraded.
 """
 
 import subprocess
@@ -49,36 +50,44 @@ def main(config):
     run_test_on_coordinator(config, AFTER_CITUS_UPGRADE_COORD_SCHEDULE)
     remove_citus(config.post_tar_path)
 
+
 def install_citus(tar_path):
     with utils.cd('/'):
         subprocess.call(['tar', 'xvf', tar_path])
 
+
 def report_initial_version(config):
     for port in NODE_PORTS.values():
         actual_citus_version = get_actual_citus_version(config.bindir, port)
-        print("port:{} citus version {}".format(port , actual_citus_version))
+        print("port:{} citus version {}".format(port, actual_citus_version))
+
 
 def get_version_number(version):
     return re.findall('\d+.\d+', version)[0]
 
+
 def get_actual_citus_version(pg_path, port):
     citus_version = utils.psql(pg_path, port, CITUS_VERSION_SQL)
     citus_version = citus_version.decode('utf-8')
     return get_version_number(citus_version)
 
+
 def run_test_on_coordinator(config, schedule):
     common.run_pg_regress(config.bindir, config.pg_srcdir,
-                   NODE_PORTS[COORDINATOR_NAME], schedule)
+                          NODE_PORTS[COORDINATOR_NAME], schedule)
+
 
 def remove_citus(tar_path):
     with utils.cd('/'):
         remove_tar_files(tar_path)
 
+
 def remove_tar_files(tar_path):
     ps = subprocess.Popen(('tar', 'tf', tar_path), stdout=subprocess.PIPE)
     output = subprocess.check_output(('xargs', 'rm', '-v'), stdin=ps.stdout)
     ps.wait()
 
+
 def restart_databases(pg_path, rel_data_path, mixed_mode):
     for node_name in NODE_NAMES:
         if mixed_mode and NODE_PORTS[node_name] == WORKER1PORT:
@@ -87,6 +96,7 @@ def restart_databases(pg_path, rel_data_path, mixed_mode):
         restart_database(
             pg_path=pg_path, abs_data_path=abs_data_path, node_name=node_name)
 
+
 def restart_database(pg_path, abs_data_path, node_name):
     command = [
         os.path.join(pg_path, 'pg_ctl'), 'restart',
@@ -97,23 +107,52 @@ def restart_database(pg_path, abs_data_path, node_name):
     ]
     subprocess.call(command)
 
+
 def run_alter_citus(pg_path, mixed_mode):
     for port in NODE_PORTS.values():
         if mixed_mode and port == WORKER1PORT:
             continue
         utils.psql(pg_path, port, "ALTER EXTENSION citus UPDATE;")
 
+
 def verify_upgrade(config, mixed_mode):
     for port in NODE_PORTS.values():
         actual_citus_version = get_actual_citus_version(config.bindir, port)
         expected_citus_version = MASTER_VERSION
         if expected_citus_version != actual_citus_version and not (mixed_mode and port == WORKER1PORT):
-            print("port: {} citus version {} expected {}".format(port, actual_citus_version, expected_citus_version))
+            print("port: {} citus version {} expected {}".format(
+                port, actual_citus_version, expected_citus_version))
             sys.exit(1)
         else:
-            print("port:{} citus version {}".format(port , actual_citus_version))
+            print("port:{} citus version {}".format(port, actual_citus_version))
+
+
+def isLocalExecution(arguments):
+    return arguments['--citus-old-version']
+
+
+def generate_citus_tarballs(citus_version):
+    tmp_dir = 'tmp_citus_tarballs'
+    citus_old_tarpath = os.path.abspath(os.path.join(
+        tmp_dir, 'install-citus{}.tar'.format(citus_version)))
+    citus_new_tarpath = os.path.abspath(os.path.join(tmp_dir, 'install-citusmaster.tar')) 
+
+    common.initialize_temp_dir_if_not_exists(tmp_dir)
+    local_script_path = os.path.abspath('upgrade/generate_citus_tarballs.sh')
+    with utils.cd(tmp_dir):
+        subprocess.check_call([
+            local_script_path, citus_version
+        ])
+
+    return [citus_old_tarpath, citus_new_tarpath]
 
 if __name__ == '__main__':
-    config = CitusUpgradeConfig(docopt(__doc__, version='citus_upgrade_test'))
+    args = docopt(__doc__, version='citus_upgrade_test')
+    if isLocalExecution(args):
+        citus_tarball_paths = generate_citus_tarballs(
+            args['--citus-old-version'])
+        args['--citus-pre-tar'] = citus_tarball_paths[0]
+        args['--citus-post-tar'] = citus_tarball_paths[1]
+    config = CitusUpgradeConfig(args)
     atexit.register(common.stop_databases, config.bindir, config.datadir)
     main(config)

src/test/regress/upgrade/config.py

@@ -18,6 +18,7 @@ CITUS_VERSION_SQL = "SELECT extversion FROM pg_extension WHERE extname = 'citus'
 
 
 class CitusUpgradeConfig():
+
     def __init__(self, arguments):
         self.bindir = arguments['--bindir']
         self.pre_tar_path = arguments['--citus-pre-tar']
@@ -32,7 +33,6 @@ class CitusUpgradeConfig():
         }
         self.mixed_mode = arguments['--mixed']
 
-
 class PGUpgradeConfig():
     def __init__(self, arguments):
         self.old_bindir = arguments['--old-bindir']

src/test/regress/upgrade/generate_citus_tarballs.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+citus_old_version=$1
+
+base="$(pwd)"
+
+
+install_citus_and_tar() {
+  # do everything in a subdirectory to avoid clutter in current directory
+  mkdir -p "${builddir}" && cd "${builddir}"
+
+  "${citus_dir}/configure" --without-libcurl
+
+  installdir="${builddir}/install"
+  make "-j$(nproc)" && mkdir -p "${installdir}" && make DESTDIR="${installdir}" install
+
+  cd "${installdir}" && find . -type f -print > "${builddir}/files.lst"
+
+  tar cvf "${basedir}/install-citus${citus_version}.tar" $(cat "${builddir}"/files.lst)
+  mv "${basedir}/install-citus${citus_version}.tar" "${base}/install-citus${citus_version}.tar"
+
+  cd "${builddir}" && rm -rf install files.lst && make clean
+}
+
+build_current() {
+  citus_version="$1"
+  basedir="${base}/${citus_version}"
+
+  mkdir -p "${basedir}"
+  citus_repo="${base}/../../../.."
+
+  cd "$citus_repo" && cp -R . /tmp/citus_copy
+  mv /tmp/citus_copy "${basedir}/citus_${citus_version}"
+  builddir="${basedir}/build"
+  cd "${basedir}"
+
+  citus_dir=${basedir}/citus_$citus_version
+
+  make -C "${citus_dir}" clean
+  cd "${citus_dir}"
+  ./configure --without-libcurl
+
+  install_citus_and_tar
+}
+
+build_ext() {
+  citus_version="$1"
+  basedir="${base}/${citus_version}"
+
+  if test -f "${base}/install-citus${citus_version}.tar"; then
+    return
+  fi
+  
+  mkdir -p "${basedir}"
+  cd "${basedir}"
+  citus_dir=${basedir}/citus_$citus_version
+  git clone --branch "$citus_version" https://github.com/citusdata/citus.git --depth 1 citus_"$citus_version"  
+  builddir="${basedir}/build"
+
+  install_citus_and_tar
+}
+
+build_current "master"
+build_ext "${citus_old_version}"

src/test/regress/upgrade/upgrade_common.py

@@ -16,6 +16,12 @@ def initialize_temp_dir(temp_dir):
     # Give full access to TEMP_DIR so that postgres user can use it.
     os.chmod(temp_dir, 0o777)
 
+def initialize_temp_dir_if_not_exists(temp_dir):
+    if os.path.exists(temp_dir):
+        return
+    os.mkdir(temp_dir)
+    # Give full access to TEMP_DIR so that postgres user can use it.
+    os.chmod(temp_dir, 0o777)
 
 def initialize_db_for_cluster(pg_path, rel_data_path, settings):
     subprocess.call(['mkdir', rel_data_path])