external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix citus upgrade local run issues (#5414) 

This PR is fixing 2 separate issues related to the local run of citus upgrade tests.

d3e7c825ab fixes the issue that, with our new testing infrastructure, we moved/renamed some of existing folders. This created a problem for local runs of citus upgrade tests since some paths were sensitive to such changes. This commit tries to make it more generic so that this issue is less likely to happen in the future, while also fixing the current issue.

93de6b60c3 we are fixing an issue that a new environment variable was added for citus upgrade tests, which is defined in the CI. 0cb51f8c37/.circleci/config.yml (L294)
This environment variable wasn't set in our local runs hence it would create problems. Instead of defining this environment variable in the local run, we change the citus_upgrade run command to use an existing env variable, which is now also set in the CI.
pull/5391/head
Talha Nisanci 2021-11-03 16:17:36 +03:00 committed by GitHub
parent 9b784e58bf
commit 19f28eabae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 54 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.circleci/config.yml
View File

@ -291,11 +291,11 @@ jobs:
# run make check-citus-upgrade for all citus versions
# the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of
for citus_version in ${CITUS_VERSIONS}; do \
export upgrade_test_old_citus_version="$citus_version"; \
gosu circleci \
make -C src/test/regress \
check-citus-upgrade \
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
citus-old-version=${citus_version} \
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \
done;
@ -306,6 +306,7 @@ jobs:
gosu circleci \
make -C src/test/regress \
check-citus-upgrade-mixed \
citus-old-version=${citus_version} \
bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \
citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \
citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \

2
src/test/regress/Makefile
View File

@ -28,7 +28,7 @@ pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsd
MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times"
pg_upgrade_check = $(citus_abs_srcdir)/citus_tests/upgrade/pg_upgrade_test.py
citus_upgrade_check = $(citus_abs_srcdir)/citus_tests/upgrade/citus_upgrade_test.py
citus_upgrade_check =CITUS_OLD_VERSION=$(citus-old-version) $(citus_abs_srcdir)/citus_tests/upgrade/citus_upgrade_test.py
arbitrary_config_check = $(citus_abs_srcdir)/citus_tests/arbitrary_configs/citus_arbitrary_configs.py
template_isolation_files = $(shell find $(citus_abs_srcdir)/spec/ -name '*.spec')

3
src/test/regress/citus_tests/upgrade/citus_upgrade_test.py
View File

@ -188,7 +188,8 @@ def generate_citus_tarballs(citus_version):
)
common.initialize_temp_dir_if_not_exists(tmp_dir)
local_script_path = os.path.abspath("upgrade/generate_citus_tarballs.sh")
dirpath = os.path.dirname(os.path.realpath(__file__))
local_script_path = os.path.join(dirpath, "generate_citus_tarballs.sh")
with utils.cd(tmp_dir):
subprocess.check_call([local_script_path, citus_version])

6
src/test/regress/citus_tests/upgrade/generate_citus_tarballs.sh
View File

@ -6,7 +6,6 @@ citus_old_version=$1
base="$(pwd)"
install_citus_and_tar() {
# do everything in a subdirectory to avoid clutter in current directory
mkdir -p "${builddir}" && cd "${builddir}"
@ -16,7 +15,7 @@ install_citus_and_tar() {
installdir="${builddir}/install"
make "-j$(nproc)" && mkdir -p "${installdir}" && make DESTDIR="${installdir}" install
cd "${installdir}" && find . -type f -print > "${builddir}/files.lst"
cd "${installdir}" && find . -type f -print >"${builddir}/files.lst"
tar cvf "${basedir}/install-citus${citus_version}.tar" $(cat "${builddir}"/files.lst)
mv "${basedir}/install-citus${citus_version}.tar" "${base}/install-citus${citus_version}.tar"
@ -29,9 +28,10 @@ build_current() {
basedir="${base}/${citus_version}"
mkdir -p "${basedir}"
citus_repo="${base}/../../../.."
citus_repo=$(git rev-parse --show-toplevel)
cd "$citus_repo" && cp -R . /tmp/citus_copy
# https://stackoverflow.com/questions/957928/is-there-a-way-to-get-the-git-root-directory-in-one-command
mv /tmp/citus_copy "${basedir}/citus_${citus_version}"
builddir="${basedir}/build"
cd "${basedir}"

2
src/test/regress/expected/upgrade_columnar_metapage_after.out
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/expected/upgrade_columnar_metapage_after_0.out
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/expected/upgrade_columnar_metapage_before.out
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/expected/upgrade_columnar_metapage_before_0.out
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/expected/upgrade_partition_constraints_after.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_partition_constraints_after_0.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_partition_constraints_before.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_partition_constraints_before_0.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_pg_dist_object_test_after.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_pg_dist_object_test_after_0.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_pg_dist_object_test_before.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/expected/upgrade_pg_dist_object_test_before_0.out
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/sql/upgrade_columnar_metapage_after.sql
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/sql/upgrade_columnar_metapage_before.sql
View File

@ -1,4 +1,4 @@
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int >= 10 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int >= 0
AS upgrade_test_old_citus_version_ge_10_0;

2
src/test/regress/sql/upgrade_partition_constraints_after.sql
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/sql/upgrade_partition_constraints_before.sql
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/sql/upgrade_pg_dist_object_test_after.sql
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;

2
src/test/regress/sql/upgrade_pg_dist_object_test_before.sql
View File

@ -1,5 +1,5 @@
-- run this test only when old citus version is 9.0
\set upgrade_test_old_citus_version `echo "$upgrade_test_old_citus_version"`
\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"`
SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int = 9 AND
substring(:'upgrade_test_old_citus_version', 'v\d+\.(\d+)\.\d+')::int = 0
AS upgrade_test_old_citus_version_e_9_0;