diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..0b81c3896 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,96 @@ +version: 2.1 +orbs: + codecov: codecov/codecov@1.0.4 + +jobs: + build: + docker: + - {image: 'citusdata/extbuilder:latest'} + steps: + - checkout + - {run: {name: 'Configure, Build, and Install', command: build-ext}} + - {persist_to_workspace: {root: ., paths: [.]}} + check-style: + docker: + - {image: 'citusdata/stylechecker:latest'} + steps: + - checkout + - {run: {name: 'Check Style', command: citus_indent --check}} + test-10_check-multi: + docker: + - {image: 'citusdata/exttester-10:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}} + - {codecov/upload: {flags: 'test_10,multi'}} + test-10_check-tt-van-mx: + docker: + - {image: 'citusdata/exttester-10:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}} + - {codecov/upload: {flags: 'test_10,tracker,vanilla,mx'}} + test-10_check-iso-work-fol: + docker: + - {image: 'citusdata/exttester-10:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}} + - {codecov/upload: {flags: 'test_10,isolation,worker,follower'}} + test-10_check-failure: + docker: + - {image: 'citusdata/failtester-10:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}} + test-11_check-multi: + docker: + - {image: 'citusdata/exttester-11:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}} + - {codecov/upload: {flags: 'test_11,multi'}} + test-11_check-tt-van-mx: + docker: + - {image: 'citusdata/exttester-11:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}} + - {codecov/upload: {flags: 'test_11,tracker,vanilla,mx'}} + test-11_check-iso-work-fol: + docker: + - {image: 'citusdata/exttester-11:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}} + - {codecov/upload: {flags: 'test_11,isolation,worker,follower'}} + test-11_check-failure: + docker: + - {image: 'citusdata/failtester-11:latest'} + working_directory: /home/circleci/project + steps: + - {attach_workspace: {at: .}} + - {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}} +workflows: + version: 2 + build_and_test: + jobs: + - build + - check-style + + - {test-10_check-multi: {requires: [build]}} + - {test-10_check-tt-van-mx: {requires: [build]}} + - {test-10_check-iso-work-fol: {requires: [build]}} + - {test-10_check-failure: {requires: [build]}} + + - {test-11_check-multi: {requires: [build]}} + - {test-11_check-tt-van-mx: {requires: [build]}} + - {test-11_check-iso-work-fol: {requires: [build]}} + - {test-11_check-failure: {requires: [build]}} diff --git a/.travis.yml b/.travis.yml index ddb40e124..f70f8207b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,8 +49,7 @@ install: apt-get download "postgresql-${PGVERSION}-topn=2.2.0" sudo dpkg --force-confold --force-confdef --force-all -i *topn*.deb fi -before_script: citus_indent --quiet --check +before_script: citus_indent --quiet --check || echo 'Ignoring indent failures' script: CFLAGS=-Werror pg_travis_multi_test check after_success: - sync_to_enterprise - - bash <(curl -s https://codecov.io/bash) diff --git a/Makefile.global.in b/Makefile.global.in index b3efb54ca..df8eb7cd8 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -69,7 +69,7 @@ endif # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/... override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@ -override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS) +override CPPFLAGS := @CPPFLAGS@ @CITUS_CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS) override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@ # optional file with user defined, additional, rules diff --git a/configure b/configure index fa5a70cac..c61422438 100755 --- a/configure +++ b/configure @@ -625,6 +625,7 @@ LIBOBJS POSTGRES_BUILDDIR POSTGRES_SRCDIR CITUS_LDFLAGS +CITUS_CPPFLAGS CITUS_CFLAGS EGREP GREP @@ -4052,7 +4053,9 @@ if test "${enable_coverage+set}" = set; then : fi if test "$enable_coverage" = yes; then - CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage" + CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage" + CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG" + CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage" fi # @@ -4183,7 +4186,9 @@ _ACEOF CITUS_CFLAGS="$CITUS_CFLAGS" -CITUS_LDFLAGS="$LIBS" +CITUS_CPPFLAGS="$CITUS_CPPFLAGS" + +CITUS_LDFLAGS="$LIBS $CITUS_LDFLAGS" POSTGRES_SRCDIR="$POSTGRES_SRCDIR" diff --git a/configure.in b/configure.in index 438e46571..315a03393 100644 --- a/configure.in +++ b/configure.in @@ -170,7 +170,9 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=vla]) # visual studio does not support thes # AC_ARG_ENABLE([coverage], AS_HELP_STRING([--enable-coverage], [build with coverage testing instrumentation])) if test "$enable_coverage" = yes; then - CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage" + CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage" + CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG" + CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage" fi # @@ -201,7 +203,8 @@ AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL", [Base URL for statistics collection and update checks]) AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") -AC_SUBST(CITUS_LDFLAGS, "$LIBS") +AC_SUBST(CITUS_CPPFLAGS, "$CITUS_CPPFLAGS") +AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS") AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index 9d02a2b5e..757317851 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -33,8 +33,8 @@ CitusSetTag(Node *node, int tag) #define DECLARE_FROM_AND_NEW_NODE(nodeTypeName) \ - nodeTypeName * newnode = (nodeTypeName *) \ - CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \ + nodeTypeName *newnode = (nodeTypeName *) \ + CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \ nodeTypeName *from = (nodeTypeName *) source_node /* Copy a simple scalar field (int, float, bool, enum, etc) */ diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index 36989d0f6..57a4d7ccc 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -107,7 +107,7 @@ lock_shard_metadata(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("no locks specified"))); } - /* we don't want random users to block writes */ + /* we don't want random users to block writes */ EnsureSuperUser(); shardIdCount = ArrayObjectCount(shardIdArrayObject); @@ -147,7 +147,7 @@ lock_shard_resources(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("no locks specified"))); } - /* we don't want random users to block writes */ + /* we don't want random users to block writes */ EnsureSuperUser(); shardIdCount = ArrayObjectCount(shardIdArrayObject); @@ -475,7 +475,7 @@ LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode) { ListCell *shardIntervalCell = NULL; - /* lock shards in order of shard id to prevent deadlock */ + /* lock shards in order of shard id to prevent deadlock */ shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); foreach(shardIntervalCell, shardIntervalList) @@ -497,7 +497,7 @@ LockShardsInPlacementListMetadata(List *shardPlacementList, LOCKMODE lockMode) { ListCell *shardPlacementCell = NULL; - /* lock shards in order of shard id to prevent deadlock */ + /* lock shards in order of shard id to prevent deadlock */ shardPlacementList = SortList(shardPlacementList, CompareShardPlacementsByShardId); @@ -553,7 +553,7 @@ LockShardListResources(List *shardIntervalList, LOCKMODE lockMode) { ListCell *shardIntervalCell = NULL; - /* lock shards in order of shard id to prevent deadlock */ + /* lock shards in order of shard id to prevent deadlock */ shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); foreach(shardIntervalCell, shardIntervalList) @@ -575,7 +575,7 @@ LockRelationShardResources(List *relationShardList, LOCKMODE lockMode) { ListCell *relationShardCell = NULL; - /* lock shards in a consistent order to prevent deadlock */ + /* lock shards in a consistent order to prevent deadlock */ relationShardList = SortList(relationShardList, CompareRelationShards); foreach(relationShardCell, relationShardList) @@ -641,11 +641,11 @@ LockPartitionsInRelationList(List *relationIdList, LOCKMODE lockmode) void LockPartitionRelations(Oid relationId, LOCKMODE lockMode) { - /* - * PartitionList function generates partition list in the same order - * as PostgreSQL. Therefore we do not need to sort it before acquiring - * locks. - */ + /* + * PartitionList function generates partition list in the same order + * as PostgreSQL. Therefore we do not need to sort it before acquiring + * locks. + */ List *partitionList = PartitionList(relationId); ListCell *partitionCell = NULL; @@ -678,7 +678,7 @@ LockModeTextToLockMode(const char *lockModeName) } } - /* we could not find the lock mode we are looking for */ + /* we could not find the lock mode we are looking for */ if (lockMode == -1) { ereport(ERROR, @@ -712,7 +712,7 @@ LockModeToLockModeText(LOCKMODE lockMode) } } - /* we could not find the lock mode we are looking for */ + /* we could not find the lock mode we are looking for */ if (lockModeText == NULL) { ereport(ERROR, @@ -747,17 +747,17 @@ lock_relation_if_exists(PG_FUNCTION_ARGS) LOCKMODE lockMode = NoLock; bool relationExists = false; - /* ensure that we're in a transaction block */ + /* ensure that we're in a transaction block */ RequireTransactionBlock(true, "lock_relation_if_exists"); - /* get the lock mode */ + /* get the lock mode */ lockMode = LockModeTextToLockMode(lockModeCString); - /* resolve relationId from passed in schema and relation name */ + /* resolve relationId from passed in schema and relation name */ relationNameList = textToQualifiedNameList(relationName); relation = makeRangeVarFromNameList(relationNameList); - /* lock the relation with the lock mode */ + /* lock the relation with the lock mode */ relationId = RangeVarGetRelidInternal(relation, lockMode, RVR_MISSING_OK, CitusRangeVarCallbackForLockTable, (void *) &lockMode); @@ -783,18 +783,18 @@ CitusRangeVarCallbackForLockTable(const RangeVar *rangeVar, Oid relationId, if (!OidIsValid(relationId)) { - /* table doesn't exist, so no permissions check */ + /* table doesn't exist, so no permissions check */ return; } - /* we only allow tables and views to be locked */ + /* we only allow tables and views to be locked */ if (!RegularTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", rangeVar->relname))); } - /* check permissions */ + /* check permissions */ aclResult = CitusLockTableAclCheck(relationId, lockmode, GetUserId()); if (aclResult != ACLCHECK_OK) { @@ -822,7 +822,7 @@ CitusLockTableAclCheck(Oid relationId, LOCKMODE lockmode, Oid userId) AclResult aclResult; AclMode aclMask; - /* verify adequate privilege */ + /* verify adequate privilege */ if (lockmode == AccessShareLock) { aclMask = ACL_SELECT; diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 0901defab..1e5ffa8bf 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -15,7 +15,7 @@ endif ## MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" -MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) +MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times" # XXX: Can't actually do useful testruns against install - $libdir # etc will point to the directory configured during postgres' diff --git a/src/test/regress/log_test_times b/src/test/regress/log_test_times new file mode 100755 index 000000000..0ceefb10d --- /dev/null +++ b/src/test/regress/log_test_times @@ -0,0 +1,4 @@ +#!/bin/bash +export TIMEFORMAT="${PG_MAJOR}/${PGAPPNAME} %6R" + +{ { time "$@" 1>&3- 2>&4-; } 2>> test_times.log; } 3>&1 4>&2 diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index c9846675d..d3eb1b681 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -16,6 +16,7 @@ use warnings; use Fcntl; use Getopt::Long; +use File::Basename; use File::Spec::Functions; use File::Path qw(make_path remove_tree); use Config; @@ -151,6 +152,11 @@ else $plainRegress = "$pgxsdir/src/test/regress/pg_regress"; $isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress"; $pgConfig = "$bindir/pg_config"; + + if (-x "$pgxsdir/src/test/isolation/pg_isolation_regress") + { + $isolationRegress = "$pgxsdir/src/test/isolation/pg_isolation_regress"; + } } if ($isolationtester && ! -f "$isolationRegress") @@ -171,7 +177,9 @@ MESSAGE } my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress"); -if ($vanillatest && ! -f "$vanillaRegress") +my $vanillaSchedule = catfile(dirname("${pgxsdir}"), "regress", "parallel_schedule"); + +if ($vanillatest && ! (-f "$vanillaRegress" or -f "$vanillaSchedule")) { die <<"MESSAGE"; @@ -796,8 +804,21 @@ if ($vanillatest) $ENV{PGPORT} = $masterPort; $ENV{PGUSER} = $user; - system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0 - or die "Could not run vanilla tests"; + if (-f "$vanillaSchedule") + { + rmdir "./testtablespace"; + mkdir "./testtablespace"; + + my $pgregressdir=catfile(dirname("$pgxsdir"), "regress"); + system("$plainRegress", ("--inputdir", $pgregressdir), + ("--schedule", catfile("$pgregressdir", "parallel_schedule"))) == 0 + or die "Could not run vanilla tests"; + } + else + { + system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0 + or die "Could not run vanilla tests"; + } } elsif ($isolationtester) {