Merge pull request #2616 from citusdata/circleci

Enable CircleCI

.circleci/config.yml

@@ -0,0 +1,96 @@
+version: 2.1
+orbs:
+  codecov: codecov/codecov@1.0.4
+
+jobs:
+  build:
+    docker:
+      - {image: 'citusdata/extbuilder:latest'}
+    steps:
+      - checkout
+      - {run: {name: 'Configure, Build, and Install', command: build-ext}}
+      - {persist_to_workspace: {root: ., paths: [.]}}
+  check-style:
+    docker:
+      - {image: 'citusdata/stylechecker:latest'}
+    steps:
+      - checkout
+      - {run: {name: 'Check Style', command: citus_indent --check}}
+  test-10_check-multi:
+    docker:
+      - {image: 'citusdata/exttester-10:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}}
+      - {codecov/upload: {flags: 'test_10,multi'}}
+  test-10_check-tt-van-mx:
+    docker:
+      - {image: 'citusdata/exttester-10:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}}
+      - {codecov/upload: {flags: 'test_10,tracker,vanilla,mx'}}
+  test-10_check-iso-work-fol:
+    docker:
+      - {image: 'citusdata/exttester-10:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}}
+      - {codecov/upload: {flags: 'test_10,isolation,worker,follower'}}
+  test-10_check-failure:
+    docker:
+      - {image: 'citusdata/failtester-10:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}}
+  test-11_check-multi:
+    docker:
+      - {image: 'citusdata/exttester-11:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}}
+      - {codecov/upload: {flags: 'test_11,multi'}}
+  test-11_check-tt-van-mx:
+    docker:
+      - {image: 'citusdata/exttester-11:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}}
+      - {codecov/upload: {flags: 'test_11,tracker,vanilla,mx'}}
+  test-11_check-iso-work-fol:
+    docker:
+      - {image: 'citusdata/exttester-11:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}}
+      - {codecov/upload: {flags: 'test_11,isolation,worker,follower'}}
+  test-11_check-failure:
+    docker:
+      - {image: 'citusdata/failtester-11:latest'}
+    working_directory: /home/circleci/project
+    steps:
+      - {attach_workspace: {at: .}}
+      - {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}}
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - build
+      - check-style
+
+      - {test-10_check-multi: {requires: [build]}}
+      - {test-10_check-tt-van-mx: {requires: [build]}}
+      - {test-10_check-iso-work-fol: {requires: [build]}}
+      - {test-10_check-failure: {requires: [build]}}
+
+      - {test-11_check-multi: {requires: [build]}}
+      - {test-11_check-tt-van-mx: {requires: [build]}}
+      - {test-11_check-iso-work-fol: {requires: [build]}}
+      - {test-11_check-failure: {requires: [build]}}

.travis.yml

@@ -49,8 +49,7 @@ install:
       apt-get download "postgresql-${PGVERSION}-topn=2.2.0"
       sudo dpkg --force-confold --force-confdef --force-all -i *topn*.deb
     fi
-before_script: citus_indent --quiet --check
+before_script: citus_indent --quiet --check || echo 'Ignoring indent failures'
 script: CFLAGS=-Werror pg_travis_multi_test check
 after_success:
   - sync_to_enterprise
-  - bash <(curl -s https://codecov.io/bash)

Makefile.global.in

@@ -69,7 +69,7 @@ endif
 
 # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/...
 override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@
-override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS)
+override CPPFLAGS := @CPPFLAGS@ @CITUS_CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS)
 override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@
 
 # optional file with user defined, additional, rules

configure

@@ -625,6 +625,7 @@ LIBOBJS
 POSTGRES_BUILDDIR
 POSTGRES_SRCDIR
 CITUS_LDFLAGS
+CITUS_CPPFLAGS
 CITUS_CFLAGS
 EGREP
 GREP
@@ -4052,7 +4053,9 @@ if test "${enable_coverage+set}" = set; then :
 fi
 
 if test "$enable_coverage" = yes; then
-    CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage"
+    CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage"
+    CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG"
+    CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
 fi
 
 #
@@ -4183,7 +4186,9 @@ _ACEOF
 
 CITUS_CFLAGS="$CITUS_CFLAGS"
 
-CITUS_LDFLAGS="$LIBS"
+CITUS_CPPFLAGS="$CITUS_CPPFLAGS"
+
+CITUS_LDFLAGS="$LIBS $CITUS_LDFLAGS"
 
 POSTGRES_SRCDIR="$POSTGRES_SRCDIR"
 

configure.in

@@ -170,7 +170,9 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=vla])  # visual studio does not support thes
 #
 AC_ARG_ENABLE([coverage], AS_HELP_STRING([--enable-coverage], [build with coverage testing instrumentation]))
 if test "$enable_coverage" = yes; then
-    CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage"
+    CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage"
+    CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG"
+    CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
 fi
 
 #
@@ -201,7 +203,8 @@ AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL",
                    [Base URL for statistics collection and update checks])
 
 AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS")
-AC_SUBST(CITUS_LDFLAGS, "$LIBS")
+AC_SUBST(CITUS_CPPFLAGS, "$CITUS_CPPFLAGS")
+AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS")
 AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR")
 AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR")
 

src/backend/distributed/utils/citus_copyfuncs.c

@@ -33,8 +33,8 @@ CitusSetTag(Node *node, int tag)
 
 
 #define DECLARE_FROM_AND_NEW_NODE(nodeTypeName) \
-	nodeTypeName * newnode = (nodeTypeName *) \
+	nodeTypeName *newnode = (nodeTypeName *) \
-							 CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \
+							CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \
 	nodeTypeName *from = (nodeTypeName *) source_node
 
 /* Copy a simple scalar field (int, float, bool, enum, etc) */

src/backend/distributed/utils/resource_lock.c

@@ -107,7 +107,7 @@ lock_shard_metadata(PG_FUNCTION_ARGS)
 		ereport(ERROR, (errmsg("no locks specified")));
 	}
 
-    /* we don't want random users to block writes */
+	/* we don't want random users to block writes */
 	EnsureSuperUser();
 
 	shardIdCount = ArrayObjectCount(shardIdArrayObject);
@@ -147,7 +147,7 @@ lock_shard_resources(PG_FUNCTION_ARGS)
 		ereport(ERROR, (errmsg("no locks specified")));
 	}
 
-    /* we don't want random users to block writes */
+	/* we don't want random users to block writes */
 	EnsureSuperUser();
 
 	shardIdCount = ArrayObjectCount(shardIdArrayObject);
@@ -475,7 +475,7 @@ LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode)
 {
 	ListCell *shardIntervalCell = NULL;
 
-    /* lock shards in order of shard id to prevent deadlock */
+	/* lock shards in order of shard id to prevent deadlock */
 	shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById);
 
 	foreach(shardIntervalCell, shardIntervalList)
@@ -497,7 +497,7 @@ LockShardsInPlacementListMetadata(List *shardPlacementList, LOCKMODE lockMode)
 {
 	ListCell *shardPlacementCell = NULL;
 
-    /* lock shards in order of shard id to prevent deadlock */
+	/* lock shards in order of shard id to prevent deadlock */
 	shardPlacementList =
 		SortList(shardPlacementList, CompareShardPlacementsByShardId);
 
@@ -553,7 +553,7 @@ LockShardListResources(List *shardIntervalList, LOCKMODE lockMode)
 {
 	ListCell *shardIntervalCell = NULL;
 
-    /* lock shards in order of shard id to prevent deadlock */
+	/* lock shards in order of shard id to prevent deadlock */
 	shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById);
 
 	foreach(shardIntervalCell, shardIntervalList)
@@ -575,7 +575,7 @@ LockRelationShardResources(List *relationShardList, LOCKMODE lockMode)
 {
 	ListCell *relationShardCell = NULL;
 
-    /* lock shards in a consistent order to prevent deadlock */
+	/* lock shards in a consistent order to prevent deadlock */
 	relationShardList = SortList(relationShardList, CompareRelationShards);
 
 	foreach(relationShardCell, relationShardList)
@@ -641,11 +641,11 @@ LockPartitionsInRelationList(List *relationIdList, LOCKMODE lockmode)
 void
 LockPartitionRelations(Oid relationId, LOCKMODE lockMode)
 {
-    /*
+	/*
-     * PartitionList function generates partition list in the same order
+	 * PartitionList function generates partition list in the same order
-     * as PostgreSQL. Therefore we do not need to sort it before acquiring
+	 * as PostgreSQL. Therefore we do not need to sort it before acquiring
-     * locks.
+	 * locks.
-     */
+	 */
 	List *partitionList = PartitionList(relationId);
 	ListCell *partitionCell = NULL;
 
@@ -678,7 +678,7 @@ LockModeTextToLockMode(const char *lockModeName)
 		}
 	}
 
-    /* we could not find the lock mode we are looking for */
+	/* we could not find the lock mode we are looking for */
 	if (lockMode == -1)
 	{
 		ereport(ERROR,
@@ -712,7 +712,7 @@ LockModeToLockModeText(LOCKMODE lockMode)
 		}
 	}
 
-    /* we could not find the lock mode we are looking for */
+	/* we could not find the lock mode we are looking for */
 	if (lockModeText == NULL)
 	{
 		ereport(ERROR,
@@ -747,17 +747,17 @@ lock_relation_if_exists(PG_FUNCTION_ARGS)
 	LOCKMODE lockMode = NoLock;
 	bool relationExists = false;
 
-    /* ensure that we're in a transaction block */
+	/* ensure that we're in a transaction block */
 	RequireTransactionBlock(true, "lock_relation_if_exists");
 
-    /* get the lock mode */
+	/* get the lock mode */
 	lockMode = LockModeTextToLockMode(lockModeCString);
 
-    /* resolve relationId from passed in schema and relation name */
+	/* resolve relationId from passed in schema and relation name */
 	relationNameList = textToQualifiedNameList(relationName);
 	relation = makeRangeVarFromNameList(relationNameList);
 
-    /* lock the relation with the lock mode */
+	/* lock the relation with the lock mode */
 	relationId = RangeVarGetRelidInternal(relation, lockMode, RVR_MISSING_OK,
 										  CitusRangeVarCallbackForLockTable,
 										  (void *) &lockMode);
@@ -783,18 +783,18 @@ CitusRangeVarCallbackForLockTable(const RangeVar *rangeVar, Oid relationId,
 
 	if (!OidIsValid(relationId))
 	{
-        /* table doesn't exist, so no permissions check */
+		/* table doesn't exist, so no permissions check */
 		return;
 	}
 
-    /* we only allow tables and views to be locked */
+	/* we only allow tables and views to be locked */
 	if (!RegularTable(relationId))
 	{
 		ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
 						errmsg("\"%s\" is not a table", rangeVar->relname)));
 	}
 
-    /* check permissions */
+	/* check permissions */
 	aclResult = CitusLockTableAclCheck(relationId, lockmode, GetUserId());
 	if (aclResult != ACLCHECK_OK)
 	{
@@ -822,7 +822,7 @@ CitusLockTableAclCheck(Oid relationId, LOCKMODE lockmode, Oid userId)
 	AclResult aclResult;
 	AclMode aclMask;
 
-    /* verify adequate privilege */
+	/* verify adequate privilege */
 	if (lockmode == AccessShareLock)
 	{
 		aclMask = ACL_SELECT;

src/test/regress/Makefile

@@ -15,7 +15,7 @@ endif
 ##
 MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install
 pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)"  --postgres-srcdir="$(postgres_abs_srcdir)"
-MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags)
+MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times"
 
 # XXX: Can't actually do useful testruns against install - $libdir
 # etc will point to the directory configured during postgres'

src/test/regress/log_test_times

@@ -0,0 +1,4 @@
+#!/bin/bash
+export TIMEFORMAT="${PG_MAJOR}/${PGAPPNAME} %6R"
+
+{ { time "$@" 1>&3- 2>&4-; } 2>> test_times.log; } 3>&1 4>&2

src/test/regress/pg_regress_multi.pl

@@ -16,6 +16,7 @@ use warnings;
 
 use Fcntl;
 use Getopt::Long;
+use File::Basename;
 use File::Spec::Functions;
 use File::Path qw(make_path remove_tree);
 use Config;
@@ -151,6 +152,11 @@ else
 	$plainRegress = "$pgxsdir/src/test/regress/pg_regress";
 	$isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress";
 	$pgConfig = "$bindir/pg_config";
+
+	if (-x "$pgxsdir/src/test/isolation/pg_isolation_regress")
+	{
+		$isolationRegress = "$pgxsdir/src/test/isolation/pg_isolation_regress";
+	}
 }
 
 if ($isolationtester && ! -f "$isolationRegress")
@@ -171,7 +177,9 @@ MESSAGE
 }
 
 my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress");
-if ($vanillatest && ! -f "$vanillaRegress")
+my $vanillaSchedule = catfile(dirname("${pgxsdir}"), "regress", "parallel_schedule");
+
+if ($vanillatest && ! (-f "$vanillaRegress" or -f "$vanillaSchedule"))
 {
     die <<"MESSAGE";
 
@@ -796,8 +804,21 @@ if ($vanillatest)
     $ENV{PGPORT} = $masterPort;
     $ENV{PGUSER} = $user;
 
-    system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0
+	if (-f "$vanillaSchedule")
-    or die "Could not run vanilla tests";
+	{
+	    rmdir "./testtablespace";
+	    mkdir "./testtablespace";
+
+	    my $pgregressdir=catfile(dirname("$pgxsdir"), "regress");
+	    system("$plainRegress", ("--inputdir",  $pgregressdir),
+	           ("--schedule",  catfile("$pgregressdir", "parallel_schedule"))) == 0
+		or die "Could not run vanilla tests";
+	}
+	else
+	{
+	    system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0
+		or die "Could not run vanilla tests";
+	}
 }
 elsif ($isolationtester)
 {