Merge pull request #8 from citusdata/circleci-project-setup

Circleci project setup

.circleci/build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euxo pipefail
+IFS=$'\n\t'
+
+status=0
+
+basedir="$(pwd)"
+installdir="${basedir}/install-${PG_MAJOR}"
+
+make install DESTDIR="${installdir}"
+pushd "${installdir}"
+find . -type f -print > "${basedir}/files.lst"
+cat "${basedir}/files.lst"
+tar cvf "${basedir}/install-${PG_MAJOR}.tar" $(cat "${basedir}/files.lst")
+popd

.circleci/config.yml

@@ -0,0 +1,101 @@
+version: 2.1
+orbs:
+  codecov: codecov/codecov@1.1.1
+
+jobs:
+  check-style:
+    docker:
+      - image: 'citus/stylechecker:latest'
+    steps:
+      - checkout
+      - run:
+          name: 'Check Style'
+          command: |
+            citus_indent --check
+      - run:
+          name: 'Check if whitespace fixing changed anything, install editorconfig if it did'
+          command: |
+            git diff --exit-code
+
+  build-11:
+    docker:
+      - image: 'citus/extbuilder:11.9'
+    steps:
+      - checkout
+      - run:
+          name: 'Configure, Build, and Install'
+          command: |
+            PG_MAJOR=11 .circleci/build.sh
+      - persist_to_workspace:
+          root: .
+          paths:
+            - install-11.tar
+
+  build-12:
+    docker:
+      - image: 'citus/extbuilder:12.4'
+    steps:
+      - checkout
+      - run:
+          name: 'Configure, Build, and Install'
+          command: |
+            PG_MAJOR=12 .circleci/build.sh
+      - persist_to_workspace:
+          root: .
+          paths:
+            - install-12.tar
+
+  test-11_checkinstall:
+    docker:
+      - image: 'citus/exttester:11.9'
+    working_directory: /home/circleci/project
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: 'Prepare Container & Install Extension'
+          command: |
+            chown -R circleci:circleci /home/circleci
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
+      - run:
+          name: 'Run Test'
+          command: |
+            gosu circleci .circleci/run_test.sh installcheck
+      - codecov/upload:
+          flags: 'test_11,installcheck'
+
+  test-12_checkinstall:
+    docker:
+      - image: 'citus/exttester:12.4'
+    working_directory: /home/circleci/project
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: 'Prepare Container & Install Extension'
+          command: |
+            chown -R circleci:circleci /home/circleci
+            tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory /
+      - run:
+          name: 'Run Test'
+          command: |
+            gosu circleci .circleci/run_test.sh installcheck
+      - codecov/upload:
+          flags: 'test_12,installcheck'
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+
+      - check-style
+
+      - build-11
+      - build-12
+
+      - test-11_checkinstall:
+          requires: [build-11]
+      - test-12_checkinstall:
+          requires: [build-12]

.circleci/run_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euxo pipefail
+IFS=$'\n\t'
+
+status=0
+
+export PGPORT=${PGPORT:-55432}
+
+function cleanup {
+  pg_ctl -D /tmp/postgres stop
+  rm -rf /tmp/postgres
+}
+trap cleanup EXIT
+
+rm -rf /tmp/postgres
+initdb -E unicode /tmp/postgres
+echo "shared_preload_libraries = 'cstore_fdw'" >> /tmp/postgres/postgresql.conf
+pg_ctl -D /tmp/postgres -o "-p ${PGPORT}" -l /tmp/postgres_logfile start || status=$?
+if [ -z $status ]; then cat /tmp/postgres_logfile; fi
+
+make "${@}" || status=$?
+diffs="regression.diffs"
+
+if test -f "${diffs}"; then cat "${diffs}"; fi
+
+exit $status

.gitignore

@@ -60,3 +60,8 @@
 .vscode
 
 *.pb-c.*
+
+# ignore files that could be created by circleci automation
+files.lst
+install-*.tar
+install-*/

.travis.yml

@@ -1,42 +0,0 @@
-sudo: required
-dist: bionic
-language: c
-cache:
-  apt: true
-  directories:
-    - /home/travis/postgresql
-env:
-  global:
-    - enable_coverage=yes
-    - PG_PRELOAD=cstore_fdw
-  matrix:
-    - PGVERSION=9.3
-    - PGVERSION=9.4
-    - PGVERSION=9.5
-    - PGVERSION=9.6
-    - PGVERSION=10
-    - PGVERSION=11
-    - PGVERSION=12
-
-before_install:
-  - git clone -b v0.7.13 --depth 1 https://github.com/citusdata/tools.git
-  - sudo make -C tools install
-  - setup_apt
-  - nuke_pg
-install:
-  - sudo apt-get install protobuf-c-compiler
-  - sudo apt-get install libprotobuf-c0-dev
-  - sudo locale-gen da_DK
-  - sudo locale-gen da_DK.utf8
-  - sudo pip install cpp-coveralls
-  - install_pg
-  - install_custom_pg
-before_script:
-  - chmod 777 .
-  - chmod 777 data
-  - chmod 666 data/*
-  - config_and_start_cluster
-script: pg_travis_test
-after_success:
-  - sudo chmod 666 *.gcda
-  - coveralls --exclude cstore.pb-c.c --exclude cstore.pb-c.h

cstore.h

@@ -204,6 +204,7 @@ typedef struct TableReadState
 	TableMetadata *tableMetadata;
 	TupleDesc tupleDescriptor;
 	Relation relation;
+
 	/*
 	 * List of Var pointers for columns in the query. We use this both for
 	 * getting vector of projected columns, and also when we want to build
@@ -301,7 +302,7 @@ extern StripeSkipList * ReadStripeSkipList(Oid relid, uint64 stripe,
 typedef struct SmgrAddr
 {
 	BlockNumber blockno;
-	uint32		offset;
+	uint32 offset;
 } SmgrAddr;
 
 /*
@@ -320,4 +321,5 @@ logical_to_smgr(uint64 logicalOffset)
 	return addr;
 }
 
+
 #endif /* CSTORE_H */

cstore_fdw.c

@@ -319,14 +319,14 @@ CStoreProcessUtility(Node * parseTree, const char * queryString,
 	}
 	else if (nodeTag(parseTree) == T_DropStmt)
 	{
-		List		*dropRelids = DroppedCStoreRelidList((DropStmt *) parseTree);
-		ListCell	*lc			= NULL;
+		List *dropRelids = DroppedCStoreRelidList((DropStmt *) parseTree);
+		ListCell *lc = NULL;
 
 		/* drop smgr storage */
 		foreach(lc, dropRelids)
 		{
-			Oid				 relid			 = lfirst_oid(lc);
-			Relation		 relation		 = cstore_fdw_open(relid, AccessExclusiveLock);
+			Oid relid = lfirst_oid(lc);
+			Relation relation = cstore_fdw_open(relid, AccessExclusiveLock);
 
 			RelationOpenSmgr(relation);
 			RelationDropStorage(relation);
@@ -334,7 +334,7 @@ CStoreProcessUtility(Node * parseTree, const char * queryString,
 		}
 
 		CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo,
-								destReceiver, completionTag);
+							  destReceiver, completionTag);
 	}
 	else if (nodeTag(parseTree) == T_TruncateStmt)
 	{
@@ -833,6 +833,7 @@ TruncateCStoreTables(List *cstoreRelationList)
 	}
 }
 
+
 /*
  * Version 11 and earlier already assign a relfilenode for foreign
  * tables. Version 12 and later do not, so we need to create one manually.
@@ -840,26 +841,28 @@ TruncateCStoreTables(List *cstoreRelationList)
 static void
 FdwNewRelFileNode(Relation relation)
 {
-	Relation        pg_class;
-	HeapTuple       tuple;
-	Form_pg_class   classform;
+	Relation pg_class;
+	HeapTuple tuple;
+	Form_pg_class classform;
 
 	pg_class = heap_open(RelationRelationId, RowExclusiveLock);
 
 	tuple = SearchSysCacheCopy1(RELOID,
 								ObjectIdGetDatum(RelationGetRelid(relation)));
 	if (!HeapTupleIsValid(tuple))
+	{
 		elog(ERROR, "could not find tuple for relation %u",
 			 RelationGetRelid(relation));
+	}
 	classform = (Form_pg_class) GETSTRUCT(tuple);
 
 	if (true)
 	{
-		char			persistence = relation->rd_rel->relpersistence;
-		Relation		tmprel;
-		Oid				tablespace;
-		Oid				filenode;
-		RelFileNode		newrnode;
+		char persistence = relation->rd_rel->relpersistence;
+		Relation tmprel;
+		Oid tablespace;
+		Oid filenode;
+		RelFileNode newrnode;
 
 		/*
 		 * Upgrade to AccessExclusiveLock, and hold until the end of the
@@ -870,12 +873,18 @@ FdwNewRelFileNode(Relation relation)
 		heap_close(tmprel, NoLock);
 
 		if (OidIsValid(relation->rd_rel->relfilenode))
+		{
 			RelationDropStorage(relation);
+		}
 
 		if (OidIsValid(relation->rd_rel->reltablespace))
+		{
 			tablespace = relation->rd_rel->reltablespace;
+		}
 		else
+		{
 			tablespace = MyDatabaseTableSpace;
+		}
 
 		filenode = GetNewRelFileNode(tablespace, NULL, persistence);
 
@@ -898,6 +907,7 @@ FdwNewRelFileNode(Relation relation)
 	heap_close(pg_class, RowExclusiveLock);
 }
 
+
 static void
 FdwCreateStorage(Relation relation)
 {
@@ -1692,7 +1702,7 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId)
 static void
 CStoreExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState)
 {
-	Relation		 relation = scanState->ss.ss_currentRelation;
+	Relation relation = scanState->ss.ss_currentRelation;
 
 	cstore_fdw_initrel(relation);
 
@@ -2187,16 +2197,22 @@ cstore_fdw_initrel(Relation rel)
 {
 #if PG_VERSION_NUM >= 120000
 	if (rel->rd_rel->relfilenode == InvalidOid)
+	{
 		FdwNewRelFileNode(rel);
+	}
 
 	/*
 	 * Copied code from RelationInitPhysicalAddr(), which doesn't
 	 * work on foreign tables.
 	 */
 	if (OidIsValid(rel->rd_rel->reltablespace))
+	{
 		rel->rd_node.spcNode = rel->rd_rel->reltablespace;
+	}
 	else
+	{
 		rel->rd_node.spcNode = MyDatabaseTableSpace;
+	}
 
 	rel->rd_node.dbNode = MyDatabaseId;
 	rel->rd_node.relNode = rel->rd_rel->relfilenode;
@@ -2204,6 +2220,7 @@ cstore_fdw_initrel(Relation rel)
 	FdwCreateStorage(rel);
 }
 
+
 static Relation
 cstore_fdw_open(Oid relationId, LOCKMODE lockmode)
 {
@@ -2214,6 +2231,7 @@ cstore_fdw_open(Oid relationId, LOCKMODE lockmode)
 	return rel;
 }
 
+
 static Relation
 cstore_fdw_openrv(RangeVar *relation, LOCKMODE lockmode)
 {

cstore_metadata_tables.c

@@ -692,9 +692,12 @@ create_estate_for_relation(Relation rel)
 	estate->es_output_cid = GetCurrentCommandId(true);
 
 #if PG_VERSION_NUM < 120000
+
 	/* Triggers might need a slot */
 	if (resultRelInfo->ri_TrigDesc)
+	{
 		estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate, NULL);
+	}
 #endif
 
 	/* Prepare to catch AFTER triggers. */

cstore_reader.c

@@ -1037,26 +1037,27 @@ ColumnDefaultValue(TupleConstr *tupleConstraints, Form_pg_attribute attributeFor
 	return defaultValue;
 }
 
+
 static StringInfo
 ReadFromSmgr(Relation rel, uint64 offset, uint32 size)
 {
-	StringInfo	resultBuffer = makeStringInfo();
-	uint64		read		 = 0;
+	StringInfo resultBuffer = makeStringInfo();
+	uint64 read = 0;
 
 	enlargeStringInfo(resultBuffer, size);
 	resultBuffer->len = size;
 
 	while (read < size)
 	{
-		Buffer		buffer;
-		Page		page;
-		PageHeader	phdr;
-		uint32		to_read;
-		SmgrAddr	addr = logical_to_smgr(offset + read);
+		Buffer buffer;
+		Page page;
+		PageHeader phdr;
+		uint32 to_read;
+		SmgrAddr addr = logical_to_smgr(offset + read);
 
 		buffer = ReadBuffer(rel, addr.blockno);
 		page = BufferGetPage(buffer);
-		phdr = (PageHeader)page;
+		phdr = (PageHeader) page;
 
 		to_read = Min(size - read, phdr->pd_upper - addr.offset);
 		memcpy(resultBuffer->data + read, page + addr.offset, to_read);
@@ -1067,6 +1068,7 @@ ReadFromSmgr(Relation rel, uint64 offset, uint32 size)
 	return resultBuffer;
 }
 
+
 /*
  * ResetUncompressedBlockData iterates over deserialized column block data
  * and sets valueBuffer field to empty buffer. This field is allocated in stripe

cstore_version_compat.h

@@ -2,7 +2,7 @@
  *
  * cstore_version_compat.h
  *
- * 	Compatibility macros for writing code agnostic to PostgreSQL versions
+ *  Compatibility macros for writing code agnostic to PostgreSQL versions
  *
  * Copyright (c) 2018, Citus Data, Inc.
  *

cstore_writer.c

@@ -363,21 +363,22 @@ CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 blockRowCount,
 	return stripeSkipList;
 }
 
+
 static void
 WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength)
 {
-	uint64		logicalOffset = writeState->currentFileOffset;
-	uint64		remaining	  = dataLength;
-	Relation	rel			  = writeState->relation;
-	Buffer		buffer;
+	uint64 logicalOffset = writeState->currentFileOffset;
+	uint64 remaining = dataLength;
+	Relation rel = writeState->relation;
+	Buffer buffer;
 
 	while (remaining > 0)
 	{
-		SmgrAddr	addr	= logical_to_smgr(logicalOffset);
+		SmgrAddr addr = logical_to_smgr(logicalOffset);
 		BlockNumber nblocks;
-		Page		page;
-		PageHeader	phdr;
-		uint64		to_write;
+		Page page;
+		PageHeader phdr;
+		uint64 to_write;
 
 		RelationOpenSmgr(rel);
 		nblocks = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
@@ -397,7 +398,9 @@ WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength)
 		page = BufferGetPage(buffer);
 		phdr = (PageHeader) page;
 		if (PageIsNew(page))
+		{
 			PageInit(page, BLCKSZ, 0);
+		}
 
 		/* always appending */
 		Assert(phdr->pd_lower == addr.offset);
@@ -434,6 +437,7 @@ WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength)
 	}
 }
 
+
 /*
  * FlushStripe flushes current stripe data into the file. The function first ensures
  * the last data block for each column is properly serialized and compressed. Then,
@@ -832,6 +836,7 @@ AppendStripeMetadata(TableMetadata *tableMetadata, StripeMetadata stripeMetadata
 												stripeMetadataCopy);
 }
 
+
 /*
  * CopyStringInfo creates a deep copy of given source string allocating only needed
  * amount of memory.