diff --git a/.circleci/build.sh b/.circleci/build.sh new file mode 100755 index 000000000..6a9f14c74 --- /dev/null +++ b/.circleci/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euxo pipefail +IFS=$'\n\t' + +status=0 + +basedir="$(pwd)" +installdir="${basedir}/install-${PG_MAJOR}" + +make install DESTDIR="${installdir}" +pushd "${installdir}" +find . -type f -print > "${basedir}/files.lst" +cat "${basedir}/files.lst" +tar cvf "${basedir}/install-${PG_MAJOR}.tar" $(cat "${basedir}/files.lst") +popd diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..9f2532c1d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,101 @@ +version: 2.1 +orbs: + codecov: codecov/codecov@1.1.1 + +jobs: + check-style: + docker: + - image: 'citus/stylechecker:latest' + steps: + - checkout + - run: + name: 'Check Style' + command: | + citus_indent --check + - run: + name: 'Check if whitespace fixing changed anything, install editorconfig if it did' + command: | + git diff --exit-code + + build-11: + docker: + - image: 'citus/extbuilder:11.9' + steps: + - checkout + - run: + name: 'Configure, Build, and Install' + command: | + PG_MAJOR=11 .circleci/build.sh + - persist_to_workspace: + root: . + paths: + - install-11.tar + + build-12: + docker: + - image: 'citus/extbuilder:12.4' + steps: + - checkout + - run: + name: 'Configure, Build, and Install' + command: | + PG_MAJOR=12 .circleci/build.sh + - persist_to_workspace: + root: . + paths: + - install-12.tar + + test-11_checkinstall: + docker: + - image: 'citus/exttester:11.9' + working_directory: /home/circleci/project + steps: + - checkout + - attach_workspace: + at: . + - run: + name: 'Prepare Container & Install Extension' + command: | + chown -R circleci:circleci /home/circleci + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / + - run: + name: 'Run Test' + command: | + gosu circleci .circleci/run_test.sh installcheck + - codecov/upload: + flags: 'test_11,installcheck' + + test-12_checkinstall: + docker: + - image: 'citus/exttester:12.4' + working_directory: /home/circleci/project + steps: + - checkout + - attach_workspace: + at: . + - run: + name: 'Prepare Container & Install Extension' + command: | + chown -R circleci:circleci /home/circleci + tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / + - run: + name: 'Run Test' + command: | + gosu circleci .circleci/run_test.sh installcheck + - codecov/upload: + flags: 'test_12,installcheck' + +workflows: + version: 2 + build_and_test: + jobs: + + - check-style + + - build-11 + - build-12 + + - test-11_checkinstall: + requires: [build-11] + - test-12_checkinstall: + requires: [build-12] diff --git a/.circleci/run_test.sh b/.circleci/run_test.sh new file mode 100755 index 000000000..f9e183b56 --- /dev/null +++ b/.circleci/run_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euxo pipefail +IFS=$'\n\t' + +status=0 + +export PGPORT=${PGPORT:-55432} + +function cleanup { + pg_ctl -D /tmp/postgres stop + rm -rf /tmp/postgres +} +trap cleanup EXIT + +rm -rf /tmp/postgres +initdb -E unicode /tmp/postgres +echo "shared_preload_libraries = 'cstore_fdw'" >> /tmp/postgres/postgresql.conf +pg_ctl -D /tmp/postgres -o "-p ${PGPORT}" -l /tmp/postgres_logfile start || status=$? +if [ -z $status ]; then cat /tmp/postgres_logfile; fi + +make "${@}" || status=$? +diffs="regression.diffs" + +if test -f "${diffs}"; then cat "${diffs}"; fi + +exit $status diff --git a/.gitignore b/.gitignore index 21c5e32ea..0c643e590 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,8 @@ .vscode *.pb-c.* + +# ignore files that could be created by circleci automation +files.lst +install-*.tar +install-*/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f83f7206e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -sudo: required -dist: bionic -language: c -cache: - apt: true - directories: - - /home/travis/postgresql -env: - global: - - enable_coverage=yes - - PG_PRELOAD=cstore_fdw - matrix: - - PGVERSION=9.3 - - PGVERSION=9.4 - - PGVERSION=9.5 - - PGVERSION=9.6 - - PGVERSION=10 - - PGVERSION=11 - - PGVERSION=12 - -before_install: - - git clone -b v0.7.13 --depth 1 https://github.com/citusdata/tools.git - - sudo make -C tools install - - setup_apt - - nuke_pg -install: - - sudo apt-get install protobuf-c-compiler - - sudo apt-get install libprotobuf-c0-dev - - sudo locale-gen da_DK - - sudo locale-gen da_DK.utf8 - - sudo pip install cpp-coveralls - - install_pg - - install_custom_pg -before_script: - - chmod 777 . - - chmod 777 data - - chmod 666 data/* - - config_and_start_cluster -script: pg_travis_test -after_success: - - sudo chmod 666 *.gcda - - coveralls --exclude cstore.pb-c.c --exclude cstore.pb-c.h diff --git a/cstore.h b/cstore.h index c7f849cc9..87b552bbf 100644 --- a/cstore.h +++ b/cstore.h @@ -204,6 +204,7 @@ typedef struct TableReadState TableMetadata *tableMetadata; TupleDesc tupleDescriptor; Relation relation; + /* * List of Var pointers for columns in the query. We use this both for * getting vector of projected columns, and also when we want to build @@ -301,7 +302,7 @@ extern StripeSkipList * ReadStripeSkipList(Oid relid, uint64 stripe, typedef struct SmgrAddr { BlockNumber blockno; - uint32 offset; + uint32 offset; } SmgrAddr; /* @@ -320,4 +321,5 @@ logical_to_smgr(uint64 logicalOffset) return addr; } + #endif /* CSTORE_H */ diff --git a/cstore_fdw.c b/cstore_fdw.c index 36f576cee..512dee5a3 100644 --- a/cstore_fdw.c +++ b/cstore_fdw.c @@ -319,14 +319,14 @@ CStoreProcessUtility(Node * parseTree, const char * queryString, } else if (nodeTag(parseTree) == T_DropStmt) { - List *dropRelids = DroppedCStoreRelidList((DropStmt *) parseTree); - ListCell *lc = NULL; + List *dropRelids = DroppedCStoreRelidList((DropStmt *) parseTree); + ListCell *lc = NULL; /* drop smgr storage */ foreach(lc, dropRelids) { - Oid relid = lfirst_oid(lc); - Relation relation = cstore_fdw_open(relid, AccessExclusiveLock); + Oid relid = lfirst_oid(lc); + Relation relation = cstore_fdw_open(relid, AccessExclusiveLock); RelationOpenSmgr(relation); RelationDropStorage(relation); @@ -334,7 +334,7 @@ CStoreProcessUtility(Node * parseTree, const char * queryString, } CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, - destReceiver, completionTag); + destReceiver, completionTag); } else if (nodeTag(parseTree) == T_TruncateStmt) { @@ -833,6 +833,7 @@ TruncateCStoreTables(List *cstoreRelationList) } } + /* * Version 11 and earlier already assign a relfilenode for foreign * tables. Version 12 and later do not, so we need to create one manually. @@ -840,26 +841,28 @@ TruncateCStoreTables(List *cstoreRelationList) static void FdwNewRelFileNode(Relation relation) { - Relation pg_class; - HeapTuple tuple; - Form_pg_class classform; + Relation pg_class; + HeapTuple tuple; + Form_pg_class classform; pg_class = heap_open(RelationRelationId, RowExclusiveLock); tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(RelationGetRelid(relation))); if (!HeapTupleIsValid(tuple)) + { elog(ERROR, "could not find tuple for relation %u", RelationGetRelid(relation)); + } classform = (Form_pg_class) GETSTRUCT(tuple); if (true) { - char persistence = relation->rd_rel->relpersistence; - Relation tmprel; - Oid tablespace; - Oid filenode; - RelFileNode newrnode; + char persistence = relation->rd_rel->relpersistence; + Relation tmprel; + Oid tablespace; + Oid filenode; + RelFileNode newrnode; /* * Upgrade to AccessExclusiveLock, and hold until the end of the @@ -870,12 +873,18 @@ FdwNewRelFileNode(Relation relation) heap_close(tmprel, NoLock); if (OidIsValid(relation->rd_rel->relfilenode)) + { RelationDropStorage(relation); + } if (OidIsValid(relation->rd_rel->reltablespace)) + { tablespace = relation->rd_rel->reltablespace; + } else + { tablespace = MyDatabaseTableSpace; + } filenode = GetNewRelFileNode(tablespace, NULL, persistence); @@ -898,6 +907,7 @@ FdwNewRelFileNode(Relation relation) heap_close(pg_class, RowExclusiveLock); } + static void FdwCreateStorage(Relation relation) { @@ -1692,7 +1702,7 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId) static void CStoreExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) { - Relation relation = scanState->ss.ss_currentRelation; + Relation relation = scanState->ss.ss_currentRelation; cstore_fdw_initrel(relation); @@ -2187,16 +2197,22 @@ cstore_fdw_initrel(Relation rel) { #if PG_VERSION_NUM >= 120000 if (rel->rd_rel->relfilenode == InvalidOid) + { FdwNewRelFileNode(rel); + } /* * Copied code from RelationInitPhysicalAddr(), which doesn't * work on foreign tables. */ if (OidIsValid(rel->rd_rel->reltablespace)) + { rel->rd_node.spcNode = rel->rd_rel->reltablespace; + } else + { rel->rd_node.spcNode = MyDatabaseTableSpace; + } rel->rd_node.dbNode = MyDatabaseId; rel->rd_node.relNode = rel->rd_rel->relfilenode; @@ -2204,6 +2220,7 @@ cstore_fdw_initrel(Relation rel) FdwCreateStorage(rel); } + static Relation cstore_fdw_open(Oid relationId, LOCKMODE lockmode) { @@ -2214,6 +2231,7 @@ cstore_fdw_open(Oid relationId, LOCKMODE lockmode) return rel; } + static Relation cstore_fdw_openrv(RangeVar *relation, LOCKMODE lockmode) { diff --git a/cstore_metadata_tables.c b/cstore_metadata_tables.c index af0eb96c4..8a67a3a9e 100644 --- a/cstore_metadata_tables.c +++ b/cstore_metadata_tables.c @@ -692,9 +692,12 @@ create_estate_for_relation(Relation rel) estate->es_output_cid = GetCurrentCommandId(true); #if PG_VERSION_NUM < 120000 + /* Triggers might need a slot */ if (resultRelInfo->ri_TrigDesc) + { estate->es_trig_tuple_slot = ExecInitExtraTupleSlot(estate, NULL); + } #endif /* Prepare to catch AFTER triggers. */ diff --git a/cstore_reader.c b/cstore_reader.c index 929c65c04..fecb45605 100644 --- a/cstore_reader.c +++ b/cstore_reader.c @@ -1037,26 +1037,27 @@ ColumnDefaultValue(TupleConstr *tupleConstraints, Form_pg_attribute attributeFor return defaultValue; } + static StringInfo ReadFromSmgr(Relation rel, uint64 offset, uint32 size) { - StringInfo resultBuffer = makeStringInfo(); - uint64 read = 0; + StringInfo resultBuffer = makeStringInfo(); + uint64 read = 0; enlargeStringInfo(resultBuffer, size); resultBuffer->len = size; while (read < size) { - Buffer buffer; - Page page; - PageHeader phdr; - uint32 to_read; - SmgrAddr addr = logical_to_smgr(offset + read); + Buffer buffer; + Page page; + PageHeader phdr; + uint32 to_read; + SmgrAddr addr = logical_to_smgr(offset + read); buffer = ReadBuffer(rel, addr.blockno); page = BufferGetPage(buffer); - phdr = (PageHeader)page; + phdr = (PageHeader) page; to_read = Min(size - read, phdr->pd_upper - addr.offset); memcpy(resultBuffer->data + read, page + addr.offset, to_read); @@ -1067,6 +1068,7 @@ ReadFromSmgr(Relation rel, uint64 offset, uint32 size) return resultBuffer; } + /* * ResetUncompressedBlockData iterates over deserialized column block data * and sets valueBuffer field to empty buffer. This field is allocated in stripe diff --git a/cstore_version_compat.h b/cstore_version_compat.h index 95521c1aa..3d1a60f93 100644 --- a/cstore_version_compat.h +++ b/cstore_version_compat.h @@ -2,7 +2,7 @@ * * cstore_version_compat.h * - * Compatibility macros for writing code agnostic to PostgreSQL versions + * Compatibility macros for writing code agnostic to PostgreSQL versions * * Copyright (c) 2018, Citus Data, Inc. * diff --git a/cstore_writer.c b/cstore_writer.c index 5e44812bd..55a314ec4 100644 --- a/cstore_writer.c +++ b/cstore_writer.c @@ -363,21 +363,22 @@ CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 blockRowCount, return stripeSkipList; } + static void WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength) { - uint64 logicalOffset = writeState->currentFileOffset; - uint64 remaining = dataLength; - Relation rel = writeState->relation; - Buffer buffer; + uint64 logicalOffset = writeState->currentFileOffset; + uint64 remaining = dataLength; + Relation rel = writeState->relation; + Buffer buffer; while (remaining > 0) { - SmgrAddr addr = logical_to_smgr(logicalOffset); + SmgrAddr addr = logical_to_smgr(logicalOffset); BlockNumber nblocks; - Page page; - PageHeader phdr; - uint64 to_write; + Page page; + PageHeader phdr; + uint64 to_write; RelationOpenSmgr(rel); nblocks = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM); @@ -397,7 +398,9 @@ WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength) page = BufferGetPage(buffer); phdr = (PageHeader) page; if (PageIsNew(page)) + { PageInit(page, BLCKSZ, 0); + } /* always appending */ Assert(phdr->pd_lower == addr.offset); @@ -434,6 +437,7 @@ WriteToSmgr(TableWriteState *writeState, char *data, uint32 dataLength) } } + /* * FlushStripe flushes current stripe data into the file. The function first ensures * the last data block for each column is properly serialized and compressed. Then, @@ -832,6 +836,7 @@ AppendStripeMetadata(TableMetadata *tableMetadata, StripeMetadata stripeMetadata stripeMetadataCopy); } + /* * CopyStringInfo creates a deep copy of given source string allocating only needed * amount of memory.