diff --git a/Makefile.global.in b/Makefile.global.in index 8c7d7f3d1..aba3be6f6 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -92,5 +92,8 @@ endif override CPPFLAGS := @CPPFLAGS@ @CITUS_CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS) override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@ +USE_FDW:=@USE_FDW@ +USE_TABLEAM:=@USE_TABLEAM@ + # optional file with user defined, additional, rules -include ${citus_abs_srcdir}/src/Makefile.custom diff --git a/configure b/configure index 9d088c3e0..e29228855 100755 --- a/configure +++ b/configure @@ -622,6 +622,8 @@ ac_includes_default="\ ac_subst_vars='LTLIBOBJS LIBOBJS +USE_TABLEAM +USE_FDW HAS_DOTGIT POSTGRES_BUILDDIR POSTGRES_SRCDIR @@ -690,6 +692,8 @@ with_extra_version enable_coverage with_libcurl with_reports_hostname +with_columnar_tableam +with_columnar_fdw ' ac_precious_vars='build_alias host_alias @@ -1323,6 +1327,9 @@ Optional Packages: --with-reports-hostname=HOSTNAME Use HOSTNAME as hostname for statistics collection and update checks + --without-columnar-tableam + compile with table access methods for columnar + --without-columnar-fdw compile with foreign data wrappers for columnar Some influential environment variables: PG_CONFIG Location to find pg_config for target PostgreSQL instalation @@ -4468,6 +4475,72 @@ cat >>confdefs.h <<_ACEOF _ACEOF +if test "$version_num" != '11'; then + + + +# Check whether --with-columnar-tableam was given. +if test "${with_columnar_tableam+set}" = set; then : + withval=$with_columnar_tableam; + case $withval in + yes) + USE_TABLEAM=yes + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-columnar-tableam option" "$LINENO" 5 + ;; + esac + +else + with_columnar_tableam=yes +USE_TABLEAM=yes +fi + + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: postgres does not support table access methodds" >&5 +$as_echo "$as_me: postgres does not support table access methodds" >&6;} +fi; + +if test "$USE_TABLEAM" = yes; then + +$as_echo "#define USE_TABLEAM 1" >>confdefs.h + +fi; + + + + +# Check whether --with-columnar-fdw was given. +if test "${with_columnar_fdw+set}" = set; then : + withval=$with_columnar_fdw; + case $withval in + yes) + USE_FDW=yes + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-columnar-fdw option" "$LINENO" 5 + ;; + esac + +else + with_columnar_fdw=yes +USE_FDW=yes +fi + + +if test "$USE_FDW" = yes; then + +$as_echo "#define USE_FDW 1" >>confdefs.h + +fi; + # Check if git is installed, when installed the gitref of the checkout will be baked in the application # Extract the first word of "git", so it can be a program name with args. set dummy git; ac_word=$2 @@ -4543,6 +4616,10 @@ POSTGRES_BUILDDIR="$POSTGRES_BUILDDIR" HAS_DOTGIT="$HAS_DOTGIT" +USE_FDW="$USE_FDW" + +USE_TABLEAM="$USE_TABLEAM" + ac_config_files="$ac_config_files Makefile.global" diff --git a/configure.in b/configure.in index 58bcc906e..b31fc4ff5 100644 --- a/configure.in +++ b/configure.in @@ -212,6 +212,25 @@ PGAC_ARG_REQ(with, reports-hostname, [HOSTNAME], AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL", [Base URL for statistics collection and update checks]) +if test "$version_num" != '11'; then + PGAC_ARG_BOOL(with, columnar-tableam, yes, + [compile with table access methods for columnar], + [USE_TABLEAM=yes]) +else + AC_MSG_NOTICE([postgres does not support table access methodds]) +fi; + +if test "$USE_TABLEAM" = yes; then + AC_DEFINE([USE_TABLEAM], 1, [Define to 1 to build with table access method support. (--with-columnar-tableam)]) +fi; + +PGAC_ARG_BOOL(with, columnar-fdw, yes, + [compile with foreign data wrappers for columnar], + [USE_FDW=yes]) +if test "$USE_FDW" = yes; then + AC_DEFINE([USE_FDW], 1, [Define to 1 to build with foreign datawrapper support. (--with-columnar-fdw)]) +fi; + # Check if git is installed, when installed the gitref of the checkout will be baked in the application AC_PATH_PROG(GIT_BIN, git) AC_CHECK_FILE(.git,[HAS_DOTGIT=yes], [HAS_DOTGIT=]) @@ -222,6 +241,8 @@ AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS") AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") AC_SUBST(HAS_DOTGIT, "$HAS_DOTGIT") +AC_SUBST(USE_FDW, "$USE_FDW") +AC_SUBST(USE_TABLEAM, "$USE_TABLEAM") AC_CONFIG_FILES([Makefile.global]) AC_CONFIG_HEADERS([src/include/citus_config.h] [src/include/citus_version.h]) diff --git a/src/backend/columnar/.circleci/build.sh b/src/backend/columnar/.circleci/build.sh deleted file mode 100755 index 6a9f14c74..000000000 --- a/src/backend/columnar/.circleci/build.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euxo pipefail -IFS=$'\n\t' - -status=0 - -basedir="$(pwd)" -installdir="${basedir}/install-${PG_MAJOR}" - -make install DESTDIR="${installdir}" -pushd "${installdir}" -find . -type f -print > "${basedir}/files.lst" -cat "${basedir}/files.lst" -tar cvf "${basedir}/install-${PG_MAJOR}.tar" $(cat "${basedir}/files.lst") -popd diff --git a/src/backend/columnar/.circleci/config.yml b/src/backend/columnar/.circleci/config.yml deleted file mode 100644 index 645211182..000000000 --- a/src/backend/columnar/.circleci/config.yml +++ /dev/null @@ -1,138 +0,0 @@ -version: 2.1 -orbs: - codecov: codecov/codecov@1.1.1 - -jobs: - check-style: - docker: - - image: 'citus/stylechecker:latest' - steps: - - checkout - - run: - name: 'Check Style' - command: | - citus_indent --check - - run: - name: 'Check if whitespace fixing changed anything, install editorconfig if it did' - command: | - git diff --exit-code - - build-11: - docker: - - image: 'citus/extbuilder:11.9' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - PG_MAJOR=11 .circleci/build.sh - - persist_to_workspace: - root: . - paths: - - install-11.tar - - build-12: - docker: - - image: 'citus/extbuilder:12.4' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - PG_MAJOR=12 .circleci/build.sh - - persist_to_workspace: - root: . - paths: - - install-12.tar - - build-13: - docker: - - image: 'citus/extbuilder:13.0' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - PG_MAJOR=13 .circleci/build.sh - - persist_to_workspace: - root: . - paths: - - install-13.tar - - test-11_checkinstall: - docker: - - image: 'citus/exttester:11.9' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Prepare Container & Install Extension' - command: | - chown -R circleci:circleci /home/circleci - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Run Test' - command: | - gosu circleci .circleci/run_test.sh installcheck - - codecov/upload: - flags: 'test_11,installcheck' - - test-12_checkinstall: - docker: - - image: 'citus/exttester:12.4' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Prepare Container & Install Extension' - command: | - chown -R circleci:circleci /home/circleci - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Run Test' - command: | - gosu circleci .circleci/run_test.sh installcheck - - codecov/upload: - flags: 'test_12,installcheck' - - test-13_checkinstall: - docker: - - image: 'citus/exttester:13.0' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Prepare Container & Install Extension' - command: | - chown -R circleci:circleci /home/circleci - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-${PG_MAJOR}.tar" --directory / - - run: - name: 'Run Test' - command: | - gosu circleci .circleci/run_test.sh installcheck - - codecov/upload: - flags: 'test_13,installcheck' - -workflows: - version: 2 - build_and_test: - jobs: - - - check-style - - - build-11 - - build-12 - - build-13 - - - test-11_checkinstall: - requires: [build-11] - - test-12_checkinstall: - requires: [build-12] - - test-13_checkinstall: - requires: [build-13] diff --git a/src/backend/columnar/.circleci/run_test.sh b/src/backend/columnar/.circleci/run_test.sh deleted file mode 100755 index f9e183b56..000000000 --- a/src/backend/columnar/.circleci/run_test.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -euxo pipefail -IFS=$'\n\t' - -status=0 - -export PGPORT=${PGPORT:-55432} - -function cleanup { - pg_ctl -D /tmp/postgres stop - rm -rf /tmp/postgres -} -trap cleanup EXIT - -rm -rf /tmp/postgres -initdb -E unicode /tmp/postgres -echo "shared_preload_libraries = 'cstore_fdw'" >> /tmp/postgres/postgresql.conf -pg_ctl -D /tmp/postgres -o "-p ${PGPORT}" -l /tmp/postgres_logfile start || status=$? -if [ -z $status ]; then cat /tmp/postgres_logfile; fi - -make "${@}" || status=$? -diffs="regression.diffs" - -if test -f "${diffs}"; then cat "${diffs}"; fi - -exit $status diff --git a/src/backend/columnar/LICENSE b/src/backend/columnar/LICENSE deleted file mode 100644 index ad410e113..000000000 --- a/src/backend/columnar/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/src/backend/columnar/META.json b/src/backend/columnar/META.json deleted file mode 100644 index 19e819daa..000000000 --- a/src/backend/columnar/META.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "cstore_fdw", - "abstract": "Columnar Store for PostgreSQL", - "description": "PostgreSQL extension which implements a Columnar Store.", - "version": "1.7.0", - "maintainer": "Murat Tuncer ", - "license": "apache_2_0", - "provides": { - "cstore_fdw": { - "abstract": "Foreign Data Wrapper for Columnar Store Tables", - "file": "cstore_fdw--1.7.sql", - "docfile": "README.md", - "version": "1.7.0" - } - }, - "prereqs": { - "runtime": { - "requires": { - "PostgreSQL": "9.3.0" - } - } - }, - "resources": { - "bugtracker": { - "web": "http://github.com/citusdata/cstore_fdw/issues/" - }, - "repository": { - "url": "git://github.com/citusdata/cstore_fdw.git", - "web": "https://github.com/citusdata/cstore_fdw/", - "type": "git" - } - }, - "generated_by": "Murat Tuncer", - "meta-spec": { - "version": "1.0.0", - "url": "http://pgxn.org/meta/spec.txt" - }, - "tags": [ - "orc", - "fdw", - "foreign data wrapper", - "cstore_fdw", - "columnar store" - ] -} diff --git a/src/backend/columnar/Makefile b/src/backend/columnar/Makefile deleted file mode 100644 index 6be7bbd45..000000000 --- a/src/backend/columnar/Makefile +++ /dev/null @@ -1,102 +0,0 @@ -# cstore_fdw/Makefile -# -# Copyright (c) 2016 Citus Data, Inc. -# - -MODULE_big = cstore_fdw - -VER := $(lastword $(shell pg_config --version)) -VER_WORDS = $(subst ., ,$(VER)) -MVER = $(firstword $(VER_WORDS)) - -# error for versions earlier than 10 so that lex comparison will work -ifneq ($(shell printf '%02d' $(MVER)),$(MVER)) -$(error version $(VER) not supported) -endif - -# lexicographic comparison of version number -ifeq ($(lastword $(sort 12 $(MVER))),$(MVER)) - USE_TABLEAM = yes - USE_FDW = yes -else ifeq ($(lastword $(sort 11 $(MVER))),$(MVER)) - USE_TABLEAM = no - USE_FDW = yes -else -$(error version $(VER) is not supported) -endif - -PG_CFLAGS = -std=c11 -Wshadow -Werror -OBJS = cstore.o cstore_writer.o cstore_reader.o \ - cstore_compression.o mod.o cstore_metadata_tables.o - -EXTENSION = cstore_fdw -DATA = cstore_fdw--1.7.sql cstore_fdw--1.6--1.7.sql cstore_fdw--1.5--1.6.sql cstore_fdw--1.4--1.5.sql \ - cstore_fdw--1.3--1.4.sql cstore_fdw--1.2--1.3.sql cstore_fdw--1.1--1.2.sql \ - cstore_fdw--1.0--1.1.sql cstore_fdw--1.7--1.8.sql - -REGRESS = extension_create -ISOLATION = create -EXTRA_CLEAN = sql/fdw_block_filtering.sql sql/fdw_create.sql sql/fdw_data_types.sql sql/fdw_load.sql \ - sql/fdw_copyto.sql expected/fdw_block_filtering.out expected/fdw_create.out \ - expected/fdw_data_types.out expected/fdw_load.out expected/fdw_copyto.out \ - sql/am_block_filtering.sql sql/am_create.sql sql/am_data_types.sql sql/am_load.sql \ - sql/am_copyto.sql expected/am_block_filtering.out \ - expected/am_data_types.out expected/am_load.out expected/am_copyto.out - -ifeq ($(USE_FDW),yes) - PG_CFLAGS += -DUSE_FDW - OBJS += cstore_fdw.o - REGRESS += fdw_create fdw_load fdw_query fdw_analyze fdw_data_types \ - fdw_functions fdw_block_filtering fdw_drop fdw_insert \ - fdw_copyto fdw_alter fdw_rollback fdw_truncate fdw_clean -endif - -ifeq ($(USE_TABLEAM),yes) - PG_CFLAGS += -DUSE_TABLEAM - OBJS += cstore_tableam.o cstore_customscan.o - REGRESS += am_create am_load am_query am_analyze am_data_types am_functions \ - am_drop am_insert am_copyto am_alter am_rollback am_truncate am_vacuum am_clean \ - am_block_filtering am_join am_trigger am_tableoptions - ISOLATION += am_write_concurrency am_vacuum_vs_insert -endif - -ifeq ($(enable_coverage),yes) - PG_CPPFLAGS += --coverage - SHLIB_LINK += --coverage - EXTRA_CLEAN += *.gcno -endif - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin) - PG_CPPFLAGS += -I/usr/local/include -endif - -# -# Users need to specify their Postgres installation path through pg_config. For -# example: /usr/local/pgsql/bin/pg_config or /usr/lib/postgresql/9.3/bin/pg_config -# - -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) - -# command for getting postgres source directory is taken from citus/configure.in -POSTGRES_SRCDIR=$(shell grep ^abs_top_srcdir $(shell dirname $(shell $(PG_CONFIG) --pgxs))/../Makefile.global|cut -d ' ' -f3-) -PGXS_ISOLATION_TESTER=$(top_builddir)/src/test/isolation/pg_isolation_regress - -# If postgres installation doesn't include pg_isolation_regress, try using the -# one in postgres source directory. -ifeq (,$(wildcard $(PGXS_ISOLATION_TESTER))) - pg_isolation_regress_installcheck = \ - $(POSTGRES_SRCDIR)/src/test/isolation/pg_isolation_regress \ - --inputdir=$(srcdir) $(EXTRA_REGRESS_OPTS) -else - pg_isolation_regress_installcheck = \ - $(PGXS_ISOLATION_TESTER) \ - --inputdir=$(srcdir) $(EXTRA_REGRESS_OPTS) -endif - -installcheck: - -reindent: - citus_indent . diff --git a/src/backend/columnar/TODO.md b/src/backend/columnar/TODO.md deleted file mode 100644 index 179fbc8c7..000000000 --- a/src/backend/columnar/TODO.md +++ /dev/null @@ -1,41 +0,0 @@ -To see the list of features and bug-fixes planned for next releases, see our -[development roadmap][roadmap]. - -Requested Features ------------------- - -* Improve write performance -* Improve read performance -* Add checksum logic -* Add new compression methods -* Enable INSERT/DELETE/UPDATE -* Enable users other than superuser to safely create columnar tables (permissions) -* Transactional semantics -* Add config setting to make pg\_fsync() optional - - -Known Issues ------------- - -* Copy command ignores NOT NULL constraints. -* Planning functions don't take into account average column width. -* Planning functions don't correctly take into account block skipping benefits. -* On 32-bit platforms, when file size is outside the 32-bit signed range, EXPLAIN - command prints incorrect file size. -* If two different columnar tables are configured to point to the same file, - writes to the underlying file aren't protected from each other. -* When a data load is in progress, concurrent reads on the table overestimate the - page count. -* We have a minor memory leak in CStoreEndWrite. We need to also free the - comparisonFunctionArray. -* block\_filtering test fails on Ubuntu because the "da\_DK" locale is not enabled - by default. -* We don't yet incorporate the compression method's impact on disk I/O into cost - estimates. -* CitusDB integration errors: -* Concurrent staging cstore\_fdw tables doesn't work. -* Setting a default value for column with ALTER TABLE has limited support for - existing rows. - -[roadmap]: https://github.com/citusdata/cstore_fdw/wiki/Roadmap - diff --git a/src/backend/columnar/cstore_customscan.c b/src/backend/columnar/cstore_customscan.c index 075061caf..5e2d4c461 100644 --- a/src/backend/columnar/cstore_customscan.c +++ b/src/backend/columnar/cstore_customscan.c @@ -10,6 +10,9 @@ *------------------------------------------------------------------------- */ +#include "citus_version.h" +#if USE_TABLEAM + #include "postgres.h" #include "access/skey.h" @@ -431,3 +434,6 @@ CStoreScan_ReScanCustomScan(CustomScanState *node) table_rescan(node->ss.ss_currentScanDesc, NULL); } } + + +#endif /* USE_TABLEAM */ diff --git a/src/backend/columnar/cstore_fdw--1.0--1.1.sql b/src/backend/columnar/cstore_fdw--1.0--1.1.sql deleted file mode 100644 index 9e8029638..000000000 --- a/src/backend/columnar/cstore_fdw--1.0--1.1.sql +++ /dev/null @@ -1,26 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.0--1.1.sql */ - --- complain if script is sourced in psql, rather than via ALTER EXTENSION UPDATE -\echo Use "ALTER EXTENSION cstore_fdw UPDATE TO '1.1'" to load this file. \quit - -CREATE FUNCTION cstore_ddl_event_end_trigger() -RETURNS event_trigger -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - -CREATE EVENT TRIGGER cstore_ddl_event_end -ON ddl_command_end -EXECUTE PROCEDURE cstore_ddl_event_end_trigger(); - -CREATE FUNCTION cstore_table_size(relation regclass) -RETURNS bigint -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - --- cstore_fdw creates directories to store files for tables with automatically --- determined filename during the CREATE SERVER statement. Since this feature --- was newly added in v1.1, servers created with v1.0 did not create them. So, --- we create a server with v1.1 to ensure that the required directories are --- created to allow users to create automatically managed tables with old servers. -CREATE SERVER cstore_server_for_updating_1_0_to_1_1 FOREIGN DATA WRAPPER cstore_fdw; -DROP SERVER cstore_server_for_updating_1_0_to_1_1; diff --git a/src/backend/columnar/cstore_fdw--1.1--1.2.sql b/src/backend/columnar/cstore_fdw--1.1--1.2.sql deleted file mode 100644 index 6cabb8c5e..000000000 --- a/src/backend/columnar/cstore_fdw--1.1--1.2.sql +++ /dev/null @@ -1,3 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.1--1.2.sql */ - --- No new functions or definitions were added in 1.2 diff --git a/src/backend/columnar/cstore_fdw--1.2--1.3.sql b/src/backend/columnar/cstore_fdw--1.2--1.3.sql deleted file mode 100644 index 3ad187d09..000000000 --- a/src/backend/columnar/cstore_fdw--1.2--1.3.sql +++ /dev/null @@ -1,3 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.2--1.3.sql */ - --- No new functions or definitions were added in 1.3 diff --git a/src/backend/columnar/cstore_fdw--1.3--1.4.sql b/src/backend/columnar/cstore_fdw--1.3--1.4.sql deleted file mode 100644 index 3b7b0f150..000000000 --- a/src/backend/columnar/cstore_fdw--1.3--1.4.sql +++ /dev/null @@ -1,3 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.3--1.4.sql */ - --- No new functions or definitions were added in 1.4 diff --git a/src/backend/columnar/cstore_fdw--1.4--1.5.sql b/src/backend/columnar/cstore_fdw--1.4--1.5.sql deleted file mode 100644 index 55bbb0b2a..000000000 --- a/src/backend/columnar/cstore_fdw--1.4--1.5.sql +++ /dev/null @@ -1,28 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.4--1.5.sql */ - -CREATE FUNCTION cstore_clean_table_resources(oid) -RETURNS void -AS 'MODULE_PATHNAME' -LANGUAGE C STRICT; - -CREATE OR REPLACE FUNCTION cstore_drop_trigger() - RETURNS event_trigger - LANGUAGE plpgsql - AS $csdt$ -DECLARE v_obj record; -BEGIN - FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP - - IF v_obj.object_type NOT IN ('table', 'foreign table') THEN - CONTINUE; - END IF; - - PERFORM cstore_clean_table_resources(v_obj.objid); - - END LOOP; -END; -$csdt$; - -CREATE EVENT TRIGGER cstore_drop_event - ON SQL_DROP - EXECUTE PROCEDURE cstore_drop_trigger(); diff --git a/src/backend/columnar/cstore_fdw--1.5--1.6.sql b/src/backend/columnar/cstore_fdw--1.5--1.6.sql deleted file mode 100644 index c8f7e8097..000000000 --- a/src/backend/columnar/cstore_fdw--1.5--1.6.sql +++ /dev/null @@ -1,19 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.5--1.6.sql */ - -CREATE OR REPLACE FUNCTION cstore_drop_trigger() - RETURNS event_trigger - LANGUAGE plpgsql - AS $csdt$ -DECLARE v_obj record; -BEGIN - FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP - - IF v_obj.object_type NOT IN ('table', 'foreign table') THEN - CONTINUE; - END IF; - - PERFORM public.cstore_clean_table_resources(v_obj.objid); - - END LOOP; -END; -$csdt$; diff --git a/src/backend/columnar/cstore_fdw--1.6--1.7.sql b/src/backend/columnar/cstore_fdw--1.6--1.7.sql deleted file mode 100644 index c7f56f059..000000000 --- a/src/backend/columnar/cstore_fdw--1.6--1.7.sql +++ /dev/null @@ -1,3 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.6--1.6.sql */ - --- No new functions or definitions were added in 1.7 diff --git a/src/backend/columnar/cstore_fdw--1.7--1.8.sql b/src/backend/columnar/cstore_fdw--1.7--1.8.sql deleted file mode 100644 index 81cbadfb4..000000000 --- a/src/backend/columnar/cstore_fdw--1.7--1.8.sql +++ /dev/null @@ -1,35 +0,0 @@ -/* cstore_fdw/cstore_fdw--1.7--1.8.sql */ - -DO $proc$ -BEGIN - -IF version() ~ '12' or version() ~ '13' THEN - EXECUTE $$ - CREATE FUNCTION cstore_tableam_handler(internal) - RETURNS table_am_handler - LANGUAGE C - AS 'MODULE_PATHNAME', 'cstore_tableam_handler'; - - CREATE ACCESS METHOD cstore_tableam - TYPE TABLE HANDLER cstore_tableam_handler; - - CREATE FUNCTION pg_catalog.alter_cstore_table_set( - table_name regclass, - block_row_count int DEFAULT NULL, - stripe_row_count int DEFAULT NULL, - compression name DEFAULT null) - RETURNS void - LANGUAGE C - AS 'MODULE_PATHNAME', 'alter_cstore_table_set'; - - CREATE FUNCTION pg_catalog.alter_cstore_table_reset( - table_name regclass, - block_row_count bool DEFAULT false, - stripe_row_count bool DEFAULT false, - compression bool DEFAULT false) - RETURNS void - LANGUAGE C - AS 'MODULE_PATHNAME', 'alter_cstore_table_reset'; - $$; -END IF; -END$proc$; diff --git a/src/backend/columnar/cstore_fdw.control b/src/backend/columnar/cstore_fdw.control deleted file mode 100644 index 57fd0808a..000000000 --- a/src/backend/columnar/cstore_fdw.control +++ /dev/null @@ -1,6 +0,0 @@ -# cstore_fdw extension -comment = 'foreign-data wrapper for flat cstore access' -default_version = '1.8' -module_pathname = '$libdir/cstore_fdw' -relocatable = false -schema = cstore diff --git a/src/backend/columnar/cstore_metadata_tables.c b/src/backend/columnar/cstore_metadata_tables.c index 1d6f36c02..4e40aee68 100644 --- a/src/backend/columnar/cstore_metadata_tables.c +++ b/src/backend/columnar/cstore_metadata_tables.c @@ -9,6 +9,7 @@ #include "postgres.h" + #include "columnar/cstore.h" #include "columnar/cstore_version_compat.h" diff --git a/src/backend/columnar/cstore_tableam.c b/src/backend/columnar/cstore_tableam.c index a8a87425f..f0dd44de2 100644 --- a/src/backend/columnar/cstore_tableam.c +++ b/src/backend/columnar/cstore_tableam.c @@ -1,3 +1,6 @@ +#include "citus_version.h" +#if USE_TABLEAM + #include "postgres.h" #include @@ -1417,3 +1420,6 @@ alter_cstore_table_reset(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + + +#endif diff --git a/src/backend/columnar/mod.c b/src/backend/columnar/mod.c index 2344a5d68..ee3ba6d98 100644 --- a/src/backend/columnar/mod.c +++ b/src/backend/columnar/mod.c @@ -15,6 +15,8 @@ #include "fmgr.h" +#include "citus_version.h" + #include "columnar/cstore.h" #include "columnar/mod.h" @@ -26,10 +28,8 @@ #include "columnar/cstore_fdw.h" #endif -PG_MODULE_MAGIC; - void -_PG_init(void) +columnar_init(void) { cstore_init(); @@ -44,7 +44,7 @@ _PG_init(void) void -_PG_fini(void) +columnar_fini(void) { #if USE_TABLEAM cstore_tableam_finish(); diff --git a/src/backend/columnar/cstore_fdw--1.7.sql b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql similarity index 66% rename from src/backend/columnar/cstore_fdw--1.7.sql rename to src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql index 1f874ce60..b80239f6a 100644 --- a/src/backend/columnar/cstore_fdw--1.7.sql +++ b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql @@ -1,7 +1,7 @@ -/* cstore_fdw/cstore_fdw--1.7.sql */ +/* columnar--9.5-1--10.0-1.sql */ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cstore_fdw" to load this file. \quit +CREATE SCHEMA cstore; +SET search_path TO cstore; CREATE FUNCTION cstore_fdw_handler() RETURNS fdw_handler @@ -86,3 +86,42 @@ FROM pg_class c JOIN cstore.cstore_data_files d USING(relfilenode); COMMENT ON VIEW cstore_options IS 'CStore per table settings'; + +DO $proc$ +BEGIN + +-- from version 12 and up we have support for tableam's if installed on pg11 we can't +-- create the objects here. Instead we rely on citus_finish_pg_upgrade to be called by the +-- user instead to add the missing objects +IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN + EXECUTE $$ + CREATE FUNCTION cstore_tableam_handler(internal) + RETURNS table_am_handler + LANGUAGE C + AS 'MODULE_PATHNAME', 'cstore_tableam_handler'; + + CREATE ACCESS METHOD cstore_tableam + TYPE TABLE HANDLER cstore_tableam_handler; + + CREATE FUNCTION pg_catalog.alter_cstore_table_set( + table_name regclass, + block_row_count int DEFAULT NULL, + stripe_row_count int DEFAULT NULL, + compression name DEFAULT null) + RETURNS void + LANGUAGE C + AS 'MODULE_PATHNAME', 'alter_cstore_table_set'; + + CREATE FUNCTION pg_catalog.alter_cstore_table_reset( + table_name regclass, + block_row_count bool DEFAULT false, + stripe_row_count bool DEFAULT false, + compression bool DEFAULT false) + RETURNS void + LANGUAGE C + AS 'MODULE_PATHNAME', 'alter_cstore_table_reset'; + $$; +END IF; +END$proc$; + +RESET search_path; diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index 507c72d9d..54aa9c3be 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -19,6 +19,8 @@ DATA_built = $(generated_sql_files) # directories with source files SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib test transaction utils worker +# columnar modules +SUBDIRS += ../columnar # enterprise modules SUBDIRS += diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 7973aaaf9..bd25d14ef 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -84,6 +84,8 @@ #include "utils/guc_tables.h" #include "utils/varlena.h" +#include "columnar/mod.h" + /* marks shared object as one loadable by the postgres version compiled against */ PG_MODULE_MAGIC; @@ -92,6 +94,7 @@ static char *CitusVersion = CITUS_VERSION; void _PG_init(void); +void _PG_fini(void); static void DoInitialCleanup(void); static void ResizeStackToMaximumDepth(void); @@ -311,6 +314,15 @@ _PG_init(void) { DoInitialCleanup(); } + columnar_init(); +} + + +/* shared library deconstruction function */ +void +_PG_fini(void) +{ + columnar_fini(); } diff --git a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql index fb96a78fe..e27645ff8 100644 --- a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql +++ b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql @@ -1,3 +1,5 @@ -- citus--9.5-1--10.0-1 -- bump version to 10.0-1 + +#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql" diff --git a/src/include/citus_config.h.in b/src/include/citus_config.h.in index 5943116c3..679f91c09 100644 --- a/src/include/citus_config.h.in +++ b/src/include/citus_config.h.in @@ -87,3 +87,11 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS + +/* Define to 1 to build with foreign datawrapper support. + (--with-columnar-fdw) */ +#undef USE_FDW + +/* Define to 1 to build with table access method support. + (--with-columnar-tableam) */ +#undef USE_TABLEAM diff --git a/src/include/citus_version.h.in b/src/include/citus_version.h.in index e2594767e..778d59233 100644 --- a/src/include/citus_version.h.in +++ b/src/include/citus_version.h.in @@ -26,3 +26,9 @@ /* Base URL for statistics collection and update checks */ #undef REPORTS_BASE_URL + +/* columnar foreign data wrapper capability */ +#undef USE_FDW + +/* columnar table access method capability */ +#undef USE_TABLEAM diff --git a/src/include/columnar/cstore.h b/src/include/columnar/cstore.h index 35598cd41..ff0051b95 100644 --- a/src/include/columnar/cstore.h +++ b/src/include/columnar/cstore.h @@ -13,6 +13,7 @@ #ifndef CSTORE_H #define CSTORE_H +#include "postgres.h" #include "fmgr.h" #include "lib/stringinfo.h" diff --git a/src/include/columnar/cstore_tableam.h b/src/include/columnar/cstore_tableam.h index 557506b9f..7bd879068 100644 --- a/src/include/columnar/cstore_tableam.h +++ b/src/include/columnar/cstore_tableam.h @@ -1,3 +1,6 @@ +#include "citus_version.h" +#if USE_TABLEAM + #include "postgres.h" #include "fmgr.h" #include "access/tableam.h" @@ -13,3 +16,4 @@ extern TableScanDesc cstore_beginscan_extended(Relation relation, Snapshot snaps ParallelTableScanDesc parallel_scan, uint32 flags, Bitmapset *attr_needed, List *scanQual); +#endif diff --git a/src/include/columnar/mod.h b/src/include/columnar/mod.h index 3196bc809..8bf8db612 100644 --- a/src/include/columnar/mod.h +++ b/src/include/columnar/mod.h @@ -15,7 +15,7 @@ #define MOD_H /* Function declarations for extension loading and unloading */ -extern void _PG_init(void); -extern void _PG_fini(void); +extern void columnar_init(void); +extern void columnar_fini(void); #endif /* MOD_H */