From 6db7d87618e91a10f57b704f816b921c78520b31 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Wed, 18 Mar 2020 17:58:02 +0100 Subject: [PATCH 1/5] Compile safestringlib using regular configure This is needed to automatically generate .bc (bitcode) files when postgres is compiled with llvmjit support. It also has the advantage that cmake is not required for the build anymore. --- src/backend/distributed/Makefile | 25 +++++-------------------- src/backend/distributed/safeclib | 1 + 2 files changed, 6 insertions(+), 20 deletions(-) create mode 120000 src/backend/distributed/safeclib diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index e3803dbdf..82e11cd11 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -7,13 +7,6 @@ safestringlib_builddir = $(citus_top_builddir)/vendor/safestringlib/build safestringlib_a = $(safestringlib_builddir)/libsafestring_static.a safestringlib_sources = $(wildcard $(safestringlib_srcdir)/safeclib/*) -# Use cmake3 if it exists, otherwise just use the string "cmake" and let bash -# figure out its exact path later. -# Using "which cmake" instead will make the error very hard to understand if -# you have neither "cmake3" or "cmake" in path. In that case the "CMAKE3" -# variable would be an empty string. -CMAKE3 = $(shell which cmake3 || echo cmake) - MODULE_big = citus EXTENSION = citus @@ -23,7 +16,11 @@ generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_fi DATA_built = $(generated_sql_files) # directories with source files -SUBDIRS = . commands connection ddl deparser executor master metadata planner progress relay test transaction utils worker +SUBDIRS = . commands connection ddl deparser executor master metadata planner progress relay safeclib test transaction utils worker + +# Symlinks are not copied over to the build directory if a separete build +# directory is used during configure (such as on CI) +ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS)) # That patsubst rule searches all directories listed in SUBDIRS for .c # files, and adds the corresponding .o files to OBJS @@ -50,23 +47,11 @@ utils/citus_version.o: $(CITUS_VERSION_INVALIDATE) SHLIB_LINK += $(filter -lssl -lcrypto -lssleay32 -leay32, $(LIBS)) -override LDFLAGS += $(safestringlib_a) override CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include SQL_DEPDIR=.deps/sql SQL_BUILDDIR=build/sql -$(safestringlib_a): $(safestringlib_sources) - rm -rf $(safestringlib_builddir) - mkdir -p $(safestringlib_builddir) - @# exports of LDFLAGS and CPPFLAGS are to make sure the ones from this - @# Makefile are not used - +cd $(safestringlib_builddir) && \ - export LDFLAGS='' && export CPPFLAGS='' && \ - $(CMAKE3) $(safestringlib_srcdir) && make - -citus.so: $(safestringlib_a) - $(generated_sql_files): $(citus_abs_srcdir)/build/%: % @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ diff --git a/src/backend/distributed/safeclib b/src/backend/distributed/safeclib new file mode 120000 index 000000000..c4da5b7ad --- /dev/null +++ b/src/backend/distributed/safeclib @@ -0,0 +1 @@ +../../../vendor/safestringlib/safeclib/ \ No newline at end of file From 9a79935f1f28dbf1dd6514c5fd4c913b3fd27072 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 19 Mar 2020 09:56:16 +0100 Subject: [PATCH 2/5] Update safestringlib --- vendor/safestringlib/CMakeLists.txt | 9 +++------ vendor/safestringlib/include/safe_mem_lib.h | 7 +------ vendor/safestringlib/safeclib/memset_s.c | 16 ++++++++-------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/vendor/safestringlib/CMakeLists.txt b/vendor/safestringlib/CMakeLists.txt index 41d236591..213d5a8f5 100644 --- a/vendor/safestringlib/CMakeLists.txt +++ b/vendor/safestringlib/CMakeLists.txt @@ -145,14 +145,14 @@ set_target_properties(${PROJECT_NAME}_objlib target_compile_definitions(${PROJECT_NAME}_objlib PRIVATE -DSTDC_HEADERS) -target_compile_options(${PROJECT_NAME}_objlib - PRIVATE -Wall -Wextra -Wsign-compare -Wformat-security) target_compile_options(${PROJECT_NAME}_objlib PRIVATE -Wno-unknown-pragmas -Wno-unused-parameter) if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 7) target_compile_options(${PROJECT_NAME}_objlib PRIVATE -Wno-implicit-fallthrough) endif() +target_compile_options(${PROJECT_NAME}_objlib + PRIVATE -Wall -Wextra -Wsign-compare -Wformat-security) target_compile_options(${PROJECT_NAME}_objlib PRIVATE -Wstack-protector -Winit-self) target_compile_options(${PROJECT_NAME}_objlib @@ -166,10 +166,7 @@ target_compile_options(${PROJECT_NAME}_objlib PRIVATE -fPIE -fPIC) if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 6) target_compile_options(${PROJECT_NAME}_objlib PRIVATE -mmitigate-rop) endif() -if(APPLE) -elseif(UNIX) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -z noexecstack -z relro -z now") -endif() +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -z noexecstack -z relro -z now") option(BUILD_ERROR_ON_WARNING "Fail compilation on warning" OFF) diff --git a/vendor/safestringlib/include/safe_mem_lib.h b/vendor/safestringlib/include/safe_mem_lib.h index da4dc2a5e..162efd2eb 100644 --- a/vendor/safestringlib/include/safe_mem_lib.h +++ b/vendor/safestringlib/include/safe_mem_lib.h @@ -97,12 +97,7 @@ extern errno_t wmemmove_s(wchar_t *dest, rsize_t dmax, /* set bytes */ -/* NOTE: This name is changed from memset_s to memset8_s out because it does - * not match with the C11 declaration of memset_s on OSX. The upstream issue - * can be found here: - * https://github.com/intel/safestringlib/issues/14 - */ -extern errno_t memset8_s(void *dest, rsize_t dmax, uint8_t value); +extern errno_t memset_s(void *dest, rsize_t dmax, uint8_t value); /* set uint16_t */ extern errno_t memset16_s(uint16_t *dest, rsize_t dmax, uint16_t value); diff --git a/vendor/safestringlib/safeclib/memset_s.c b/vendor/safestringlib/safeclib/memset_s.c index a1ca0ac4e..f19717d03 100644 --- a/vendor/safestringlib/safeclib/memset_s.c +++ b/vendor/safestringlib/safeclib/memset_s.c @@ -1,5 +1,5 @@ /*------------------------------------------------------------------ - * memset8_s + * memset_s * * October 2008, Bo Berry * @@ -37,12 +37,12 @@ /** * NAME - * memset8_s + * memset_s * * SYNOPSIS * #include "safe_mem_lib.h" * errno_t - * memset8_s(void *dest, rsize_t len, uint8_t value) + * memset_s(void *dest, rsize_t len, uint8_t value) * * DESCRIPTION * Sets len bytes starting at dest to the specified value. @@ -78,22 +78,22 @@ * */ errno_t -memset8_s (void *dest, rsize_t len, uint8_t value) +memset_s (void *dest, rsize_t len, uint8_t value) { if (dest == NULL) { - invoke_safe_mem_constraint_handler("memset8_s: dest is null", + invoke_safe_mem_constraint_handler("memset_s: dest is null", NULL, ESNULLP); return (RCNEGATE(ESNULLP)); } if (len == 0) { - invoke_safe_mem_constraint_handler("memset8_s: len is 0", + invoke_safe_mem_constraint_handler("memset_s: len is 0", NULL, ESZEROL); return (RCNEGATE(ESZEROL)); } if (len > RSIZE_MAX_MEM) { - invoke_safe_mem_constraint_handler("memset8_s: len exceeds max", + invoke_safe_mem_constraint_handler("memset_s: len exceeds max", NULL, ESLEMAX); return (RCNEGATE(ESLEMAX)); } @@ -102,4 +102,4 @@ memset8_s (void *dest, rsize_t len, uint8_t value) return (RCNEGATE(EOK)); } -EXPORT_SYMBOL(memset8_s) +EXPORT_SYMBOL(memset_s) From dc2a371d9f8b28ad0e68c5230bb998b4463d8131 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 19 Mar 2020 10:08:33 +0100 Subject: [PATCH 3/5] Fix compilation issues with safestringlib Based on 92d7a40d1de472d23d05967be9b77eda30af85cb --- vendor/safestringlib/include/safe_mem_lib.h | 7 ++++++- .../safestringlib/safeclib/mem_primitives_lib.c | 2 ++ vendor/safestringlib/safeclib/memset_s.c | 16 ++++++++-------- vendor/safestringlib/safeclib/safeclib_private.h | 1 + vendor/safestringlib/safeclib/snprintf_support.c | 1 + 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/vendor/safestringlib/include/safe_mem_lib.h b/vendor/safestringlib/include/safe_mem_lib.h index 162efd2eb..da4dc2a5e 100644 --- a/vendor/safestringlib/include/safe_mem_lib.h +++ b/vendor/safestringlib/include/safe_mem_lib.h @@ -97,7 +97,12 @@ extern errno_t wmemmove_s(wchar_t *dest, rsize_t dmax, /* set bytes */ -extern errno_t memset_s(void *dest, rsize_t dmax, uint8_t value); +/* NOTE: This name is changed from memset_s to memset8_s out because it does + * not match with the C11 declaration of memset_s on OSX. The upstream issue + * can be found here: + * https://github.com/intel/safestringlib/issues/14 + */ +extern errno_t memset8_s(void *dest, rsize_t dmax, uint8_t value); /* set uint16_t */ extern errno_t memset16_s(uint16_t *dest, rsize_t dmax, uint16_t value); diff --git a/vendor/safestringlib/safeclib/mem_primitives_lib.c b/vendor/safestringlib/safeclib/mem_primitives_lib.c index cc189e5ea..54b596d02 100644 --- a/vendor/safestringlib/safeclib/mem_primitives_lib.c +++ b/vendor/safestringlib/safeclib/mem_primitives_lib.c @@ -31,6 +31,8 @@ #include "mem_primitives_lib.h" +#pragma GCC diagnostic ignored "-Wpragmas" +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" /* * mem_primitives_lib.c provides unguarded memory routines * that are used by the safe_mem_library. These routines diff --git a/vendor/safestringlib/safeclib/memset_s.c b/vendor/safestringlib/safeclib/memset_s.c index f19717d03..a1ca0ac4e 100644 --- a/vendor/safestringlib/safeclib/memset_s.c +++ b/vendor/safestringlib/safeclib/memset_s.c @@ -1,5 +1,5 @@ /*------------------------------------------------------------------ - * memset_s + * memset8_s * * October 2008, Bo Berry * @@ -37,12 +37,12 @@ /** * NAME - * memset_s + * memset8_s * * SYNOPSIS * #include "safe_mem_lib.h" * errno_t - * memset_s(void *dest, rsize_t len, uint8_t value) + * memset8_s(void *dest, rsize_t len, uint8_t value) * * DESCRIPTION * Sets len bytes starting at dest to the specified value. @@ -78,22 +78,22 @@ * */ errno_t -memset_s (void *dest, rsize_t len, uint8_t value) +memset8_s (void *dest, rsize_t len, uint8_t value) { if (dest == NULL) { - invoke_safe_mem_constraint_handler("memset_s: dest is null", + invoke_safe_mem_constraint_handler("memset8_s: dest is null", NULL, ESNULLP); return (RCNEGATE(ESNULLP)); } if (len == 0) { - invoke_safe_mem_constraint_handler("memset_s: len is 0", + invoke_safe_mem_constraint_handler("memset8_s: len is 0", NULL, ESZEROL); return (RCNEGATE(ESZEROL)); } if (len > RSIZE_MAX_MEM) { - invoke_safe_mem_constraint_handler("memset_s: len exceeds max", + invoke_safe_mem_constraint_handler("memset8_s: len exceeds max", NULL, ESLEMAX); return (RCNEGATE(ESLEMAX)); } @@ -102,4 +102,4 @@ memset_s (void *dest, rsize_t len, uint8_t value) return (RCNEGATE(EOK)); } -EXPORT_SYMBOL(memset_s) +EXPORT_SYMBOL(memset8_s) diff --git a/vendor/safestringlib/safeclib/safeclib_private.h b/vendor/safestringlib/safeclib/safeclib_private.h index 7280e879a..89ee97727 100644 --- a/vendor/safestringlib/safeclib/safeclib_private.h +++ b/vendor/safestringlib/safeclib/safeclib_private.h @@ -32,6 +32,7 @@ #ifndef __SAFECLIB_PRIVATE_H__ #define __SAFECLIB_PRIVATE_H__ +#include "citus_config.h" #ifdef __KERNEL__ /* linux kernel environment */ diff --git a/vendor/safestringlib/safeclib/snprintf_support.c b/vendor/safestringlib/safeclib/snprintf_support.c index 52a0739f8..8b482c1a4 100644 --- a/vendor/safestringlib/safeclib/snprintf_support.c +++ b/vendor/safestringlib/safeclib/snprintf_support.c @@ -54,6 +54,7 @@ #define CHK_FORMAT(X,Y) (((X)==(Y))?1:0) +#pragma GCC diagnostic ignored "-Wmissing-prototypes" unsigned int parse_format(const char *format, char pformatList[], unsigned int maxFormats) From 605b9016375097a9642d8d976a0682becc0e0e3d Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 19 Mar 2020 10:23:40 +0100 Subject: [PATCH 4/5] Update cherry-pick hash in vendor README --- vendor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/README.md b/vendor/README.md index e6da4b639..43b27ead1 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -6,5 +6,5 @@ git clone https://github.com/intel/safestringlib rm -rf safestringlib/{.git,unittests} git add safestringlib/ git commit -m "Update safestringlib" -git cherry-pick -x 92d7a40d1de472d23d05967be9b77eda30af85cb +git cherry-pick -x dc2a371d9f8b28ad0e68c5230bb998b4463d8131 ``` From a3513c89028ba7d423a4d61b4dd68002eaefeb73 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 19 Mar 2020 11:13:20 +0100 Subject: [PATCH 5/5] Ignore symlinks and directories editorconfig CI script --- ci/editorconfig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/editorconfig.sh b/ci/editorconfig.sh index 661aec1a7..427c45061 100755 --- a/ci/editorconfig.sh +++ b/ci/editorconfig.sh @@ -4,6 +4,7 @@ for f in $(git ls-tree -r HEAD --name-only); do if [ "$f" = "${f%.out}" ] && [ "$f" = "${f%.data}" ] && [ "$f" = "${f%.png}" ] && + [ -f "$f" ] && [ "$(echo "$f" | cut -d / -f1)" != "vendor" ] && [ "$(dirname "$f")" != "src/test/regress/output" ] then