From c485a04139739e5d34f9e4d243be14f61c88da4f Mon Sep 17 00:00:00 2001 From: jeff-davis Date: Thu, 31 Mar 2022 19:47:17 -0700 Subject: [PATCH] Separate build of citus.so and citus_columnar.so. (#5805) * Separate build of citus.so and citus_columnar.so. Because columnar code is statically-linked to both modules, it doesn't make sense to load them both at once. A subsequent commit will make the modules entirely separate and allow loading them both simultaneously. Author: Yanwen Jin * Separate citus and citus_columnar modules. Now the modules are independent. Columnar can be loaded by itself, or along with citus. Co-authored-by: Jeff Davis --- Makefile | 12 ++- src/backend/columnar/Makefile | 17 ++++ src/backend/columnar/columnar.c | 9 ++ src/backend/columnar/columnar_debug.c | 1 - src/backend/columnar/columnar_tableam.c | 9 +- src/backend/columnar/mod.c | 12 +-- src/backend/columnar/safeclib | 1 + src/backend/distributed/Makefile | 5 +- .../distributed/commands/alter_table.c | 5 +- .../distributed/operations/node_protocol.c | 8 +- src/backend/distributed/shared_library_init.c | 84 +++++++++++++++++-- src/include/columnar/columnar.h | 11 ++- src/include/columnar/mod.h | 19 ----- src/include/distributed/shared_library_init.h | 7 ++ src/test/regress/expected/columnar_create.out | 2 + src/test/regress/sql/columnar_create.sql | 3 + 16 files changed, 158 insertions(+), 47 deletions(-) create mode 100644 src/backend/columnar/Makefile create mode 120000 src/backend/columnar/safeclib delete mode 100644 src/include/columnar/mod.h diff --git a/Makefile b/Makefile index 7569fd6ce..f119a204f 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,16 @@ include Makefile.global all: extension + +# build columnar only +columnar: + $(MAKE) -C src/backend/columnar all # build extension -extension: $(citus_top_builddir)/src/include/citus_version.h +extension: $(citus_top_builddir)/src/include/citus_version.h columnar $(MAKE) -C src/backend/distributed/ all -install-extension: extension +install-columnar: columnar + $(MAKE) -C src/backend/columnar install +install-extension: extension install-columnar $(MAKE) -C src/backend/distributed/ install install-headers: extension $(MKDIR_P) '$(DESTDIR)$(includedir_server)/distributed/' @@ -27,6 +33,7 @@ install-headers: extension clean-extension: $(MAKE) -C src/backend/distributed/ clean + $(MAKE) -C src/backend/columnar/ clean clean-full: $(MAKE) -C src/backend/distributed/ clean-full .PHONY: extension install-extension clean-extension clean-full @@ -35,6 +42,7 @@ install: install-extension install-headers install-downgrades: $(MAKE) -C src/backend/distributed/ install-downgrades install-all: install-headers + $(MAKE) -C src/backend/columnar/ install-all $(MAKE) -C src/backend/distributed/ install-all clean: clean-extension diff --git a/src/backend/columnar/Makefile b/src/backend/columnar/Makefile new file mode 100644 index 000000000..abda9c90d --- /dev/null +++ b/src/backend/columnar/Makefile @@ -0,0 +1,17 @@ +citus_subdir = src/backend/columnar +citus_top_builddir = ../../.. +safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib +SUBDIRS = . safeclib +SUBDIRS += +ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS)) +OBJS += \ + $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c)))) + +MODULE_big = citus_columnar + +PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include + +include $(citus_top_builddir)/Makefile.global + +.PHONY: install-all +install-all: install diff --git a/src/backend/columnar/columnar.c b/src/backend/columnar/columnar.c index 35a6f6da9..85ec06d00 100644 --- a/src/backend/columnar/columnar.c +++ b/src/backend/columnar/columnar.c @@ -22,6 +22,7 @@ #include "citus_version.h" #include "columnar/columnar.h" +#include "columnar/columnar_tableam.h" /* Default values for option parameters */ #define DEFAULT_STRIPE_ROW_COUNT 150000 @@ -53,6 +54,14 @@ static const struct config_enum_entry columnar_compression_options[] = { NULL, 0, false } }; +void +columnar_init(void) +{ + columnar_init_gucs(); + columnar_tableam_init(); +} + + void columnar_init_gucs() { diff --git a/src/backend/columnar/columnar_debug.c b/src/backend/columnar/columnar_debug.c index f72ec5f8f..220d259fe 100644 --- a/src/backend/columnar/columnar_debug.c +++ b/src/backend/columnar/columnar_debug.c @@ -11,7 +11,6 @@ #include "postgres.h" #include "funcapi.h" -#include "pg_config.h" #include "access/nbtree.h" #include "access/table.h" #include "catalog/pg_am.h" diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 916962b4a..b6179ac8c 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -103,8 +103,8 @@ typedef struct IndexFetchColumnarData MemoryContext scanContext; } IndexFetchColumnarData; - -ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook = NULL; +/* available to other extensions using find_rendezvous_variable() */ +static ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook = NULL; static object_access_hook_type PrevObjectAccessHook = NULL; static ProcessUtility_hook_type PrevProcessUtilityHook = NULL; @@ -1910,6 +1910,11 @@ ColumnarSubXactCallback(SubXactEvent event, SubTransactionId mySubid, void columnar_tableam_init() { + ColumnarTableSetOptions_hook_type **ColumnarTableSetOptions_hook_ptr = + (ColumnarTableSetOptions_hook_type **) find_rendezvous_variable( + COLUMNAR_SETOPTIONS_HOOK_SYM); + *ColumnarTableSetOptions_hook_ptr = &ColumnarTableSetOptions_hook; + RegisterXactCallback(ColumnarXactCallback, NULL); RegisterSubXactCallback(ColumnarSubXactCallback, NULL); diff --git a/src/backend/columnar/mod.c b/src/backend/columnar/mod.c index f2679f326..c5112a5f4 100644 --- a/src/backend/columnar/mod.c +++ b/src/backend/columnar/mod.c @@ -18,13 +18,15 @@ #include "citus_version.h" #include "columnar/columnar.h" -#include "columnar/mod.h" - #include "columnar/columnar_tableam.h" + +PG_MODULE_MAGIC; + +void _PG_init(void); + void -columnar_init(void) +_PG_init(void) { - columnar_init_gucs(); - columnar_tableam_init(); + columnar_init(); } diff --git a/src/backend/columnar/safeclib b/src/backend/columnar/safeclib new file mode 120000 index 000000000..c4da5b7ad --- /dev/null +++ b/src/backend/columnar/safeclib @@ -0,0 +1 @@ +../../../vendor/safestringlib/safeclib/ \ No newline at end of file diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index a2ffcc142..f537d4cd8 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -19,8 +19,6 @@ DATA_built = $(generated_sql_files) # directories with source files SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib test transaction utils worker -# columnar modules -SUBDIRS += ../columnar # enterprise modules SUBDIRS += @@ -84,7 +82,8 @@ endif .PHONY: clean-full install install-downgrades install-all cleanup-before-install: - rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus* + rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus.control + rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus--* install: cleanup-before-install diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index 6664b6c1c..17e84dfaa 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -52,6 +52,7 @@ #include "distributed/multi_partitioning_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" +#include "distributed/shared_library_init.h" #include "distributed/shard_utils.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" @@ -687,7 +688,7 @@ ConvertTable(TableConversionState *con) strcmp(con->originalAccessMethod, "columnar") == 0) { ColumnarOptions options = { 0 }; - ReadColumnarOptions(con->relationId, &options); + extern_ReadColumnarOptions(con->relationId, &options); ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0( sizeof(ColumnarTableDDLContext)); @@ -843,7 +844,7 @@ DropIndexesNotSupportedByColumnar(Oid relationId, bool suppressNoticeMessages) foreach_oid(indexId, indexIdList) { char *indexAmName = GetIndexAccessMethodName(indexId); - if (ColumnarSupportsIndexAM(indexAmName)) + if (extern_ColumnarSupportsIndexAM(indexAmName)) { continue; } diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index d18ef749c..bd1a53ad4 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -48,6 +48,7 @@ #include "distributed/metadata_sync.h" #include "distributed/namespace_utils.h" #include "distributed/pg_dist_shard.h" +#include "distributed/shared_library_init.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "foreign/foreign.h" @@ -613,7 +614,7 @@ GetPreLoadTableCreationCommands(Oid relationId, /* add columnar options for cstore tables */ - if (accessMethod == NULL && IsColumnarTableAmTable(relationId)) + if (accessMethod == NULL && extern_IsColumnarTableAmTable(relationId)) { TableDDLCommand *cstoreOptionsDDL = ColumnarGetTableOptionsDDL(relationId); if (cstoreOptionsDDL != NULL) @@ -1047,7 +1048,8 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, options->chunkRowCount, options->stripeRowCount, options->compressionLevel, - quote_literal_cstr(CompressionTypeStr(options->compressionType))); + quote_literal_cstr(extern_CompressionTypeStr( + options->compressionType))); return buf.data; } @@ -1136,7 +1138,7 @@ ColumnarGetTableOptionsDDL(Oid relationId) char *relationName = get_rel_name(relationId); ColumnarOptions options = { 0 }; - ReadColumnarOptions(relationId, &options); + extern_ReadColumnarOptions(relationId, &options); return ColumnarGetCustomTableOptionsDDL(schemaName, relationName, options); } diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index e92c7d136..0c48dc8a4 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -95,11 +95,42 @@ #include "utils/syscache.h" #include "utils/varlena.h" -#include "columnar/mod.h" +#include "columnar/columnar.h" /* marks shared object as one loadable by the postgres version compiled against */ PG_MODULE_MAGIC; +ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM = NULL; +CompressionTypeStr_type extern_CompressionTypeStr = NULL; +IsColumnarTableAmTable_type extern_IsColumnarTableAmTable = NULL; +ReadColumnarOptions_type extern_ReadColumnarOptions = NULL; + +/* + * Define "pass-through" functions so that a SQL function defined as one of + * these symbols in the citus module can use the definition in the columnar + * module. + */ +#define DEFINE_COLUMNAR_PASSTHROUGH_FUNC(funcname) \ + static PGFunction CppConcat(extern_, funcname); \ + PG_FUNCTION_INFO_V1(funcname); \ + Datum funcname(PG_FUNCTION_ARGS) \ + { \ + return CppConcat(extern_, funcname)(fcinfo); \ + } +#define INIT_COLUMNAR_SYMBOL(typename, funcname) \ + CppConcat(extern_, funcname) = \ + (typename) (void *) lookup_external_function(handle, # funcname) + +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_handler) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_set) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_reset) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(upgrade_columnar_storage) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(downgrade_columnar_storage) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_relation_storageid) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_storage_info) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_store_memory_stats) +DEFINE_COLUMNAR_PASSTHROUGH_FUNC(test_columnar_storage_write_new_page) + #define DUMMY_REAL_TIME_EXECUTOR_ENUM_VALUE 9999999 static char *CitusVersion = CITUS_VERSION; @@ -323,12 +354,6 @@ _PG_init(void) original_client_auth_hook = ClientAuthentication_hook; ClientAuthentication_hook = CitusAuthHook; - /* - * When the options change on a columnar table, we may need to propagate - * the changes to shards. - */ - ColumnarTableSetOptions_hook = ColumnarTableSetOptionsHook; - InitializeMaintenanceDaemon(); /* initialize coordinated transaction management */ @@ -357,7 +382,50 @@ _PG_init(void) { DoInitialCleanup(); } - columnar_init(); + + /* ensure columnar module is loaded at the right time */ + load_file(COLUMNAR_MODULE_NAME, false); + + /* + * Now, acquire symbols from columnar module. First, acquire + * the address of the set options hook, and set it so that we + * can propagate options changes. + */ + ColumnarTableSetOptions_hook_type **ColumnarTableSetOptions_hook_ptr = + (ColumnarTableSetOptions_hook_type **) find_rendezvous_variable( + COLUMNAR_SETOPTIONS_HOOK_SYM); + + /* rendezvous variable registered during columnar initialization */ + Assert(ColumnarTableSetOptions_hook_ptr != NULL); + Assert(*ColumnarTableSetOptions_hook_ptr != NULL); + + **ColumnarTableSetOptions_hook_ptr = ColumnarTableSetOptionsHook; + + /* + * Acquire symbols for columnar functions that citus calls. + */ + void *handle = NULL; + + /* use load_external_function() the first time to initialize the handle */ + extern_ColumnarSupportsIndexAM = (ColumnarSupportsIndexAM_type) (void *) + load_external_function(COLUMNAR_MODULE_NAME, + "ColumnarSupportsIndexAM", + true, &handle); + + INIT_COLUMNAR_SYMBOL(CompressionTypeStr_type, CompressionTypeStr); + INIT_COLUMNAR_SYMBOL(IsColumnarTableAmTable_type, IsColumnarTableAmTable); + INIT_COLUMNAR_SYMBOL(ReadColumnarOptions_type, ReadColumnarOptions); + + /* initialize symbols for "pass-through" functions */ + INIT_COLUMNAR_SYMBOL(PGFunction, columnar_handler); + INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_set); + INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_reset); + INIT_COLUMNAR_SYMBOL(PGFunction, upgrade_columnar_storage); + INIT_COLUMNAR_SYMBOL(PGFunction, downgrade_columnar_storage); + INIT_COLUMNAR_SYMBOL(PGFunction, columnar_relation_storageid); + INIT_COLUMNAR_SYMBOL(PGFunction, columnar_storage_info); + INIT_COLUMNAR_SYMBOL(PGFunction, columnar_store_memory_stats); + INIT_COLUMNAR_SYMBOL(PGFunction, test_columnar_storage_write_new_page); } diff --git a/src/include/columnar/columnar.h b/src/include/columnar/columnar.h index 5195cbfee..4d31a45ed 100644 --- a/src/include/columnar/columnar.h +++ b/src/include/columnar/columnar.h @@ -25,6 +25,10 @@ #include "columnar/columnar_compression.h" #include "columnar/columnar_metadata.h" +#define COLUMNAR_MODULE_NAME "citus_columnar" + +#define COLUMNAR_SETOPTIONS_HOOK_SYM "ColumnarTableSetOptions_hook" + /* Defines for valid option names */ #define OPTION_NAME_COMPRESSION_TYPE "compression" #define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_limit" @@ -187,6 +191,10 @@ typedef enum StripeWriteStateEnum STRIPE_WRITE_IN_PROGRESS } StripeWriteStateEnum; +typedef bool (*ColumnarSupportsIndexAM_type)(char *); +typedef const char *(*CompressionTypeStr_type)(CompressionType); +typedef bool (*IsColumnarTableAmTable_type)(Oid); +typedef bool (*ReadColumnarOptions_type)(Oid, ColumnarOptions *); /* ColumnarReadState represents state of a columnar scan. */ struct ColumnarReadState; @@ -205,8 +213,8 @@ extern int columnar_compression_level; /* called when the user changes options on the given relation */ typedef void (*ColumnarTableSetOptions_hook_type)(Oid relid, ColumnarOptions options); -extern ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook; +extern void columnar_init(void); extern void columnar_init_gucs(void); extern CompressionType ParseCompressionType(const char *compressionTypeString); @@ -315,5 +323,4 @@ extern bool PendingWritesInUpperTransactions(Oid relfilenode, SubTransactionId currentSubXid); extern MemoryContext GetWriteContextForDebug(void); - #endif /* COLUMNAR_H */ diff --git a/src/include/columnar/mod.h b/src/include/columnar/mod.h deleted file mode 100644 index 7440a9fd7..000000000 --- a/src/include/columnar/mod.h +++ /dev/null @@ -1,19 +0,0 @@ -/*------------------------------------------------------------------------- - * - * mod.h - * - * Type and function declarations for columnar - * - * Copyright (c) Citus Data, Inc. - * - *------------------------------------------------------------------------- - */ - -#ifndef MOD_H -#define MOD_H - -/* Function declarations for extension loading and unloading */ -extern void columnar_init(void); -extern void columnar_fini(void); - -#endif /* MOD_H */ diff --git a/src/include/distributed/shared_library_init.h b/src/include/distributed/shared_library_init.h index b5bc09da3..485ab553f 100644 --- a/src/include/distributed/shared_library_init.h +++ b/src/include/distributed/shared_library_init.h @@ -11,10 +11,17 @@ #ifndef SHARED_LIBRARY_INIT_H #define SHARED_LIBRARY_INIT_H +#include "columnar/columnar.h" + #define GUC_STANDARD 0 #define MAX_SHARD_COUNT 64000 #define MAX_SHARD_REPLICATION_FACTOR 100 +extern ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM; +extern CompressionTypeStr_type extern_CompressionTypeStr; +extern IsColumnarTableAmTable_type extern_IsColumnarTableAmTable; +extern ReadColumnarOptions_type extern_ReadColumnarOptions; + extern void StartupCitusBackend(void); #endif /* SHARED_LIBRARY_INIT_H */ diff --git a/src/test/regress/expected/columnar_create.out b/src/test/regress/expected/columnar_create.out index 9cb025336..9b5fdace9 100644 --- a/src/test/regress/expected/columnar_create.out +++ b/src/test/regress/expected/columnar_create.out @@ -185,3 +185,5 @@ SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_sto f (1 row) +-- make sure citus_columnar can be loaded +LOAD 'citus_columnar'; diff --git a/src/test/regress/sql/columnar_create.sql b/src/test/regress/sql/columnar_create.sql index a5861bb45..4037dd28b 100644 --- a/src/test/regress/sql/columnar_create.sql +++ b/src/test/regress/sql/columnar_create.sql @@ -129,3 +129,6 @@ SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp'; SELECT COUNT(*)=0 FROM columnar_temp; -- since we deleted all the rows, we shouldn't have any stripes for table SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); + +-- make sure citus_columnar can be loaded +LOAD 'citus_columnar';