Separate build of citus.so and citus_columnar.so. (#5805)

* Separate build of citus.so and citus_columnar.so.

Because columnar code is statically-linked to both modules, it doesn't
make sense to load them both at once.

A subsequent commit will make the modules entirely separate and allow
loading them both simultaneously.

Author: Yanwen Jin

* Separate citus and citus_columnar modules.

Now the modules are independent. Columnar can be loaded by itself, or
along with citus.

Co-authored-by: Jeff Davis <jefdavi@microsoft.com>

Makefile

@@ -13,10 +13,16 @@ include Makefile.global
 
 all: extension
 
+
+# build columnar only
+columnar:
+	$(MAKE) -C src/backend/columnar all
 # build extension
-extension: $(citus_top_builddir)/src/include/citus_version.h
+extension: $(citus_top_builddir)/src/include/citus_version.h columnar
 	$(MAKE) -C src/backend/distributed/ all
-install-extension: extension
+install-columnar: columnar
+	$(MAKE) -C src/backend/columnar install
+install-extension: extension install-columnar
 	$(MAKE) -C src/backend/distributed/ install
 install-headers: extension
 	$(MKDIR_P) '$(DESTDIR)$(includedir_server)/distributed/'
@@ -27,6 +33,7 @@ install-headers: extension
 
 clean-extension:
 	$(MAKE) -C src/backend/distributed/ clean
+	$(MAKE) -C src/backend/columnar/ clean
 clean-full:
 	$(MAKE) -C src/backend/distributed/ clean-full
 .PHONY: extension install-extension clean-extension clean-full
@@ -35,6 +42,7 @@ install: install-extension install-headers
 install-downgrades:
 	$(MAKE) -C src/backend/distributed/ install-downgrades
 install-all: install-headers
+	$(MAKE) -C src/backend/columnar/ install-all
 	$(MAKE) -C src/backend/distributed/ install-all
 
 clean: clean-extension

src/backend/columnar/Makefile

@@ -0,0 +1,17 @@
+citus_subdir = src/backend/columnar
+citus_top_builddir = ../../..
+safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib
+SUBDIRS = . safeclib
+SUBDIRS +=
+ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS))
+OBJS += \
+	$(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c))))
+
+MODULE_big = citus_columnar
+
+PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include
+
+include $(citus_top_builddir)/Makefile.global
+
+.PHONY: install-all
+install-all: install

src/backend/columnar/columnar.c

@@ -22,6 +22,7 @@
 
 #include "citus_version.h"
 #include "columnar/columnar.h"
+#include "columnar/columnar_tableam.h"
 
 /* Default values for option parameters */
 #define DEFAULT_STRIPE_ROW_COUNT 150000
@@ -53,6 +54,14 @@ static const struct config_enum_entry columnar_compression_options[] =
 	{ NULL, 0, false }
 };
 
+void
+columnar_init(void)
+{
+	columnar_init_gucs();
+	columnar_tableam_init();
+}
+
+
 void
 columnar_init_gucs()
 {

src/backend/columnar/columnar_debug.c

@@ -11,7 +11,6 @@
 #include "postgres.h"
 
 #include "funcapi.h"
-#include "pg_config.h"
 #include "access/nbtree.h"
 #include "access/table.h"
 #include "catalog/pg_am.h"

src/backend/columnar/columnar_tableam.c

@@ -103,8 +103,8 @@ typedef struct IndexFetchColumnarData
 	MemoryContext scanContext;
 } IndexFetchColumnarData;
 
-
+/* available to other extensions using find_rendezvous_variable() */
-ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook = NULL;
+static ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook = NULL;
 
 static object_access_hook_type PrevObjectAccessHook = NULL;
 static ProcessUtility_hook_type PrevProcessUtilityHook = NULL;
@@ -1910,6 +1910,11 @@ ColumnarSubXactCallback(SubXactEvent event, SubTransactionId mySubid,
 void
 columnar_tableam_init()
 {
+	ColumnarTableSetOptions_hook_type **ColumnarTableSetOptions_hook_ptr =
+		(ColumnarTableSetOptions_hook_type **) find_rendezvous_variable(
+			COLUMNAR_SETOPTIONS_HOOK_SYM);
+	*ColumnarTableSetOptions_hook_ptr = &ColumnarTableSetOptions_hook;
+
 	RegisterXactCallback(ColumnarXactCallback, NULL);
 	RegisterSubXactCallback(ColumnarSubXactCallback, NULL);
 

src/backend/columnar/mod.c

@@ -18,13 +18,15 @@
 #include "citus_version.h"
 
 #include "columnar/columnar.h"
-#include "columnar/mod.h"
-
 #include "columnar/columnar_tableam.h"
 
+
+PG_MODULE_MAGIC;
+
+void _PG_init(void);
+
 void
-columnar_init(void)
+_PG_init(void)
 {
-	columnar_init_gucs();
+	columnar_init();
-	columnar_tableam_init();
 }

src/backend/columnar/safeclib

@@ -0,0 +1 @@
+../../../vendor/safestringlib/safeclib/

src/backend/distributed/Makefile

@@ -19,8 +19,6 @@ DATA_built = $(generated_sql_files)
 
 # directories with source files
 SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib test transaction utils worker
-# columnar modules
-SUBDIRS += ../columnar
 # enterprise modules
 SUBDIRS +=
 
@@ -84,7 +82,8 @@ endif
 .PHONY: clean-full install install-downgrades install-all
 
 cleanup-before-install:
-	rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus*
+	rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus.control
+	rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus--*
 
 install: cleanup-before-install
 

src/backend/distributed/commands/alter_table.c

@@ -52,6 +52,7 @@
 #include "distributed/multi_partitioning_utils.h"
 #include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
+#include "distributed/shared_library_init.h"
 #include "distributed/shard_utils.h"
 #include "distributed/worker_protocol.h"
 #include "distributed/worker_transaction.h"
@@ -687,7 +688,7 @@ ConvertTable(TableConversionState *con)
 		strcmp(con->originalAccessMethod, "columnar") == 0)
 	{
 		ColumnarOptions options = { 0 };
-		ReadColumnarOptions(con->relationId, &options);
+		extern_ReadColumnarOptions(con->relationId, &options);
 
 		ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0(
 			sizeof(ColumnarTableDDLContext));
@@ -843,7 +844,7 @@ DropIndexesNotSupportedByColumnar(Oid relationId, bool suppressNoticeMessages)
 	foreach_oid(indexId, indexIdList)
 	{
 		char *indexAmName = GetIndexAccessMethodName(indexId);
-		if (ColumnarSupportsIndexAM(indexAmName))
+		if (extern_ColumnarSupportsIndexAM(indexAmName))
 		{
 			continue;
 		}

src/backend/distributed/operations/node_protocol.c

@@ -48,6 +48,7 @@
 #include "distributed/metadata_sync.h"
 #include "distributed/namespace_utils.h"
 #include "distributed/pg_dist_shard.h"
+#include "distributed/shared_library_init.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_manager.h"
 #include "foreign/foreign.h"
@@ -613,7 +614,7 @@ GetPreLoadTableCreationCommands(Oid relationId,
 
 
 	/* add columnar options for cstore tables */
-	if (accessMethod == NULL && IsColumnarTableAmTable(relationId))
+	if (accessMethod == NULL && extern_IsColumnarTableAmTable(relationId))
 	{
 		TableDDLCommand *cstoreOptionsDDL = ColumnarGetTableOptionsDDL(relationId);
 		if (cstoreOptionsDDL != NULL)
@@ -1047,7 +1048,8 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName,
 					 options->chunkRowCount,
 					 options->stripeRowCount,
 					 options->compressionLevel,
-					 quote_literal_cstr(CompressionTypeStr(options->compressionType)));
+					 quote_literal_cstr(extern_CompressionTypeStr(
+											options->compressionType)));
 
 	return buf.data;
 }
@@ -1136,7 +1138,7 @@ ColumnarGetTableOptionsDDL(Oid relationId)
 	char *relationName = get_rel_name(relationId);
 
 	ColumnarOptions options = { 0 };
-	ReadColumnarOptions(relationId, &options);
+	extern_ReadColumnarOptions(relationId, &options);
 
 	return ColumnarGetCustomTableOptionsDDL(schemaName, relationName, options);
 }

src/backend/distributed/shared_library_init.c

@@ -95,11 +95,42 @@
 #include "utils/syscache.h"
 #include "utils/varlena.h"
 
-#include "columnar/mod.h"
+#include "columnar/columnar.h"
 
 /* marks shared object as one loadable by the postgres version compiled against */
 PG_MODULE_MAGIC;
 
+ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM = NULL;
+CompressionTypeStr_type extern_CompressionTypeStr = NULL;
+IsColumnarTableAmTable_type extern_IsColumnarTableAmTable = NULL;
+ReadColumnarOptions_type extern_ReadColumnarOptions = NULL;
+
+/*
+ * Define "pass-through" functions so that a SQL function defined as one of
+ * these symbols in the citus module can use the definition in the columnar
+ * module.
+ */
+#define DEFINE_COLUMNAR_PASSTHROUGH_FUNC(funcname) \
+	static PGFunction CppConcat(extern_, funcname); \
+	PG_FUNCTION_INFO_V1(funcname); \
+	Datum funcname(PG_FUNCTION_ARGS) \
+	{ \
+		return CppConcat(extern_, funcname)(fcinfo); \
+	}
+#define INIT_COLUMNAR_SYMBOL(typename, funcname) \
+	CppConcat(extern_, funcname) = \
+		(typename) (void *) lookup_external_function(handle, # funcname)
+
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_handler)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_set)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_reset)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(upgrade_columnar_storage)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(downgrade_columnar_storage)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_relation_storageid)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_storage_info)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_store_memory_stats)
+DEFINE_COLUMNAR_PASSTHROUGH_FUNC(test_columnar_storage_write_new_page)
+
 #define DUMMY_REAL_TIME_EXECUTOR_ENUM_VALUE 9999999
 static char *CitusVersion = CITUS_VERSION;
 
@@ -323,12 +354,6 @@ _PG_init(void)
 	original_client_auth_hook = ClientAuthentication_hook;
 	ClientAuthentication_hook = CitusAuthHook;
 
-	/*
-	 * When the options change on a columnar table, we may need to propagate
-	 * the changes to shards.
-	 */
-	ColumnarTableSetOptions_hook = ColumnarTableSetOptionsHook;
-
 	InitializeMaintenanceDaemon();
 
 	/* initialize coordinated transaction management */
@@ -357,7 +382,50 @@ _PG_init(void)
 	{
 		DoInitialCleanup();
 	}
-	columnar_init();
+
+	/* ensure columnar module is loaded at the right time */
+	load_file(COLUMNAR_MODULE_NAME, false);
+
+	/*
+	 * Now, acquire symbols from columnar module. First, acquire
+	 * the address of the set options hook, and set it so that we
+	 * can propagate options changes.
+	 */
+	ColumnarTableSetOptions_hook_type **ColumnarTableSetOptions_hook_ptr =
+		(ColumnarTableSetOptions_hook_type **) find_rendezvous_variable(
+			COLUMNAR_SETOPTIONS_HOOK_SYM);
+
+	/* rendezvous variable registered during columnar initialization */
+	Assert(ColumnarTableSetOptions_hook_ptr != NULL);
+	Assert(*ColumnarTableSetOptions_hook_ptr != NULL);
+
+	**ColumnarTableSetOptions_hook_ptr = ColumnarTableSetOptionsHook;
+
+	/*
+	 * Acquire symbols for columnar functions that citus calls.
+	 */
+	void *handle = NULL;
+
+	/* use load_external_function() the first time to initialize the handle */
+	extern_ColumnarSupportsIndexAM = (ColumnarSupportsIndexAM_type) (void *)
+									 load_external_function(COLUMNAR_MODULE_NAME,
+															"ColumnarSupportsIndexAM",
+															true, &handle);
+
+	INIT_COLUMNAR_SYMBOL(CompressionTypeStr_type, CompressionTypeStr);
+	INIT_COLUMNAR_SYMBOL(IsColumnarTableAmTable_type, IsColumnarTableAmTable);
+	INIT_COLUMNAR_SYMBOL(ReadColumnarOptions_type, ReadColumnarOptions);
+
+	/* initialize symbols for "pass-through" functions */
+	INIT_COLUMNAR_SYMBOL(PGFunction, columnar_handler);
+	INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_set);
+	INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_reset);
+	INIT_COLUMNAR_SYMBOL(PGFunction, upgrade_columnar_storage);
+	INIT_COLUMNAR_SYMBOL(PGFunction, downgrade_columnar_storage);
+	INIT_COLUMNAR_SYMBOL(PGFunction, columnar_relation_storageid);
+	INIT_COLUMNAR_SYMBOL(PGFunction, columnar_storage_info);
+	INIT_COLUMNAR_SYMBOL(PGFunction, columnar_store_memory_stats);
+	INIT_COLUMNAR_SYMBOL(PGFunction, test_columnar_storage_write_new_page);
 }
 
 

src/include/columnar/columnar.h

@@ -25,6 +25,10 @@
 #include "columnar/columnar_compression.h"
 #include "columnar/columnar_metadata.h"
 
+#define COLUMNAR_MODULE_NAME "citus_columnar"
+
+#define COLUMNAR_SETOPTIONS_HOOK_SYM "ColumnarTableSetOptions_hook"
+
 /* Defines for valid option names */
 #define OPTION_NAME_COMPRESSION_TYPE "compression"
 #define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_limit"
@@ -187,6 +191,10 @@ typedef enum StripeWriteStateEnum
 	STRIPE_WRITE_IN_PROGRESS
 } StripeWriteStateEnum;
 
+typedef bool (*ColumnarSupportsIndexAM_type)(char *);
+typedef const char *(*CompressionTypeStr_type)(CompressionType);
+typedef bool (*IsColumnarTableAmTable_type)(Oid);
+typedef bool (*ReadColumnarOptions_type)(Oid, ColumnarOptions *);
 
 /* ColumnarReadState represents state of a columnar scan. */
 struct ColumnarReadState;
@@ -205,8 +213,8 @@ extern int columnar_compression_level;
 
 /* called when the user changes options on the given relation */
 typedef void (*ColumnarTableSetOptions_hook_type)(Oid relid, ColumnarOptions options);
-extern ColumnarTableSetOptions_hook_type ColumnarTableSetOptions_hook;
 
+extern void columnar_init(void);
 extern void columnar_init_gucs(void);
 
 extern CompressionType ParseCompressionType(const char *compressionTypeString);
@@ -315,5 +323,4 @@ extern bool PendingWritesInUpperTransactions(Oid relfilenode,
 											 SubTransactionId currentSubXid);
 extern MemoryContext GetWriteContextForDebug(void);
 
-
 #endif /* COLUMNAR_H */

src/include/columnar/mod.h

@@ -1,19 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * mod.h
- *
- * Type and function declarations for columnar
- *
- * Copyright (c) Citus Data, Inc.
- *
- *-------------------------------------------------------------------------
- */
-
-#ifndef MOD_H
-#define MOD_H
-
-/* Function declarations for extension loading and unloading */
-extern void columnar_init(void);
-extern void columnar_fini(void);
-
-#endif /* MOD_H */

src/include/distributed/shared_library_init.h

@@ -11,10 +11,17 @@
 #ifndef SHARED_LIBRARY_INIT_H
 #define SHARED_LIBRARY_INIT_H
 
+#include "columnar/columnar.h"
+
 #define GUC_STANDARD 0
 #define MAX_SHARD_COUNT 64000
 #define MAX_SHARD_REPLICATION_FACTOR 100
 
+extern ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM;
+extern CompressionTypeStr_type extern_CompressionTypeStr;
+extern IsColumnarTableAmTable_type extern_IsColumnarTableAmTable;
+extern ReadColumnarOptions_type extern_ReadColumnarOptions;
+
 extern void StartupCitusBackend(void);
 
 #endif /* SHARED_LIBRARY_INIT_H */

src/test/regress/expected/columnar_create.out

@@ -185,3 +185,5 @@ SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_sto
  f
 (1 row)
 
+-- make sure citus_columnar can be loaded
+LOAD 'citus_columnar';

src/test/regress/sql/columnar_create.sql

@@ -129,3 +129,6 @@ SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp';
 SELECT COUNT(*)=0 FROM columnar_temp;
 -- since we deleted all the rows, we shouldn't have any stripes for table
 SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
+
+-- make sure citus_columnar can be loaded
+LOAD 'citus_columnar';