add backtrace

src/backend/distributed/Makefile

@@ -48,7 +48,7 @@ utils/citus_version.o: $(CITUS_VERSION_INVALIDATE)
 SHLIB_LINK += $(filter -lssl -lcrypto -lssleay32 -leay32, $(LIBS))
 
 override LDFLAGS += /usr/lib/gcc/x86_64-linux-gnu/7/libbacktrace.a
-override CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include -I/usr/lib/gcc/x86_64-linux-gnu/7/include/
+override CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include -I/usr/lib/gcc/x86_64-linux-gnu/7/include/ -L/usr/lib/gcc/x86_64-linux-gnu/7/ -lbacktrace
 
 SQL_DEPDIR=.deps/sql
 SQL_BUILDDIR=build/sql

src/backend/distributed/executor/multi_server_executor.c

@@ -104,8 +104,8 @@ JobExecutorType(DistributedPlan *distributedPlan)
 		{
 			if (!EnableRepartitionJoins)
 			{
-				ereport(ERROR, (errmsg(
-									"the query contains a join that requires repartitioning"),
+				ereport(ERROR, (
+								errmsg("the query contains a join that requires repartitioning"),
 								errhint("Set citus.enable_repartition_joins to on "
 										"to enable repartitioning")));
 			}
@@ -134,7 +134,6 @@ JobExecutorType(DistributedPlan *distributedPlan)
 	return executorType;
 }
 
-
 /*
  * HasReplicatedDistributedTable returns true if there is any
  * table in the given list that is:

src/backend/distributed/shared_library_init.c

@@ -21,8 +21,6 @@
 #include "fmgr.h"
 #include "miscadmin.h"
 
-#include "backtrace-supported.h"
-#include "backtrace.h"
 #include "safe_lib.h"
 
 #include "citus_version.h"
@@ -368,12 +366,8 @@ multi_log_hook(ErrorData *edata)
 		edata->message = "canceling the transaction since it was "
 						 "involved in a distributed deadlock";
 	}
-
-	struct backtrace_state* m = backtrace_create_state(NULL, 0 , NULL, NULL);
-	backtrace_full(m, 0, NULL, NULL, NULL);
 }
 
-
 /*
  * StartupCitusBackend initializes per-backend infrastructure, and is called
  * the first time citus is used in a database.

src/backend/distributed/utils/backtrace.c

@@ -0,0 +1,75 @@
+/*
+ * backtrace.c
+ *
+ * Utilities related to backtrace
+ *
+ * Copyright (c) Citus Data, Inc.
+ */
+
+#include "postgres.h"
+#include "backtrace.h"
+#include "backtrace-supported.h"
+#include "lib/stringinfo.h"
+
+#include "distributed/backtrace.h"
+
+#define BACKTRACE_HEADER "\nBACKTRACE:\n"
+#define BACKTRACE_SKIP 2
+
+static int BacktraceFullCallback(void *data, uintptr_t pc,
+					const char *filename, int lineno,
+					const char *function);
+
+static void BacktraceErrorCallback(void *data, const char *msg, int errnum);
+static void InitBackTrace(void);
+static bool ShouldLogBacktrace(int elevel);
+static char* GenerateBackTrace(void);
+
+static struct backtrace_state* backTracestate;
+
+static void InitBackTrace(void) {
+    const char* filename = NULL;
+    void* data = NULL;
+	backTracestate = backtrace_create_state(filename, BACKTRACE_SUPPORTS_THREADS,
+         BacktraceErrorCallback, data);
+}
+
+static bool ShouldLogBacktrace(int elevel) {
+    return elevel >= ERROR;
+}
+
+
+void Backtrace(int elevel) {
+    if (!ShouldLogBacktrace(elevel)) {
+        return;
+    }
+    errdetail("%s", GenerateBackTrace());
+ }
+
+static char* GenerateBackTrace(void) {
+    if (backTracestate == NULL) {
+        InitBackTrace();
+    }
+    StringInfo msgWithBacktrace = makeStringInfo();
+
+	appendStringInfoString(msgWithBacktrace, BACKTRACE_HEADER);
+	backtrace_full(backTracestate, BACKTRACE_SKIP, BacktraceFullCallback,
+        BacktraceErrorCallback, msgWithBacktrace);
+
+    return msgWithBacktrace->data;
+}
+
+static int BacktraceFullCallback(void *data, uintptr_t pc, const char *filename, int lineno,
+					const char *function) {
+        
+    StringInfo str = (StringInfo) data;
+    if (function && filename) {
+        appendStringInfo(str, "%s:%s:%d\n",filename, function, lineno);
+    }
+    /* returning 0 means we will continue the backtrace */
+    return 0;
+}
+
+static void BacktraceErrorCallback(void *data, const char *msg, int errnum) {
+    // currently NO-OP
+}

src/include/distributed/backtrace.h

@@ -0,0 +1,18 @@
+/*
+ * backtrace.h
+ *
+ * Backtrace utilities.
+ *
+ * Copyright (c) Citus Data, Inc.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef CITUS_BACKTRACE_H
+#define CITUS_BACKTRACE_H
+
+void Backtrace(int elevel);
+
+#endif
+
+

src/include/distributed/log_utils.h

@@ -11,6 +11,7 @@
 
 
 #include "utils/guc.h"
+#include "distributed/backtrace.h"
 
 /* do not log */
 #define CITUS_LOG_LEVEL_OFF 0
@@ -21,6 +22,31 @@ extern char * HashLogMessage(const char *text);
 
 #define ApplyLogRedaction(text) \
 	(log_min_messages <= ereport_loglevel ? HashLogMessage(text) : text)
+#undef ereport_domain
+#ifdef HAVE__BUILTIN_CONSTANT_P
+#define ereport_domain(elevel, domain, ...)	\
+	do { \
+		pg_prevent_errno_in_scope(); \
+		if (errstart(elevel, __FILE__, __LINE__, PG_FUNCNAME_MACRO, domain)) { \
+			__VA_ARGS__, Backtrace(elevel);errfinish(0); \
+		}\
+		if (__builtin_constant_p(elevel) && (elevel) >= ERROR) \
+			pg_unreachable(); \
+	} while(0)
+#else							/* !HAVE__BUILTIN_CONSTANT_P */
+#define ereport_domain(elevel, domain, ...)	\
+	do { \
+		const int elevel_ = (elevel); \
+		pg_prevent_errno_in_scope(); \
+		if (errstart(elevel_, __FILE__, __LINE__, PG_FUNCNAME_MACRO, domain)) {\
+			__VA_ARGS__, Backtrace(elevel);errfinish(0); \
+		}\
+		if (elevel_ >= ERROR) \
+			pg_unreachable(); \
+	} while(0)
+#endif							/* HAVE__BUILTIN_CONSTANT_P */
+
+
 
 #undef ereport
 #define ereport(elevel, rest) \