mirror of https://github.com/citusdata/citus.git
Merge c7a55c8606 into 4e47293f9f
Onur Tirtir
GitHub
commit
fc6169e24d
|
|
@ -669,7 +669,6 @@ infodir
|
|||
docdir
|
||||
oldincludedir
|
||||
includedir
|
||||
runstatedir
|
||||
localstatedir
|
||||
sharedstatedir
|
||||
sysconfdir
|
||||
|
|
@ -695,8 +694,6 @@ enable_option_checking
|
|||
with_extra_version
|
||||
with_pg_version_check
|
||||
enable_coverage
|
||||
with_libcurl
|
||||
with_reports_hostname
|
||||
with_lz4
|
||||
with_zstd
|
||||
with_security_flags
|
||||
|
|
@ -750,7 +747,6 @@ datadir='${datarootdir}'
|
|||
sysconfdir='${prefix}/etc'
|
||||
sharedstatedir='${prefix}/com'
|
||||
localstatedir='${prefix}/var'
|
||||
runstatedir='${localstatedir}/run'
|
||||
includedir='${prefix}/include'
|
||||
oldincludedir='/usr/include'
|
||||
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
|
||||
|
|
@ -1003,15 +999,6 @@ do
|
|||
| -silent | --silent | --silen | --sile | --sil)
|
||||
silent=yes ;;
|
||||
|
||||
-runstatedir | --runstatedir | --runstatedi | --runstated \
|
||||
| --runstate | --runstat | --runsta | --runst | --runs \
|
||||
| --run | --ru | --r)
|
||||
ac_prev=runstatedir ;;
|
||||
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
|
||||
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
|
||||
| --run=* | --ru=* | --r=*)
|
||||
runstatedir=$ac_optarg ;;
|
||||
|
||||
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
|
||||
ac_prev=sbindir ;;
|
||||
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
|
||||
|
|
@ -1149,7 +1136,7 @@ fi
|
|||
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
|
||||
datadir sysconfdir sharedstatedir localstatedir includedir \
|
||||
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
|
||||
libdir localedir mandir runstatedir
|
||||
libdir localedir mandir
|
||||
do
|
||||
eval ac_val=\$$ac_var
|
||||
# Remove trailing slashes.
|
||||
|
|
@ -1302,7 +1289,6 @@ Fine tuning of the installation directories:
|
|||
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
|
||||
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
|
||||
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
|
||||
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
|
||||
--libdir=DIR object code libraries [EPREFIX/lib]
|
||||
--includedir=DIR C header files [PREFIX/include]
|
||||
--oldincludedir=DIR C header files for non-gcc [/usr/include]
|
||||
|
|
@ -1341,11 +1327,6 @@ Optional Packages:
|
|||
append STRING to version
|
||||
--without-pg-version-check
|
||||
do not check postgres version during configure
|
||||
--without-libcurl do not use libcurl for anonymous statistics
|
||||
collection
|
||||
--with-reports-hostname=HOSTNAME
|
||||
Use HOSTNAME as hostname for statistics collection
|
||||
and update checks
|
||||
--without-lz4 do not use lz4
|
||||
--without-zstd do not use zstd
|
||||
--with-security-flags use security flags
|
||||
|
|
@ -4439,132 +4420,6 @@ if test "$enable_coverage" = yes; then
|
|||
CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
|
||||
fi
|
||||
|
||||
#
|
||||
# libcurl
|
||||
#
|
||||
|
||||
|
||||
|
||||
# Check whether --with-libcurl was given.
|
||||
if test "${with_libcurl+set}" = set; then :
|
||||
withval=$with_libcurl;
|
||||
case $withval in
|
||||
yes)
|
||||
|
||||
$as_echo "#define HAVE_LIBCURL 1" >>confdefs.h
|
||||
|
||||
;;
|
||||
no)
|
||||
:
|
||||
;;
|
||||
*)
|
||||
as_fn_error $? "no argument expected for --with-libcurl option" "$LINENO" 5
|
||||
;;
|
||||
esac
|
||||
|
||||
else
|
||||
with_libcurl=yes
|
||||
|
||||
$as_echo "#define HAVE_LIBCURL 1" >>confdefs.h
|
||||
|
||||
fi
|
||||
|
||||
|
||||
|
||||
if test "$with_libcurl" = yes; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_global_init in -lcurl" >&5
|
||||
$as_echo_n "checking for curl_global_init in -lcurl... " >&6; }
|
||||
if ${ac_cv_lib_curl_curl_global_init+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
ac_check_lib_save_LIBS=$LIBS
|
||||
LIBS="-lcurl $LIBS"
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
/* end confdefs.h. */
|
||||
|
||||
/* Override any GCC internal prototype to avoid an error.
|
||||
Use char because int might match the return type of a GCC
|
||||
builtin and then its argument prototype would still apply. */
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
#endif
|
||||
char curl_global_init ();
|
||||
int
|
||||
main ()
|
||||
{
|
||||
return curl_global_init ();
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
if ac_fn_c_try_link "$LINENO"; then :
|
||||
ac_cv_lib_curl_curl_global_init=yes
|
||||
else
|
||||
ac_cv_lib_curl_curl_global_init=no
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext \
|
||||
conftest$ac_exeext conftest.$ac_ext
|
||||
LIBS=$ac_check_lib_save_LIBS
|
||||
fi
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_global_init" >&5
|
||||
$as_echo "$ac_cv_lib_curl_curl_global_init" >&6; }
|
||||
if test "x$ac_cv_lib_curl_curl_global_init" = xyes; then :
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
#define HAVE_LIBCURL 1
|
||||
_ACEOF
|
||||
|
||||
LIBS="-lcurl $LIBS"
|
||||
|
||||
else
|
||||
as_fn_error $? "libcurl not found
|
||||
If you have libcurl already installed, see config.log for details on the
|
||||
failure. It is possible the compiler isn't looking in the proper directory.
|
||||
Use --without-libcurl to disable anonymous statistics collection." "$LINENO" 5
|
||||
fi
|
||||
|
||||
ac_fn_c_check_header_mongrel "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default"
|
||||
if test "x$ac_cv_header_curl_curl_h" = xyes; then :
|
||||
|
||||
else
|
||||
as_fn_error $? "libcurl header not found
|
||||
If you have libcurl already installed, see config.log for details on the
|
||||
failure. It is possible the compiler isn't looking in the proper directory.
|
||||
Use --without-libcurl to disable libcurl support." "$LINENO" 5
|
||||
fi
|
||||
|
||||
|
||||
fi
|
||||
|
||||
# REPORTS_BASE_URL definition
|
||||
|
||||
|
||||
|
||||
# Check whether --with-reports-hostname was given.
|
||||
if test "${with_reports_hostname+set}" = set; then :
|
||||
withval=$with_reports_hostname;
|
||||
case $withval in
|
||||
yes)
|
||||
as_fn_error $? "argument required for --with-reports-hostname option" "$LINENO" 5
|
||||
;;
|
||||
no)
|
||||
as_fn_error $? "argument required for --with-reports-hostname option" "$LINENO" 5
|
||||
;;
|
||||
*)
|
||||
REPORTS_BASE_URL="https://${withval}"
|
||||
;;
|
||||
esac
|
||||
|
||||
else
|
||||
REPORTS_BASE_URL="https://reports.citusdata.com"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
#define REPORTS_BASE_URL "$REPORTS_BASE_URL"
|
||||
_ACEOF
|
||||
|
||||
|
||||
#
|
||||
# LZ4
|
||||
#
|
||||
|
|
|
|||
27
configure.ac
27
configure.ac
|
|
@ -195,33 +195,6 @@ if test "$enable_coverage" = yes; then
|
|||
CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
|
||||
fi
|
||||
|
||||
#
|
||||
# libcurl
|
||||
#
|
||||
PGAC_ARG_BOOL(with, libcurl, yes,
|
||||
[do not use libcurl for anonymous statistics collection],
|
||||
[AC_DEFINE([HAVE_LIBCURL], 1, [Define to 1 to build with libcurl support. (--with-libcurl)])])
|
||||
|
||||
if test "$with_libcurl" = yes; then
|
||||
AC_CHECK_LIB(curl, curl_global_init, [],
|
||||
[AC_MSG_ERROR([libcurl not found
|
||||
If you have libcurl already installed, see config.log for details on the
|
||||
failure. It is possible the compiler isn't looking in the proper directory.
|
||||
Use --without-libcurl to disable anonymous statistics collection.])])
|
||||
AC_CHECK_HEADER(curl/curl.h, [], [AC_MSG_ERROR([libcurl header not found
|
||||
If you have libcurl already installed, see config.log for details on the
|
||||
failure. It is possible the compiler isn't looking in the proper directory.
|
||||
Use --without-libcurl to disable libcurl support.])])
|
||||
fi
|
||||
|
||||
# REPORTS_BASE_URL definition
|
||||
PGAC_ARG_REQ(with, reports-hostname, [HOSTNAME],
|
||||
[Use HOSTNAME as hostname for statistics collection and update checks],
|
||||
[REPORTS_BASE_URL="https://${withval}"],
|
||||
[REPORTS_BASE_URL="https://reports.citusdata.com"])
|
||||
AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL",
|
||||
[Base URL for statistics collection and update checks])
|
||||
|
||||
#
|
||||
# LZ4
|
||||
#
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@
|
|||
#include "distributed/shardsplit_shared_memory.h"
|
||||
#include "distributed/shared_connection_stats.h"
|
||||
#include "distributed/shared_library_init.h"
|
||||
#include "distributed/statistics_collection.h"
|
||||
#include "distributed/stats/query_stats.h"
|
||||
#include "distributed/stats/stat_counters.h"
|
||||
#include "distributed/stats/stat_tenants.h"
|
||||
|
|
@ -161,6 +160,7 @@ static char *MitmfifoEmptyString = "";
|
|||
static bool DeprecatedDeferShardDeleteOnMove = true;
|
||||
static bool DeprecatedDeferShardDeleteOnSplit = true;
|
||||
static bool DeprecatedReplicateReferenceTablesOnActivate = false;
|
||||
static bool DeprecatedEnableStatisticsCollection = false;
|
||||
|
||||
/* deprecated GUC value that should not be used anywhere outside this file */
|
||||
static int ReplicationModel = REPLICATION_MODEL_STREAMING;
|
||||
|
|
@ -210,8 +210,6 @@ static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource sour
|
|||
static void NodeConninfoGucAssignHook(const char *newval, void *extra);
|
||||
static const char * MaxSharedPoolSizeGucShowHook(void);
|
||||
static const char * LocalPoolSizeGucShowHook(void);
|
||||
static bool StatisticsCollectionGucCheckHook(bool *newval, void **extra, GucSource
|
||||
source);
|
||||
static bool WarnIfLocalExecutionDisabled(bool *newval, void **extra, GucSource source);
|
||||
static void CitusAuthHook(Port *port, int status);
|
||||
static bool IsSuperuser(char *userName);
|
||||
|
|
@ -1583,21 +1581,13 @@ RegisterCitusConfigVariables(void)
|
|||
|
||||
DefineCustomBoolVariable(
|
||||
"citus.enable_statistics_collection",
|
||||
gettext_noop("Enables sending basic usage statistics to Citus."),
|
||||
gettext_noop("Citus uploads daily anonymous usage reports containing "
|
||||
"rounded node count, shard size, distributed table count, "
|
||||
"and operating system name. This configuration value controls "
|
||||
"whether these reports are sent."),
|
||||
&EnableStatisticsCollection,
|
||||
#if defined(HAVE_LIBCURL) && defined(ENABLE_CITUS_STATISTICS_COLLECTION)
|
||||
true,
|
||||
#else
|
||||
gettext_noop("Deprecated."),
|
||||
NULL,
|
||||
&DeprecatedEnableStatisticsCollection,
|
||||
false,
|
||||
#endif
|
||||
PGC_SIGHUP,
|
||||
GUC_SUPERUSER_ONLY,
|
||||
&StatisticsCollectionGucCheckHook,
|
||||
NULL, NULL);
|
||||
NULL, NULL, NULL);
|
||||
|
||||
DefineCustomBoolVariable(
|
||||
"citus.enable_unique_job_ids",
|
||||
|
|
@ -3205,28 +3195,6 @@ LocalPoolSizeGucShowHook(void)
|
|||
}
|
||||
|
||||
|
||||
static bool
|
||||
StatisticsCollectionGucCheckHook(bool *newval, void **extra, GucSource source)
|
||||
{
|
||||
#ifdef HAVE_LIBCURL
|
||||
return true;
|
||||
#else
|
||||
|
||||
/* if libcurl is not installed, only accept false */
|
||||
if (*newval)
|
||||
{
|
||||
GUC_check_errcode(ERRCODE_FEATURE_NOT_SUPPORTED);
|
||||
GUC_check_errdetail("Citus was compiled without libcurl support.");
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* CitusAuthHook is a callback for client authentication that Postgres provides.
|
||||
* Citus uses this hook to count the number of active backends.
|
||||
|
|
|
|||
|
|
@ -16,12 +16,14 @@
|
|||
|
||||
#include "postgres.h"
|
||||
|
||||
#include "fmgr.h"
|
||||
#include "libpq-fe.h"
|
||||
#include "miscadmin.h"
|
||||
|
||||
#include "access/xact.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "utils/uuid.h"
|
||||
|
||||
#include "distributed/connection_management.h"
|
||||
#include "distributed/jsonbutils.h"
|
||||
|
|
@ -36,6 +38,10 @@
|
|||
#include "distributed/worker_manager.h"
|
||||
#include "distributed/worker_transaction.h"
|
||||
|
||||
|
||||
PG_FUNCTION_INFO_V1(citus_server_id);
|
||||
|
||||
|
||||
static void SendBareCommandListToMetadataNodesInternal(List *commandList,
|
||||
TargetWorkerSet targetWorkerSet);
|
||||
static void SendCommandToMetadataWorkersParams(const char *command,
|
||||
|
|
@ -835,3 +841,36 @@ IsWorkerTheCurrentNode(WorkerNode *workerNode)
|
|||
|
||||
return strcmp(workerServerId, currentServerId) == 0;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* citus_server_id returns a random UUID value as server identifier. This is
|
||||
* modeled after PostgreSQL's pg_random_uuid().
|
||||
*/
|
||||
Datum
|
||||
citus_server_id(PG_FUNCTION_ARGS)
|
||||
{
|
||||
uint8 *buf = (uint8 *) palloc(UUID_LEN);
|
||||
|
||||
/*
|
||||
* If pg_strong_random() fails, fall-back to using random(). In previous
|
||||
* versions of postgres we don't have pg_strong_random(), so use it by
|
||||
* default in that case.
|
||||
*/
|
||||
if (!pg_strong_random((char *) buf, UUID_LEN))
|
||||
{
|
||||
for (int bufIdx = 0; bufIdx < UUID_LEN; bufIdx++)
|
||||
{
|
||||
buf[bufIdx] = (uint8) (random() & 0xFF);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Set magic numbers for a "version 4" (pseudorandom) UUID, see
|
||||
* http://tools.ietf.org/html/rfc4122#section-4.4
|
||||
*/
|
||||
buf[6] = (buf[6] & 0x0f) | 0x40; /* "version" field */
|
||||
buf[8] = (buf[8] & 0x3f) | 0x80; /* "variant" field */
|
||||
|
||||
PG_RETURN_UUID_P((pg_uuid_t *) buf);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@
|
|||
#include "distributed/metadata_sync.h"
|
||||
#include "distributed/resource_lock.h"
|
||||
#include "distributed/shard_cleaner.h"
|
||||
#include "distributed/statistics_collection.h"
|
||||
#include "distributed/stats/query_stats.h"
|
||||
#include "distributed/transaction_recovery.h"
|
||||
#include "distributed/version_compat.h"
|
||||
|
|
@ -461,9 +460,6 @@ void
|
|||
CitusMaintenanceDaemonMain(Datum main_arg)
|
||||
{
|
||||
Oid databaseOid = DatumGetObjectId(main_arg);
|
||||
TimestampTz nextStatsCollectionTime USED_WITH_LIBCURL_ONLY =
|
||||
TimestampTzPlusMilliseconds(GetCurrentTimestamp(), 60 * 1000);
|
||||
bool retryStatsCollection USED_WITH_LIBCURL_ONLY = false;
|
||||
TimestampTz lastRecoveryTime = 0;
|
||||
TimestampTz lastShardCleanTime = 0;
|
||||
TimestampTz lastStatStatementsPurgeTime = 0;
|
||||
|
|
@ -518,59 +514,6 @@ CitusMaintenanceDaemonMain(Datum main_arg)
|
|||
* tasks should do their own time math about whether to re-run checks.
|
||||
*/
|
||||
|
||||
#ifdef HAVE_LIBCURL
|
||||
if (EnableStatisticsCollection &&
|
||||
GetCurrentTimestamp() >= nextStatsCollectionTime)
|
||||
{
|
||||
bool statsCollectionSuccess = false;
|
||||
InvalidateMetadataSystemCache();
|
||||
StartTransactionCommand();
|
||||
|
||||
/*
|
||||
* Lock the extension such that it cannot be dropped or created
|
||||
* concurrently. Skip statistics collection if citus extension is
|
||||
* not accessible.
|
||||
*
|
||||
* Similarly, we skip statistics collection if there exists any
|
||||
* version mismatch or the extension is not fully created yet.
|
||||
*/
|
||||
if (!LockCitusExtension())
|
||||
{
|
||||
ereport(DEBUG1, (errmsg("could not lock the citus extension, "
|
||||
"skipping statistics collection")));
|
||||
}
|
||||
else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded())
|
||||
{
|
||||
FlushDistTableCache();
|
||||
WarnIfSyncDNS();
|
||||
statsCollectionSuccess = CollectBasicUsageStatistics();
|
||||
}
|
||||
|
||||
/*
|
||||
* If statistics collection was successful the next collection is
|
||||
* 24-hours later. Also, if this was a retry attempt we don't do
|
||||
* any more retries until 24-hours later, so we limit number of
|
||||
* retries to one.
|
||||
*/
|
||||
if (statsCollectionSuccess || retryStatsCollection)
|
||||
{
|
||||
nextStatsCollectionTime =
|
||||
TimestampTzPlusMilliseconds(GetCurrentTimestamp(),
|
||||
STATS_COLLECTION_TIMEOUT_MILLIS);
|
||||
retryStatsCollection = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
nextStatsCollectionTime =
|
||||
TimestampTzPlusMilliseconds(GetCurrentTimestamp(),
|
||||
STATS_COLLECTION_RETRY_TIMEOUT_MILLIS);
|
||||
retryStatsCollection = true;
|
||||
}
|
||||
|
||||
CommitTransactionCommand();
|
||||
}
|
||||
#endif
|
||||
|
||||
pid_t metadataSyncBgwPid = 0;
|
||||
BgwHandleStatus metadataSyncStatus =
|
||||
metadataSyncBgwHandle != NULL ?
|
||||
|
|
|
|||
|
|
@ -1,355 +0,0 @@
|
|||
/*-------------------------------------------------------------------------
|
||||
*
|
||||
* statistics_collection.c
|
||||
* Anonymous reports and statistics collection.
|
||||
*
|
||||
* Copyright (c) Citus Data, Inc.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
#include "postgres.h"
|
||||
|
||||
#include "fmgr.h"
|
||||
|
||||
#include "utils/uuid.h"
|
||||
|
||||
#include "citus_version.h"
|
||||
|
||||
#if defined(HAVE_LIBCURL) && defined(ENABLE_CITUS_STATISTICS_COLLECTION)
|
||||
bool EnableStatisticsCollection = true; /* send basic usage statistics to Citus */
|
||||
#else
|
||||
bool EnableStatisticsCollection = false;
|
||||
#endif
|
||||
|
||||
PG_FUNCTION_INFO_V1(citus_server_id);
|
||||
|
||||
#ifdef HAVE_LIBCURL
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <sys/utsname.h>
|
||||
|
||||
#include "access/xact.h"
|
||||
#include "lib/stringinfo.h"
|
||||
#include "utils/builtins.h"
|
||||
#include "utils/fmgrprotos.h"
|
||||
#include "utils/json.h"
|
||||
#include "utils/jsonb.h"
|
||||
|
||||
#include "distributed/listutils.h"
|
||||
#include "distributed/metadata_cache.h"
|
||||
#include "distributed/multi_join_order.h"
|
||||
#include "distributed/shardinterval_utils.h"
|
||||
#include "distributed/statistics_collection.h"
|
||||
#include "distributed/version_compat.h"
|
||||
#include "distributed/worker_manager.h"
|
||||
|
||||
static size_t StatisticsCallback(char *contents, size_t size, size_t count,
|
||||
void *userData);
|
||||
static uint64 NextPow2(uint64 n);
|
||||
static uint64 DistributedTablesSize(List *distTableOids);
|
||||
static bool SendHttpPostJsonRequest(const char *url, const char *postFields,
|
||||
long timeoutSeconds,
|
||||
curl_write_callback responseCallback);
|
||||
static bool PerformHttpRequest(CURL *curl);
|
||||
|
||||
|
||||
/* WarnIfSyncDNS warns if libcurl is compiled with synchronous DNS. */
|
||||
void
|
||||
WarnIfSyncDNS(void)
|
||||
{
|
||||
curl_version_info_data *versionInfo = curl_version_info(CURLVERSION_NOW);
|
||||
if (!(versionInfo->features & CURL_VERSION_ASYNCHDNS))
|
||||
{
|
||||
ereport(WARNING, (errmsg("your current libcurl version doesn't support "
|
||||
"asynchronous DNS, which might cause unexpected "
|
||||
"delays in the operation of Citus"),
|
||||
errhint("Install a libcurl version with asynchronous DNS "
|
||||
"support.")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* CollectBasicUsageStatistics sends basic usage statistics to Citus servers.
|
||||
* This includes Citus version, table count rounded to next power of 2, cluster
|
||||
* size rounded to next power of 2, worker node count, and uname data. Returns
|
||||
* true if we actually have sent statistics to the server.
|
||||
*/
|
||||
bool
|
||||
CollectBasicUsageStatistics(void)
|
||||
{
|
||||
List *citusTableIdList = NIL;
|
||||
uint64 roundedDistTableCount = 0;
|
||||
uint64 roundedClusterSize = 0;
|
||||
uint32 workerNodeCount = 0;
|
||||
StringInfo fields = makeStringInfo();
|
||||
Datum metadataJsonbDatum = 0;
|
||||
char *metadataJsonbStr = NULL;
|
||||
MemoryContext savedContext = CurrentMemoryContext;
|
||||
int unameResult PG_USED_FOR_ASSERTS_ONLY = 0;
|
||||
bool metadataCollectionFailed = false;
|
||||
struct utsname unameData;
|
||||
memset(&unameData, 0, sizeof(unameData));
|
||||
|
||||
/*
|
||||
* Start a subtransaction so we can rollback database's state to it in case
|
||||
* of error.
|
||||
*/
|
||||
BeginInternalSubTransaction(NULL);
|
||||
|
||||
PG_TRY();
|
||||
{
|
||||
citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE);
|
||||
roundedDistTableCount = NextPow2(list_length(citusTableIdList));
|
||||
roundedClusterSize = NextPow2(DistributedTablesSize(citusTableIdList));
|
||||
workerNodeCount = ActivePrimaryNonCoordinatorNodeCount();
|
||||
metadataJsonbDatum = DistNodeMetadata();
|
||||
metadataJsonbStr = DatumGetCString(DirectFunctionCall1(jsonb_out,
|
||||
metadataJsonbDatum));
|
||||
|
||||
/*
|
||||
* Releasing a subtransaction doesn't free its memory context, since the
|
||||
* data it contains will be needed at upper commit. See the comments for
|
||||
* AtSubCommit_Memory() at postgres/src/backend/access/transam/xact.c.
|
||||
*/
|
||||
ReleaseCurrentSubTransaction();
|
||||
}
|
||||
PG_CATCH();
|
||||
{
|
||||
MemoryContextSwitchTo(savedContext);
|
||||
ErrorData *edata = CopyErrorData();
|
||||
FlushErrorState();
|
||||
|
||||
RollbackAndReleaseCurrentSubTransaction();
|
||||
|
||||
/* rethrow as WARNING */
|
||||
edata->elevel = WARNING;
|
||||
ThrowErrorData(edata);
|
||||
metadataCollectionFailed = true;
|
||||
}
|
||||
PG_END_TRY();
|
||||
|
||||
/*
|
||||
* Returning here instead of in PG_CATCH() since PG_END_TRY() resets couple
|
||||
* of global variables.
|
||||
*/
|
||||
if (metadataCollectionFailed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
unameResult = uname(&unameData);
|
||||
Assert(unameResult == 0); /* uname() always succeeds if we pass valid buffer */
|
||||
|
||||
appendStringInfoString(fields, "{\"citus_version\": ");
|
||||
escape_json(fields, CITUS_VERSION);
|
||||
appendStringInfo(fields, ",\"table_count\": " UINT64_FORMAT, roundedDistTableCount);
|
||||
appendStringInfo(fields, ",\"cluster_size\": " UINT64_FORMAT, roundedClusterSize);
|
||||
appendStringInfo(fields, ",\"worker_node_count\": %u", workerNodeCount);
|
||||
appendStringInfoString(fields, ",\"os_name\": ");
|
||||
escape_json(fields, unameData.sysname);
|
||||
appendStringInfoString(fields, ",\"os_release\": ");
|
||||
escape_json(fields, unameData.release);
|
||||
appendStringInfoString(fields, ",\"hwid\": ");
|
||||
escape_json(fields, unameData.machine);
|
||||
appendStringInfo(fields, ",\"node_metadata\": %s", metadataJsonbStr);
|
||||
appendStringInfoString(fields, "}");
|
||||
|
||||
return SendHttpPostJsonRequest(REPORTS_BASE_URL "/v1/usage_reports",
|
||||
fields->data, HTTP_TIMEOUT_SECONDS,
|
||||
StatisticsCallback);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* StatisticsCallback receives the response for the request sent by
|
||||
* CollectBasicUsageStatistics. For now, it doesn't check the contents of the
|
||||
* response and succeeds for any response.
|
||||
*/
|
||||
static size_t
|
||||
StatisticsCallback(char *contents, size_t size, size_t count, void *userData)
|
||||
{
|
||||
return size * count;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* DistributedTablesSize returns total size of data store in the cluster consisting
|
||||
* of given distributed tables. We ignore tables which we cannot get their size.
|
||||
*/
|
||||
static uint64
|
||||
DistributedTablesSize(List *distTableOids)
|
||||
{
|
||||
uint64 totalSize = 0;
|
||||
|
||||
Oid relationId = InvalidOid;
|
||||
foreach_declared_oid(relationId, distTableOids)
|
||||
{
|
||||
/*
|
||||
* Relations can get dropped after getting the Oid list and before we
|
||||
* reach here. Acquire a lock to make sure the relation is available
|
||||
* while we are getting its size.
|
||||
*/
|
||||
Relation relation = try_relation_open(relationId, AccessShareLock);
|
||||
if (relation == NULL)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ignore hash partitioned tables with size greater than 1, since
|
||||
* citus_table_size() doesn't work on them.
|
||||
*/
|
||||
if (IsCitusTableType(relationId, HASH_DISTRIBUTED) &&
|
||||
!SingleReplicatedTable(relationId))
|
||||
{
|
||||
table_close(relation, AccessShareLock);
|
||||
continue;
|
||||
}
|
||||
|
||||
Datum tableSizeDatum = DirectFunctionCall1(citus_table_size,
|
||||
ObjectIdGetDatum(relationId));
|
||||
totalSize += DatumGetInt64(tableSizeDatum);
|
||||
table_close(relation, AccessShareLock);
|
||||
}
|
||||
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* NextPow2 returns smallest power of 2 less than or equal to n. If n is greater
|
||||
* than 2^63, it returns 2^63. Returns 0 when n is 0.
|
||||
*/
|
||||
static uint64
|
||||
NextPow2(uint64 n)
|
||||
{
|
||||
uint64 result = 1;
|
||||
|
||||
if (n == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* if there is no 64-bit power of 2 greater than n, return 2^63 */
|
||||
if (n > (1ull << 63))
|
||||
{
|
||||
return (1ull << 63);
|
||||
}
|
||||
|
||||
while (result < n)
|
||||
{
|
||||
result *= 2;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* SendHttpPostJsonRequest sends a HTTP/HTTPS POST request to the given URL with
|
||||
* the given json object. responseCallback is called with the content of response.
|
||||
*/
|
||||
static bool
|
||||
SendHttpPostJsonRequest(const char *url, const char *jsonObj, long timeoutSeconds,
|
||||
curl_write_callback responseCallback)
|
||||
{
|
||||
bool success = false;
|
||||
|
||||
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||
CURL *curl = curl_easy_init();
|
||||
if (curl)
|
||||
{
|
||||
struct curl_slist *headers = NULL;
|
||||
headers = curl_slist_append(headers, "Accept: application/json");
|
||||
headers = curl_slist_append(headers, "Content-Type: application/json");
|
||||
headers = curl_slist_append(headers, "charsets: utf-8");
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url);
|
||||
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, jsonObj);
|
||||
curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeoutSeconds);
|
||||
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, responseCallback);
|
||||
|
||||
success = PerformHttpRequest(curl);
|
||||
|
||||
curl_slist_free_all(headers);
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
|
||||
curl_global_cleanup();
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* PerformHttpRequest sends the HTTP request with the parameters set in the given
|
||||
* curl object, and returns if it was successful or not. If the request was not
|
||||
* successful, it may log some warnings. This method expects to take place after
|
||||
* curl_easy_init() but before curl_easy_cleanup().
|
||||
*/
|
||||
static bool
|
||||
PerformHttpRequest(CURL *curl)
|
||||
{
|
||||
bool success = false;
|
||||
CURLcode curlCode = curl_easy_perform(curl);
|
||||
if (curlCode == CURLE_OK)
|
||||
{
|
||||
int64 httpCode = 0;
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode);
|
||||
if (httpCode == 200)
|
||||
{
|
||||
success = true;
|
||||
}
|
||||
else if (httpCode >= 400 && httpCode < 500)
|
||||
{
|
||||
ereport(WARNING, (errmsg("HTTP request failed."),
|
||||
errhint("HTTP response code: " INT64_FORMAT,
|
||||
httpCode)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ereport(WARNING, (errmsg("Sending HTTP request failed."),
|
||||
errhint("Error code: %s.", curl_easy_strerror(curlCode))));
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
|
||||
#endif /* HAVE_LIBCURL */
|
||||
|
||||
/*
|
||||
* citus_server_id returns a random UUID value as server identifier. This is
|
||||
* modeled after PostgreSQL's pg_random_uuid().
|
||||
*/
|
||||
Datum
|
||||
citus_server_id(PG_FUNCTION_ARGS)
|
||||
{
|
||||
uint8 *buf = (uint8 *) palloc(UUID_LEN);
|
||||
|
||||
/*
|
||||
* If pg_strong_random() fails, fall-back to using random(). In previous
|
||||
* versions of postgres we don't have pg_strong_random(), so use it by
|
||||
* default in that case.
|
||||
*/
|
||||
if (!pg_strong_random((char *) buf, UUID_LEN))
|
||||
{
|
||||
for (int bufIdx = 0; bufIdx < UUID_LEN; bufIdx++)
|
||||
{
|
||||
buf[bufIdx] = (uint8) (random() & 0xFF);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Set magic numbers for a "version 4" (pseudorandom) UUID, see
|
||||
* http://tools.ietf.org/html/rfc4122#section-4.4
|
||||
*/
|
||||
buf[6] = (buf[6] & 0x0f) | 0x40; /* "version" field */
|
||||
buf[8] = (buf[8] & 0x3f) | 0x80; /* "variant" field */
|
||||
|
||||
PG_RETURN_UUID_P((pg_uuid_t *) buf);
|
||||
}
|
||||
|
|
@ -37,21 +37,18 @@
|
|||
/* Define to 1 if you have the <inttypes.h> header file. */
|
||||
#undef HAVE_INTTYPES_H
|
||||
|
||||
/* Define to 1 if you have the `curl' library (-lcurl). */
|
||||
#undef HAVE_LIBCURL
|
||||
|
||||
/* Define to 1 if you have the `lz4' library (-llz4). */
|
||||
#undef HAVE_LIBLZ4
|
||||
|
||||
/* Define to 1 if you have the `zstd' library (-lzstd). */
|
||||
#undef HAVE_LIBZSTD
|
||||
|
||||
/* Define to 1 if you have the <memory.h> header file. */
|
||||
#undef HAVE_MEMORY_H
|
||||
|
||||
/* Define to 1 if you have the <stdint.h> header file. */
|
||||
#undef HAVE_STDINT_H
|
||||
|
||||
/* Define to 1 if you have the <stdio.h> header file. */
|
||||
#undef HAVE_STDIO_H
|
||||
|
||||
/* Define to 1 if you have the <stdlib.h> header file. */
|
||||
#undef HAVE_STDLIB_H
|
||||
|
||||
|
|
@ -88,13 +85,8 @@
|
|||
/* Define to the version of this package. */
|
||||
#undef PACKAGE_VERSION
|
||||
|
||||
/* Base URL for statistics collection and update checks */
|
||||
#undef REPORTS_BASE_URL
|
||||
|
||||
/* The size of `void *', as computed by sizeof. */
|
||||
#undef SIZEOF_VOID_P
|
||||
|
||||
/* Define to 1 if all of the C90 standard headers exist (not just the ones
|
||||
required in a freestanding environment). This macro is provided for
|
||||
backward compatibility; new code need not use it. */
|
||||
/* Define to 1 if you have the ANSI C header files. */
|
||||
#undef STDC_HEADERS
|
||||
|
|
|
|||
|
|
@ -21,9 +21,6 @@
|
|||
/* A string containing the version number, platform, and C compiler */
|
||||
#undef CITUS_VERSION_STR
|
||||
|
||||
/* Define to 1 if you have the `curl' library (-lcurl). */
|
||||
#undef HAVE_LIBCURL
|
||||
|
||||
/* Define to 1 if you have the `liblz4' library (-llz4). */
|
||||
#undef HAVE_CITUS_LIBLZ4
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
/*-------------------------------------------------------------------------
|
||||
*
|
||||
* statistics_collection.h
|
||||
*
|
||||
* Copyright (c) Citus Data, Inc.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
#ifndef STATISTICS_COLLECTION_H
|
||||
#define STATISTICS_COLLECTION_H
|
||||
|
||||
#include "citus_version.h"
|
||||
|
||||
/*
|
||||
* Append USED_WITH_LIBCURL_ONLY to definitions of variables that are only used
|
||||
* when compiled with libcurl, to avoid compiler warnings about unused variables
|
||||
* when built without libcurl.
|
||||
*/
|
||||
#ifdef HAVE_LIBCURL
|
||||
#define USED_WITH_LIBCURL_ONLY
|
||||
#else
|
||||
#define USED_WITH_LIBCURL_ONLY pg_attribute_unused()
|
||||
#endif
|
||||
|
||||
/* Config variables managed via guc.c */
|
||||
extern bool EnableStatisticsCollection;
|
||||
|
||||
#ifdef HAVE_LIBCURL
|
||||
|
||||
#define HTTP_TIMEOUT_SECONDS 5
|
||||
|
||||
extern void WarnIfSyncDNS(void);
|
||||
extern bool CollectBasicUsageStatistics(void);
|
||||
|
||||
#endif /* HAVE_LIBCURL */
|
||||
|
||||
#endif /* STATISTICS_COLLECTION_H */
|
||||
|
|
@ -11,7 +11,7 @@ install_citus_and_tar() {
|
|||
# do everything in a subdirectory to avoid clutter in current directory
|
||||
mkdir -p "${builddir}" && cd "${builddir}"
|
||||
|
||||
"${citus_dir}/configure" --without-libcurl
|
||||
"${citus_dir}/configure"
|
||||
|
||||
installdir="${builddir}/install"
|
||||
make "-j$(nproc)" && mkdir -p "${installdir}" && make DESTDIR="${installdir}" install
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
-- you're doing
|
||||
ALTER SYSTEM SET citus.distributed_deadlock_detection_factor TO -1;
|
||||
ALTER SYSTEM SET citus.recover_2pc_interval TO -1;
|
||||
ALTER SYSTEM set citus.enable_statistics_collection TO false;
|
||||
SELECT pg_reload_conf();
|
||||
pg_reload_conf
|
||||
---------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -724,7 +724,7 @@ END; $$ language plpgsql;
|
|||
-- ignore details such as exact costs or row counts. These filter
|
||||
-- functions replace changeable output details with fixed strings.
|
||||
-- Copied from PG explain.sql
|
||||
create function explain_filter(text) returns setof text
|
||||
create or replace function explain_filter(text) returns setof text
|
||||
language plpgsql as
|
||||
$$
|
||||
declare
|
||||
|
|
|
|||
|
|
@ -584,7 +584,6 @@ if($isolationtester)
|
|||
# maintenance daemon.
|
||||
push(@pgOptions, "citus.distributed_deadlock_detection_factor=-1");
|
||||
push(@pgOptions, "citus.recover_2pc_interval=-1");
|
||||
push(@pgOptions, "citus.enable_statistics_collection=false");
|
||||
push(@pgOptions, "citus.defer_shard_delete_interval=-1");
|
||||
push(@pgOptions, "citus.stat_statements_purge_interval=-1");
|
||||
push(@pgOptions, "citus.background_task_queue_interval=-1");
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
-- you're doing
|
||||
ALTER SYSTEM SET citus.distributed_deadlock_detection_factor TO -1;
|
||||
ALTER SYSTEM SET citus.recover_2pc_interval TO -1;
|
||||
ALTER SYSTEM set citus.enable_statistics_collection TO false;
|
||||
SELECT pg_reload_conf();
|
||||
|
||||
-- Add some helper functions for sending commands to mitmproxy
|
||||
|
|
|
|||
|
|
@ -755,7 +755,7 @@ END; $$ language plpgsql;
|
|||
-- functions replace changeable output details with fixed strings.
|
||||
-- Copied from PG explain.sql
|
||||
|
||||
create function explain_filter(text) returns setof text
|
||||
create or replace function explain_filter(text) returns setof text
|
||||
language plpgsql as
|
||||
$$
|
||||
declare
|
||||
|
|
|
|||
Loading…
Reference in New Issue