diff --git a/Makefile.global.in b/Makefile.global.in index 2481facbb..b3efb54ca 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -70,7 +70,7 @@ endif # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/... override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@ override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS) -override LDFLAGS += @LDFLAGS@ +override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@ # optional file with user defined, additional, rules -include ${citus_abs_srcdir}/src/Makefile.custom diff --git a/aclocal.m4 b/aclocal.m4 new file mode 100644 index 000000000..11fd3ca5d --- /dev/null +++ b/aclocal.m4 @@ -0,0 +1,2 @@ +dnl aclocal.m4 +m4_include([config/general.m4]) diff --git a/config/general.m4 b/config/general.m4 new file mode 100644 index 000000000..55097b727 --- /dev/null +++ b/config/general.m4 @@ -0,0 +1,151 @@ +# config/general.m4 +# Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, The Regents of the University of California + +# This file defines new macros to process configure command line +# arguments, to replace the brain-dead AC_ARG_WITH and AC_ARG_ENABLE. +# The flaw in these is particularly that they only differentiate +# between "given" and "not given" and do not provide enough help to +# process arguments that only accept "yes/no", that require an +# argument (other than "yes/no"), etc. +# +# The point of this implementation is to reduce code size and +# redundancy in configure.in and to improve robustness and consistency +# in the option evaluation code. + + +# Convert type and name to shell variable name (e.g., "enable_long_strings") +m4_define([pgac_arg_to_variable], + [$1[]_[]patsubst($2, -, _)]) + + +# PGAC_ARG(TYPE, NAME, HELP-STRING-LHS-EXTRA, HELP-STRING-RHS, +# [ACTION-IF-YES], [ACTION-IF-NO], [ACTION-IF-ARG], +# [ACTION-IF-OMITTED]) +# ------------------------------------------------------------ +# This is the base layer. TYPE is either "with" or "enable", depending +# on what you like. NAME is the rest of the option name. +# HELP-STRING-LHS-EXTRA is a string to append to the option name on +# the left-hand side of the help output, e.g., an argument name. If +# set to "-", append nothing, but let the option appear in the +# negative form (disable/without). HELP-STRING-RHS is the option +# description, for the right-hand side of the help output. +# ACTION-IF-YES is executed if the option is given without an argument +# (or "yes", which is the same); similar for ACTION-IF-NO. + +AC_DEFUN([PGAC_ARG], +[ +m4_case([$1], + +enable, [ +AC_ARG_ENABLE([$2], [AS_HELP_STRING([--]m4_if($3, -, disable, enable)[-$2]m4_if($3, -, , $3), [$4])], [ + case [$]enableval in + yes) + m4_default([$5], :) + ;; + no) + m4_default([$6], :) + ;; + *) + $7 + ;; + esac +], +[$8])[]dnl AC_ARG_ENABLE +], + +with, [ +AC_ARG_WITH([$2], [AS_HELP_STRING([--]m4_if($3, -, without, with)[-$2]m4_if($3, -, , $3), [$4])], [ + case [$]withval in + yes) + m4_default([$5], :) + ;; + no) + m4_default([$6], :) + ;; + *) + $7 + ;; + esac +], +[$8])[]dnl AC_ARG_WITH +], + +[m4_fatal([first argument of $0 must be 'enable' or 'with', not '$1'])] +) +])# PGAC_ARG + + +# PGAC_ARG_BOOL(TYPE, NAME, DEFAULT, HELP-STRING-RHS, +# [ACTION-IF-YES], [ACTION-IF-NO]) +# --------------------------------------------------- +# Accept a boolean option, that is, one that only takes yes or no. +# ("no" is equivalent to "disable" or "without"). DEFAULT is what +# should be done if the option is omitted; it should be "yes" or "no". +# (Consequently, one of ACTION-IF-YES and ACTION-IF-NO will always +# execute.) + +AC_DEFUN([PGAC_ARG_BOOL], +[dnl The following hack is necessary because in a few instances this +dnl macro is called twice for the same option with different default +dnl values. But we only want it to appear once in the help. We achieve +dnl that by making the help string look the same, which is why we need to +dnl save the default that was passed in previously. +m4_define([_pgac_helpdefault], m4_ifdef([pgac_defined_$1_$2_bool], [m4_defn([pgac_defined_$1_$2_bool])], [$3]))dnl +PGAC_ARG([$1], [$2], [m4_if(_pgac_helpdefault, yes, -)], [$4], [$5], [$6], + [AC_MSG_ERROR([no argument expected for --$1-$2 option])], + [m4_case([$3], + yes, [pgac_arg_to_variable([$1], [$2])=yes +$5], + no, [pgac_arg_to_variable([$1], [$2])=no +$6], + [m4_fatal([third argument of $0 must be 'yes' or 'no', not '$3'])])])[]dnl +m4_define([pgac_defined_$1_$2_bool], [$3])dnl +])# PGAC_ARG_BOOL + + +# PGAC_ARG_REQ(TYPE, NAME, HELP-ARGNAME, HELP-STRING-RHS, +# [ACTION-IF-GIVEN], [ACTION-IF-NOT-GIVEN]) +# ------------------------------------------------------- +# This option will require an argument; "yes" or "no" will not be +# accepted. HELP-ARGNAME is a name for the argument for the help output. + +AC_DEFUN([PGAC_ARG_REQ], +[PGAC_ARG([$1], [$2], [=$3], [$4], + [AC_MSG_ERROR([argument required for --$1-$2 option])], + [AC_MSG_ERROR([argument required for --$1-$2 option])], + [$5], + [$6])])# PGAC_ARG_REQ + + +# PGAC_ARG_OPTARG(TYPE, NAME, HELP-ARGNAME, HELP-STRING-RHS, +# [DEFAULT-ACTION], [ARG-ACTION], +# [ACTION-ENABLED], [ACTION-DISABLED]) +# ---------------------------------------------------------- +# This will create an option that behaves as follows: If omitted, or +# called with "no", then set the enable_variable to "no" and do +# nothing else. If called with "yes", then execute DEFAULT-ACTION. If +# called with argument, set enable_variable to "yes" and execute +# ARG-ACTION. Additionally, execute ACTION-ENABLED if we ended up with +# "yes" either way, else ACTION-DISABLED. +# +# The intent is to allow enabling a feature, and optionally pass an +# additional piece of information. + +AC_DEFUN([PGAC_ARG_OPTARG], +[PGAC_ARG([$1], [$2], [@<:@=$3@:>@], [$4], [$5], [], + [pgac_arg_to_variable([$1], [$2])=yes +$6], + [pgac_arg_to_variable([$1], [$2])=no]) +dnl Add this code only if there's a ACTION-ENABLED or ACTION-DISABLED. +m4_ifval([$7[]$8], +[ +if test "[$]pgac_arg_to_variable([$1], [$2])" = yes; then + m4_default([$7], :) +m4_ifval([$8], +[else + $8 +])[]dnl +fi +])[]dnl +])# PGAC_ARG_OPTARG diff --git a/configure b/configure index 194c7c1c2..817f8d37e 100755 --- a/configure +++ b/configure @@ -584,11 +584,51 @@ PACKAGE_STRING='Citus 7.1devel' PACKAGE_BUGREPORT='' PACKAGE_URL='' +# Factoring default headers for most tests. +ac_includes_default="\ +#include +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif +#ifdef STDC_HEADERS +# include +# include +#else +# ifdef HAVE_STDLIB_H +# include +# endif +#endif +#ifdef HAVE_STRING_H +# if !defined STDC_HEADERS && defined HAVE_MEMORY_H +# include +# endif +# include +#endif +#ifdef HAVE_STRINGS_H +# include +#endif +#ifdef HAVE_INTTYPES_H +# include +#endif +#ifdef HAVE_STDINT_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif" + ac_subst_vars='LTLIBOBJS LIBOBJS POSTGRES_BUILDDIR POSTGRES_SRCDIR +CITUS_LDFLAGS CITUS_CFLAGS +EGREP +GREP +CPP OBJEXT EXEEXT ac_ct_CC @@ -621,7 +661,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -645,6 +684,7 @@ ac_subst_files='' ac_user_opts=' enable_option_checking enable_coverage +with_libcurl ' ac_precious_vars='build_alias host_alias @@ -655,7 +695,8 @@ CC CFLAGS LDFLAGS LIBS -CPPFLAGS' +CPPFLAGS +CPP' # Initialize some variables set by options. @@ -694,7 +735,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -947,15 +987,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1093,7 +1124,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1246,7 +1277,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1278,6 +1308,12 @@ Optional Features: --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-coverage build with coverage testing instrumentation +Optional Packages: + --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] + --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) + --without-libcurl do not use libcurl for anonymous statistics + collection + Some influential environment variables: PG_CONFIG Location to find pg_config for target PostgreSQL instalation (default PATH) @@ -1289,6 +1325,7 @@ Some influential environment variables: LIBS libraries to pass to the linker, e.g. -l CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory + CPP C preprocessor Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -1409,6 +1446,249 @@ fi as_fn_set_status $ac_retval } # ac_fn_c_try_compile + +# ac_fn_c_try_link LINENO +# ----------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_link () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext conftest$ac_exeext + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + test -x conftest$ac_exeext + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information + # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would + # interfere with the next link command; also delete a directory that is + # left behind by Apple's compiler. We do this before executing the actions. + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_link + +# ac_fn_c_try_cpp LINENO +# ---------------------- +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_cpp + +# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists, giving a warning if it cannot be compiled using +# the include files in INCLUDES and setting the cache variable VAR +# accordingly. +ac_fn_c_check_header_mongrel () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if eval \${$3+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <$2> +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_mongrel + +# ac_fn_c_try_run LINENO +# ---------------------- +# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes +# that executables *can* be run. +ac_fn_c_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; }; then : + ac_retval=0 +else + $as_echo "$as_me: program exited with status $ac_status" >&5 + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_run + +# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists and can be compiled using the include files in +# INCLUDES, setting the cache variable VAR accordingly. +ac_fn_c_check_header_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + eval "$3=yes" +else + eval "$3=no" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_compile cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. @@ -3058,8 +3338,501 @@ if test "$enable_coverage" = yes; then CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage" fi +# +# libcurl +# + + + +# Check whether --with-libcurl was given. +if test "${with_libcurl+set}" = set; then : + withval=$with_libcurl; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-libcurl option" "$LINENO" 5 + ;; + esac + +else + with_libcurl=yes + +fi + + + +if test "$with_libcurl" = yes; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_global_init in -lcurl" >&5 +$as_echo_n "checking for curl_global_init in -lcurl... " >&6; } +if ${ac_cv_lib_curl_curl_global_init+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lcurl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char curl_global_init (); +int +main () +{ +return curl_global_init (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_curl_curl_global_init=yes +else + ac_cv_lib_curl_curl_global_init=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_global_init" >&5 +$as_echo "$ac_cv_lib_curl_curl_global_init" >&6; } +if test "x$ac_cv_lib_curl_curl_global_init" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBCURL 1 +_ACEOF + + LIBS="-lcurl $LIBS" + +else + as_fn_error $? "libcurl not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable anonymous statistics collection." "$LINENO" 5 +fi + + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 +$as_echo_n "checking how to run the C preprocessor... " >&6; } +# On Suns, sometimes $CPP names a directory. +if test -n "$CPP" && test -d "$CPP"; then + CPP= +fi +if test -z "$CPP"; then + if ${ac_cv_prog_CPP+:} false; then : + $as_echo_n "(cached) " >&6 +else + # Double quotes because CPP needs to be expanded + for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" + do + ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + break +fi + + done + ac_cv_prog_CPP=$CPP + +fi + CPP=$ac_cv_prog_CPP +else + ac_cv_prog_CPP=$CPP +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 +$as_echo "$CPP" >&6; } +ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # Prefer to if __STDC__ is defined, since + # exists even on freestanding compilers. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#ifdef __STDC__ +# include +#else +# include +#endif + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + +else + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + # Broken: success on invalid input. +continue +else + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok; then : + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } +fi + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 +$as_echo_n "checking for grep that handles long lines and -e... " >&6; } +if ${ac_cv_path_GREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -z "$GREP"; then + ac_path_GREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in grep ggrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" + as_fn_executable_p "$ac_path_GREP" || continue +# Check for GNU ac_path_GREP and select it if it is found. + # Check for GNU $ac_path_GREP +case `"$ac_path_GREP" --version 2>&1` in +*GNU*) + ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'GREP' >> "conftest.nl" + "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_GREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_GREP="$ac_path_GREP" + ac_path_GREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_GREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_GREP"; then + as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_GREP=$GREP +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 +$as_echo "$ac_cv_path_GREP" >&6; } + GREP="$ac_cv_path_GREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 +$as_echo_n "checking for egrep... " >&6; } +if ${ac_cv_path_EGREP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 + then ac_cv_path_EGREP="$GREP -E" + else + if test -z "$EGREP"; then + ac_path_EGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in egrep; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" + as_fn_executable_p "$ac_path_EGREP" || continue +# Check for GNU ac_path_EGREP and select it if it is found. + # Check for GNU $ac_path_EGREP +case `"$ac_path_EGREP" --version 2>&1` in +*GNU*) + ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo 'EGREP' >> "conftest.nl" + "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_EGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_EGREP="$ac_path_EGREP" + ac_path_EGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_EGREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_EGREP"; then + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + fi +else + ac_cv_path_EGREP=$EGREP +fi + + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 +$as_echo "$ac_cv_path_EGREP" >&6; } + EGREP="$ac_cv_path_EGREP" + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 +$as_echo_n "checking for ANSI C header files... " >&6; } +if ${ac_cv_header_stdc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_stdc=yes +else + ac_cv_header_stdc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +if test $ac_cv_header_stdc = yes; then + # SunOS 4.x string.h does not declare mem*, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "memchr" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "free" >/dev/null 2>&1; then : + +else + ac_cv_header_stdc=no +fi +rm -f conftest* + +fi + +if test $ac_cv_header_stdc = yes; then + # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. + if test "$cross_compiling" = yes; then : + : +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#if ((' ' & 0x0FF) == 0x020) +# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') +# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) +#else +# define ISLOWER(c) \ + (('a' <= (c) && (c) <= 'i') \ + || ('j' <= (c) && (c) <= 'r') \ + || ('s' <= (c) && (c) <= 'z')) +# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) +#endif + +#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) +int +main () +{ + int i; + for (i = 0; i < 256; i++) + if (XOR (islower (i), ISLOWER (i)) + || toupper (i) != TOUPPER (i)) + return 2; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + +else + ac_cv_header_stdc=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 +$as_echo "$ac_cv_header_stdc" >&6; } +if test $ac_cv_header_stdc = yes; then + +$as_echo "#define STDC_HEADERS 1" >>confdefs.h + +fi + +# On IRIX 5.3, sys/types and inttypes.h are conflicting. +for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ + inttypes.h stdint.h unistd.h +do : + as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default +" +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 +_ACEOF + +fi + +done + + +ac_fn_c_check_header_mongrel "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default" +if test "x$ac_cv_header_curl_curl_h" = xyes; then : + +else + as_fn_error $? "libcurl header not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable libcurl support." "$LINENO" 5 +fi + + + CITUS_CFLAGS="$CITUS_CFLAGS -DHAVE_LIBCURL" +fi + CITUS_CFLAGS="$CITUS_CFLAGS" +CITUS_LDFLAGS="$LIBS" + POSTGRES_SRCDIR="$POSTGRES_SRCDIR" POSTGRES_BUILDDIR="$POSTGRES_BUILDDIR" diff --git a/configure.in b/configure.in index 99824e1f5..dc911c3a6 100644 --- a/configure.in +++ b/configure.in @@ -140,7 +140,27 @@ if test "$enable_coverage" = yes; then CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage" fi +# +# libcurl +# +PGAC_ARG_BOOL(with, libcurl, yes, + [do not use libcurl for anonymous statistics collection]) + +if test "$with_libcurl" = yes; then + AC_CHECK_LIB(curl, curl_global_init, [], + [AC_MSG_ERROR([libcurl not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable anonymous statistics collection.])]) + AC_CHECK_HEADER(curl/curl.h, [], [AC_MSG_ERROR([libcurl header not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable libcurl support.])]) + CITUS_CFLAGS="$CITUS_CFLAGS -DHAVE_LIBCURL" +fi + AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") +AC_SUBST(CITUS_LDFLAGS, "$LIBS") AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index bb56b8670..75dbff997 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -41,6 +41,7 @@ #include "distributed/placement_connection.h" #include "distributed/remote_commands.h" #include "distributed/shared_library_init.h" +#include "distributed/statistics_collection.h" #include "distributed/task_tracker.h" #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" @@ -65,6 +66,8 @@ static void WarningForEnableDeadlockPrevention(bool newval, void *extra); static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra, GucSource source); static void NormalizeWorkerListPath(void); +static bool StatisticsCollectionGucCheckHook(bool *newval, void **extra, GucSource + source); /* *INDENT-OFF* */ @@ -785,6 +788,24 @@ RegisterCitusConfigVariables(void) 0, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.enable_statistics_collection", + gettext_noop("Enables sending basic usage statistics to Citus."), + gettext_noop("Citus uploads daily anonymous usage reports containing " + "rounded node count, shard size, distributed table count, " + "and operating system name. This configuration value controls " + "whether these reports are sent."), + &EnableStatisticsCollection, +#if HAVE_LIBCURL + true, +#else + false, +#endif + PGC_SIGHUP, + GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL, + &StatisticsCollectionGucCheckHook, + NULL, NULL); + /* warn about config items in the citus namespace that are not registered above */ EmitWarningsOnPlaceholders("citus"); } @@ -866,3 +887,25 @@ NormalizeWorkerListPath(void) PGC_S_OVERRIDE); free(absoluteFileName); } + + +static bool +StatisticsCollectionGucCheckHook(bool *newval, void **extra, GucSource source) +{ +#if HAVE_LIBCURL + return true; +#else + + /* if libcurl is not installed, only accept false */ + if (*newval) + { + GUC_check_errcode(ERRCODE_FEATURE_NOT_SUPPORTED); + GUC_check_errdetail("Citus was compiled without libcurl support."); + return false; + } + else + { + return true; + } +#endif +} diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index a3580428d..557f174cb 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -16,6 +16,7 @@ #include "postgres.h" +#include #include "miscadmin.h" #include "pgstat.h" @@ -27,7 +28,9 @@ #include "catalog/namespace.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/maintenanced.h" +#include "distributed/master_protocol.h" #include "distributed/metadata_cache.h" +#include "distributed/statistics_collection.h" #include "nodes/makefuncs.h" #include "postmaster/bgworker.h" #include "storage/ipc.h" @@ -36,6 +39,7 @@ #include "storage/lmgr.h" #include "storage/lwlock.h" #include "tcop/tcopprot.h" +#include "utils/memutils.h" /* @@ -209,6 +213,8 @@ CitusMaintenanceDaemonMain(Datum main_arg) { Oid databaseOid = DatumGetObjectId(main_arg); MaintenanceDaemonDBData *myDbData = NULL; + time_t prevStatsCollection = 0; + bool prevStatsCollectionFailed = false; ErrorContextCallback errorCallback; /* @@ -271,9 +277,22 @@ CitusMaintenanceDaemonMain(Datum main_arg) int latchFlags = WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH; double timeout = 10000.0; /* use this if the deadlock detection is disabled */ bool foundDeadlock = false; + time_t currentTime = time(NULL); + double secondsSincePrevStatsCollection = difftime(currentTime, + prevStatsCollection); + bool citusHasBeenLoaded = false; CHECK_FOR_INTERRUPTS(); + StartTransactionCommand(); + citusHasBeenLoaded = CitusHasBeenLoaded(); + CommitTransactionCommand(); + + if (!citusHasBeenLoaded) + { + continue; + } + /* * XXX: We clear the metadata cache before every iteration because otherwise * it might contain stale OIDs. It appears that in some cases invalidation @@ -289,6 +308,31 @@ CitusMaintenanceDaemonMain(Datum main_arg) * tasks should do their own time math about whether to re-run checks. */ + if (secondsSincePrevStatsCollection >= STATISTICS_COLLECTION_INTERVAL || + (prevStatsCollectionFailed && + secondsSincePrevStatsCollection >= STATISTICS_COLLECTION_RETRY_INTERVAL)) + { +#if HAVE_LIBCURL + if (EnableStatisticsCollection) + { + MemoryContext statsCollectionContext = + AllocSetContextCreate(CurrentMemoryContext, + "StatsCollection", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + MemoryContext oldContext = + MemoryContextSwitchTo(statsCollectionContext); + + WarnIfSyncDNS(); + prevStatsCollectionFailed = !CollectBasicUsageStatistics(); + + MemoryContextSwitchTo(oldContext); + prevStatsCollection = currentTime; + } +#endif + } + /* the config value -1 disables the distributed deadlock detection */ if (DistributedDeadlockDetectionTimeoutFactor != -1.0) { diff --git a/src/backend/distributed/utils/statistics_collection.c b/src/backend/distributed/utils/statistics_collection.c new file mode 100644 index 000000000..14dc6dd53 --- /dev/null +++ b/src/backend/distributed/utils/statistics_collection.c @@ -0,0 +1,213 @@ +/*------------------------------------------------------------------------- + * + * statistics_collection.c + * Anonymous reports and statistics collection. + * + * Copyright (c) 2017, Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include + +#include "access/xact.h" +#include "citus_version.h" +#include "distributed/metadata_cache.h" +#include "distributed/statistics_collection.h" +#include "distributed/worker_manager.h" +#include "lib/stringinfo.h" +#include "utils/json.h" + +bool EnableStatisticsCollection = true; /* send basic usage statistics to Citus */ + +#if HAVE_LIBCURL + +static uint64 NextPow2(uint64 n); +static uint64 ClusterSize(List *distributedTableList); +static bool SendHttpPostJsonRequest(const char *url, const char *postFields, long + timeoutSeconds); + +/* WarnIfSyncDNS warns if libcurl is compiled with synchronous DNS. */ +void +WarnIfSyncDNS(void) +{ + curl_version_info_data *versionInfo = curl_version_info(CURLVERSION_NOW); + if (!(versionInfo->features & CURL_VERSION_ASYNCHDNS)) + { + ereport(WARNING, (errmsg("your current libcurl version doesn't support " + "asynchronous DNS, which might cause unexpected " + "delays in the operation of Citus"), + errhint("Install a libcurl version with asynchronous DNS " + "support."))); + } +} + + +/* + * CollectBasicUsageStatistics sends basic usage statistics to Citus servers. + * This includes Citus version, table count rounded to next power of 2, cluster + * size rounded to next power of 2, worker node count, and uname data. Returns + * true if we actually have sent statistics to the server. + */ +bool +CollectBasicUsageStatistics(void) +{ + List *distributedTables = NIL; + uint64 roundedDistTableCount = 0; + uint64 roundedClusterSize = 0; + uint32 workerNodeCount = 0; + StringInfo fields = makeStringInfo(); + struct utsname unameData; + memset(&unameData, 0, sizeof(unameData)); + + StartTransactionCommand(); + distributedTables = DistributedTableList(); + roundedDistTableCount = NextPow2(list_length(distributedTables)); + roundedClusterSize = NextPow2(ClusterSize(distributedTables)); + workerNodeCount = ActivePrimaryNodeCount(); + CommitTransactionCommand(); + + uname(&unameData); + + appendStringInfoString(fields, "{\"citus_version\": "); + escape_json(fields, CITUS_VERSION); + appendStringInfo(fields, ",\"table_count\": " UINT64_FORMAT, roundedDistTableCount); + appendStringInfo(fields, ",\"cluster_size\": " UINT64_FORMAT, roundedClusterSize); + appendStringInfo(fields, ",\"worker_node_count\": %u", workerNodeCount); + appendStringInfoString(fields, ",\"os_name\": "); + escape_json(fields, unameData.sysname); + appendStringInfoString(fields, ",\"os_release\": "); + escape_json(fields, unameData.release); + appendStringInfoString(fields, ",\"hwid\": "); + escape_json(fields, unameData.machine); + appendStringInfoString(fields, "}"); + + return SendHttpPostJsonRequest(STATS_COLLECTION_HOST "/v1/usage_reports", + fields->data, HTTP_TIMEOUT_SECONDS); +} + + +/* + * ClusterSize returns total size of data store in the cluster consisting of + * given distributed tables. We ignore tables which we cannot get their size. + */ +static uint64 +ClusterSize(List *distributedTableList) +{ + uint64 clusterSize = 0; + ListCell *distTableCacheEntryCell = NULL; + + foreach(distTableCacheEntryCell, distributedTableList) + { + DistTableCacheEntry *distTableCacheEntry = lfirst(distTableCacheEntryCell); + Oid relationId = distTableCacheEntry->relationId; + MemoryContext savedContext = CurrentMemoryContext; + + PG_TRY(); + { + Datum distTableSizeDatum = DirectFunctionCall1(citus_table_size, + ObjectIdGetDatum(relationId)); + clusterSize += DatumGetInt64(distTableSizeDatum); + } + PG_CATCH(); + { + FlushErrorState(); + + /* citus_table_size() throws an error while the memory context is changed */ + MemoryContextSwitchTo(savedContext); + } + PG_END_TRY(); + } + + return clusterSize; +} + + +/* + * NextPow2 returns smallest power of 2 less than or equal to n. If n is greater + * than 2^63, it returns 2^63. Returns 0 when n is 0. + */ +static uint64 +NextPow2(uint64 n) +{ + uint64 result = 1; + + if (n == 0) + { + return 0; + } + + /* if there is no 64-bit power of 2 greater than n, return 2^63 */ + if (n > (1ull << 63)) + { + return (1ull << 63); + } + + while (result < n) + { + result *= 2; + } + + return result; +} + + +/* + * SendHttpPostJsonRequest sends a HTTP/HTTPS POST request to the given URL with + * the given json object. + */ +static bool +SendHttpPostJsonRequest(const char *url, const char *jsonObj, long timeoutSeconds) +{ + bool success = false; + CURLcode curlCode = false; + CURL *curl = NULL; + + curl_global_init(CURL_GLOBAL_DEFAULT); + curl = curl_easy_init(); + if (curl) + { + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, "charsets: utf-8"); + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, jsonObj); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeoutSeconds); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curlCode = curl_easy_perform(curl); + if (curlCode == CURLE_OK) + { + int httpCode = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode); + if (httpCode == 200) + { + success = true; + } + else if (httpCode >= 400 && httpCode < 500) + { + ereport(WARNING, (errmsg("HTTP request failed."), + errhint("HTTP response code: %d", httpCode))); + } + } + else + { + ereport(WARNING, (errmsg("Sending HTTP POST request failed."), + errhint("Error code: %s.", curl_easy_strerror(curlCode)))); + } + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + } + + curl_global_cleanup(); + + return success; +} + + +#endif /* HAVE_LIBCURL */ diff --git a/src/include/citus_config.h.in b/src/include/citus_config.h.in index 364d22d47..cfabb9809 100644 --- a/src/include/citus_config.h.in +++ b/src/include/citus_config.h.in @@ -22,6 +22,36 @@ /* Citus version as a number */ #undef CITUS_VERSION_NUM +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Define to 1 if you have the `curl' library (-lcurl). */ +#undef HAVE_LIBCURL + +/* Define to 1 if you have the header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_UNISTD_H + /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT @@ -39,3 +69,6 @@ /* Define to the version of this package. */ #undef PACKAGE_VERSION + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS diff --git a/src/include/distributed/maintenanced.h b/src/include/distributed/maintenanced.h index effc15440..da17fe4ea 100644 --- a/src/include/distributed/maintenanced.h +++ b/src/include/distributed/maintenanced.h @@ -12,6 +12,12 @@ #ifndef MAINTENANCED_H #define MAINTENANCED_H +/* collect statistics every 24 hours */ +#define STATISTICS_COLLECTION_INTERVAL 86400 + +/* if statistics collection fails, retry in 1 minute */ +#define STATISTICS_COLLECTION_RETRY_INTERVAL 60 + /* config variable for */ extern double DistributedDeadlockDetectionTimeoutFactor; diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index 8f0eba6c3..9670d554b 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -107,6 +107,11 @@ typedef struct ShardPlacement /* Config variable managed via guc.c */ extern int ReplicationModel; +/* Size functions */ +extern Datum citus_table_size(PG_FUNCTION_ARGS); +extern Datum citus_total_relation_size(PG_FUNCTION_ARGS); +extern Datum citus_relation_size(PG_FUNCTION_ARGS); + /* Function declarations to read shard and shard placement data */ extern uint32 TableShardReplicationFactor(Oid relationId); extern List * LoadShardIntervalList(Oid relationId); diff --git a/src/include/distributed/statistics_collection.h b/src/include/distributed/statistics_collection.h new file mode 100644 index 000000000..a74cadd08 --- /dev/null +++ b/src/include/distributed/statistics_collection.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * statistics_collection.h + * + * Copyright (c) 2017, Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#ifndef STATISTICS_COLLECTION_H +#define STATISTICS_COLLECTION_H + +/* Config variables managed via guc.c */ +extern bool EnableStatisticsCollection; + +#if HAVE_LIBCURL + +#define STATS_COLLECTION_HOST "https://citus-statistics.herokuapp.com" +#define HTTP_TIMEOUT_SECONDS 5 + +extern void WarnIfSyncDNS(void); +extern bool CollectBasicUsageStatistics(void); + +#endif /* HAVE_LIBCURL */ + +#endif /* STATISTICS_COLLECTION_H */ diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 678e3575d..b705d80bd 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -43,36 +43,44 @@ check-full: check-multi check-multi-mx check-multi-task-tracker-extra check-work # for check-worker. But that's harmless besides a few cycles. check-worker: all $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.enable_statistics_collection=0 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/worker_schedule $(EXTRA_TESTS) check-multi: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.enable_statistics_collection=0 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-vg: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus --valgrind \ + --server-option=citus.enable_statistics_collection=0 \ --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-isolation: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus --isolationtester \ + --server-option=citus.enable_statistics_collection=0 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/isolation_schedule $(EXTRA_TESTS) check-vanilla: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citus --vanillatest + $(pg_regress_multi_check) --load-extension=citus --vanillatest \ + --server-option=citus.enable_statistics_collection=0 check-multi-mx: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.enable_statistics_collection=0 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_mx_schedule $(EXTRA_TESTS) check-multi-task-tracker-extra: all tempinstall-main $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.enable_statistics_collection=0 \ --server-option=citus.task_executor_type=task-tracker \ --server-option=citus.large_table_shard_count=1 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_task_tracker_extra_schedule $(EXTRA_TESTS) check-follower-cluster: all $(pg_regress_multi_check) --load-extension=citus --follower-cluster \ + --server-option=citus.enable_statistics_collection=0 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_follower_schedule $(EXTRA_TESTS) clean distclean maintainer-clean: