diff --git a/.gitattributes b/.gitattributes index e3ee9cf5b..5eb7bda7b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,3 +20,23 @@ src/test/regress/output/*.source -whitespace # These files are maintained or generated elsewhere. We take them as is. configure -whitespace + +# all C files (implementation and header) use our style... +*.[ch] citus-style + +# except these exceptions... +src/backend/distributed/utils/citus_outfuncs.c -citus-style +src/backend/distributed/utils/citus_read.c -citus-style +src/backend/distributed/utils/citus_readfuncs_94.c -citus-style +src/backend/distributed/utils/citus_readfuncs_95.c -citus-style +src/backend/distributed/utils/ruleutils_94.c -citus-style +src/backend/distributed/utils/ruleutils_95.c -citus-style +src/include/distributed/citus_nodes.h -citus-style +src/include/dumputils.h -citus-style + +# all csql files use PostgreSQL style... +src/bin/csql/*.[ch] -citus-style + +# except these exceptions +src/bin/csql/copy_options.c citus-style +src/bin/csql/stage.[ch] citus-style diff --git a/Makefile b/Makefile index e024ef4ff..191a6e6f7 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ -# CitusDB toplevel Makefile +# Citus toplevel Makefile -citusdb_subdir = . -citusdb_top_builddir = . +citus_subdir = . +citus_top_builddir = . # Hint that configure should be run first ifeq (,$(wildcard Makefile.global)) - $(error ./configure needs to be run before compiling CitusDB) + $(error ./configure needs to be run before compiling Citus) endif include Makefile.global @@ -20,9 +20,9 @@ install-extension: install-headers: $(MKDIR_P) '$(includedir_server)/distributed/' # generated headers are located in the build directory - $(INSTALL_DATA) src/include/citusdb_config.h '$(includedir_server)/' + $(INSTALL_DATA) src/include/citus_config.h '$(includedir_server)/' # the rest in the source tree - $(INSTALL_DATA) $(citusdb_abs_srcdir)/src/include/distributed/*.h '$(includedir_server)/distributed/' + $(INSTALL_DATA) $(citus_abs_srcdir)/src/include/distributed/*.h '$(includedir_server)/distributed/' clean-extension: $(MAKE) -C src/backend/distributed/ clean .PHONY: extension install-extension clean-extension @@ -42,6 +42,13 @@ clean-csql: install: install-csql clean: clean-csql +# apply or check style +reindent: + cd ${citus_abs_top_srcdir} && citus_indent --quiet +check-style: + cd ${citus_abs_top_srcdir} && citus_indent --quiet --check +.PHONY: reindent check-style + # depend on install for now check: all install $(MAKE) -C src/test/regress check-full diff --git a/Makefile.global.in b/Makefile.global.in index 087072749..60b3d31f0 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -9,40 +9,40 @@ # makefiles, particulary central handling of compilation flags and # rules. -citusdb_abs_srcdir:=@abs_top_srcdir@/${citusdb_subdir} -citusdb_abs_top_srcdir:=@abs_top_srcdir@ +citus_abs_srcdir:=@abs_top_srcdir@/${citus_subdir} +citus_abs_top_srcdir:=@abs_top_srcdir@ PG_CONFIG:=@PG_CONFIG@ PGXS:=$(shell $(PG_CONFIG) --pgxs) # Support for VPATH builds (i.e. builds from outside the source tree) vpath_build=@vpath_build@ ifeq ($(vpath_build),yes) - VPATH:=$(citusdb_abs_srcdir) + VPATH:=$(citus_abs_srcdir) USE_VPATH:=$(VPATH) endif -# CitusDB is built using PostgreSQL's pgxs +# Citus is built using PostgreSQL's pgxs USE_PGXS=1 include $(PGXS) # Remake Makefile.global from Makefile.global.in if the latter # changed. In order to trigger this rule, the including file must -# write `include $(citusdb_top_builddir)/Makefile.global', not some +# write `include $(citus_top_builddir)/Makefile.global', not some # shortcut thereof. This makes it less likely to accidentally run # with some outdated Makefile.global. # Make internally restarts whenever included Makefiles are # regenerated. -$(citusdb_top_builddir)/Makefile.global: $(citusdb_top_builddir)/Makefile.global.in @top_srcdir@/configure $(citusdb_top_builddir)/config.status +$(citus_top_builddir)/Makefile.global: $(citus_top_builddir)/Makefile.global.in @top_srcdir@/configure $(citus_top_builddir)/config.status cd @abs_top_builddir@ && ./config.status Makefile.global # Ensure configuration is generated by the most recent configure, # useful for longer existing build directories. -$(citusdb_top_builddir)/config.status: @top_srcdir@/configure +$(citus_top_builddir)/config.status: @top_srcdir@/configure cd @abs_top_builddir@ && ./config.status --recheck # Regenerate configure if configure.in changed -@top_srcdir@/configure: $(citusdb_abs_srcdir)/configure.in - cd ${citusdb_abs_srcdir} && ./autogen.sh +@top_srcdir@/configure: $(citus_abs_srcdir)/configure.in + cd ${citus_abs_srcdir} && ./autogen.sh # If specified via configure, replace the default compiler. Normally # we'll build with the one postgres was built with. But it's useful to @@ -54,8 +54,8 @@ endif # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/... override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@ -override CPPFLAGS := @CPPFLAGS@ -I '${citusdb_abs_top_srcdir}/src/include' $(CPPFLAGS) +override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' $(CPPFLAGS) override LDFLAGS += @LDFLAGS@ # optional file with user defined, additional, rules --include ${citusdb_abs_srcdir}/src/Makefile.custom +-include ${citus_abs_srcdir}/src/Makefile.custom diff --git a/autogen.sh b/autogen.sh index 9db001b0f..145d333cf 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,7 +1,7 @@ #!/bin/bash # # autogen.sh converts configure.in to configure and creates -# citusdb_config.h.in. The resuting resulting files are checked into +# citus_config.h.in. The resuting resulting files are checked into # the SCM, to avoid everyone needing autoconf installed. autoreconf -f diff --git a/configure b/configure index ce37e3d49..aaf84be6a 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for CitusDB 5.0. +# Generated by GNU Autoconf 2.69 for Citus 5.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,10 +577,10 @@ MFLAGS= MAKEFLAGS= # Identity of this package. -PACKAGE_NAME='CitusDB' -PACKAGE_TARNAME='citusdb' +PACKAGE_NAME='Citus' +PACKAGE_TARNAME='citus' PACKAGE_VERSION='5.0' -PACKAGE_STRING='CitusDB 5.0' +PACKAGE_STRING='Citus 5.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1190,7 +1190,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures CitusDB 5.0 to adapt to many kinds of systems. +\`configure' configures Citus 5.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1238,7 +1238,7 @@ Fine tuning of the installation directories: --infodir=DIR info documentation [DATAROOTDIR/info] --localedir=DIR locale-dependent data [DATAROOTDIR/locale] --mandir=DIR man documentation [DATAROOTDIR/man] - --docdir=DIR documentation root [DATAROOTDIR/doc/citusdb] + --docdir=DIR documentation root [DATAROOTDIR/doc/citus] --htmldir=DIR html documentation [DOCDIR] --dvidir=DIR dvi documentation [DOCDIR] --pdfdir=DIR pdf documentation [DOCDIR] @@ -1251,7 +1251,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of CitusDB 5.0:";; + short | recursive ) echo "Configuration of Citus 5.0:";; esac cat <<\_ACEOF @@ -1333,7 +1333,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -CitusDB configure 5.0 +Citus configure 5.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1390,7 +1390,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by CitusDB $as_me 5.0, which was +It was created by Citus $as_me 5.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -1871,7 +1871,7 @@ if test -z "$version_num"; then fi if test "$version_num" != '9.4' -a "$version_num" != '9.5'; then - as_fn_error $? "CitusDB is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 + as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 $as_echo "$as_me: building against PostgreSQL $version_num" >&6;} @@ -2893,7 +2893,7 @@ CITUS_CFLAGS="$CITUS_CFLAGS" ac_config_files="$ac_config_files Makefile.global" -ac_config_headers="$ac_config_headers src/include/citusdb_config.h" +ac_config_headers="$ac_config_headers src/include/citus_config.h" cat >confcache <<\_ACEOF @@ -3402,7 +3402,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by CitusDB $as_me 5.0, which was +This file was extended by Citus $as_me 5.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3464,7 +3464,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -CitusDB config.status 5.0 +Citus config.status 5.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -3586,7 +3586,7 @@ for ac_config_target in $ac_config_targets do case $ac_config_target in "Makefile.global") CONFIG_FILES="$CONFIG_FILES Makefile.global" ;; - "src/include/citusdb_config.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citusdb_config.h" ;; + "src/include/citus_config.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citus_config.h" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.in b/configure.in index b3bacd4cc..5248f36f5 100644 --- a/configure.in +++ b/configure.in @@ -1,11 +1,11 @@ -# CitusDB autoconf input script. +# Citus autoconf input script. # # Converted into an actual configure script by autogen.sh. This # conversion only has to be done when configure.in changes. To avoid # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([CitusDB], [5.0], [], [citusdb], []) +AC_INIT([Citus], [5.0], [], [citus], []) AC_COPYRIGHT([Copyright (c) Copyright (c) 2012-2015, Citus Data, Inc.]) AC_PROG_SED @@ -32,7 +32,7 @@ if test -z "$version_num"; then fi if test "$version_num" != '9.4' -a "$version_num" != '9.5'; then - AC_MSG_ERROR([CitusDB is not compatible with the detected PostgreSQL version ${version_num}.]) + AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) fi; @@ -96,11 +96,11 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-prototypes]) AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") AC_CONFIG_FILES([Makefile.global]) -AC_CONFIG_HEADERS([src/include/citusdb_config.h]) +AC_CONFIG_HEADERS([src/include/citus_config.h]) AH_TOP([ /* - * citusdb_config.h.in is generated by autoconf/autoheader and - * converted into citusdb_config.h by configure. Include when code needs to + * citus_config.h.in is generated by autoconf/autoheader and + * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! diff --git a/prep_buildtree b/prep_buildtree index dcd7869d7..f39f36c90 100644 --- a/prep_buildtree +++ b/prep_buildtree @@ -1,8 +1,8 @@ #! /bin/sh # -# CitusDB copy of PostgreSQL's config/prep_buildtree +# Citus copy of PostgreSQL's config/prep_buildtree # -# This script prepares a CitusDB build tree for an out-of-tree/VPATH +# This script prepares a Citus build tree for an out-of-tree/VPATH # build. It is intended to be run by the configure script. me=`basename $0` diff --git a/src/backend/distributed/.gitignore b/src/backend/distributed/.gitignore index b5abe3631..3c9a69e03 100644 --- a/src/backend/distributed/.gitignore +++ b/src/backend/distributed/.gitignore @@ -10,4 +10,4 @@ /tmp_check* # ignore latest install file -citusdb--5.0.sql +citus--5.0.sql diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index ab24267ef..8f4bd899a 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -1,13 +1,13 @@ -# Makefile for the CitusDB extension +# Makefile for the Citus extension -citusdb_subdir = src/backend/distributed -citusdb_top_builddir = ../../.. +citus_subdir = src/backend/distributed +citus_top_builddir = ../../.. -MODULE_big = citusdb -EXTENSION = citusdb +MODULE_big = citus +EXTENSION = citus EXTVERSION = 5.0 DATA_built = $(EXTENSION)--$(EXTVERSION).sql -SCRIPTS = $(wildcard $(citusdb_top_builddir)/src/bin/scripts/*) +SCRIPTS = $(wildcard $(citus_top_builddir)/src/bin/scripts/*) # directories with source files SUBDIRS = . commands executor master planner relay test utils worker @@ -15,7 +15,7 @@ SUBDIRS = . commands executor master planner relay test utils worker # That patsubst rule searches all directories listed in SUBDIRS for .c # files, and adds the corresponding .o files to OBJS OBJS += \ - $(patsubst $(citusdb_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(wildcard $(citusdb_abs_srcdir)/$(dir)/*.c))) + $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(wildcard $(citus_abs_srcdir)/$(dir)/*.c))) # define build process for latest install file $(EXTENSION)--$(EXTVERSION).sql: $(EXTENSION).sql @@ -28,6 +28,6 @@ NO_PGXS = 1 SHLIB_LINK = $(libpq) -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global override CPPFLAGS += -I$(libpq_srcdir) diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control new file mode 100644 index 000000000..0f25b1dd7 --- /dev/null +++ b/src/backend/distributed/citus.control @@ -0,0 +1,6 @@ +# Citus extension +comment = 'Citus distributed database' +default_version = '5.0' +module_pathname = '$libdir/citus' +relocatable = false +schema = pg_catalog diff --git a/src/backend/distributed/citusdb.sql b/src/backend/distributed/citus.sql similarity index 91% rename from src/backend/distributed/citusdb.sql rename to src/backend/distributed/citus.sql index c2bb330e5..9fb23bb7f 100644 --- a/src/backend/distributed/citusdb.sql +++ b/src/backend/distributed/citus.sql @@ -1,24 +1,24 @@ -/* citusdb.sql */ +/* citus.sql */ -- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION citusdb" to load this file. \quit +\echo Use "CREATE EXTENSION citus" to load this file. \quit -CREATE SCHEMA citusdb; +CREATE SCHEMA citus; --- Ensure CREATE EXTENSION is not run against an old citusdb data +-- Ensure CREATE EXTENSION is not run against an old citus data -- directory, we're not compatible (due to the builtin functions/tables) DO $$ BEGIN IF EXISTS(SELECT * FROM pg_proc WHERE proname = 'worker_apply_shard_ddl_command') THEN - RAISE 'cannot install citusdb extension in CitusDB 4 data directory'; + RAISE 'cannot install citus extension in Citus 4 data directory'; END IF; END; $$; /***************************************************************************** - * CitusDB data types + * Citus data types *****************************************************************************/ -CREATE TYPE citusdb.distribution_type AS ENUM ( +CREATE TYPE citus.distribution_type AS ENUM ( 'hash', 'range', 'append' @@ -26,18 +26,18 @@ CREATE TYPE citusdb.distribution_type AS ENUM ( /***************************************************************************** - * CitusDB tables & corresponding indexes + * Citus tables & corresponding indexes *****************************************************************************/ -CREATE TABLE citusdb.pg_dist_partition( +CREATE TABLE citus.pg_dist_partition( logicalrelid Oid NOT NULL, partmethod "char" NOT NULL, partkey text NOT NULL ); CREATE UNIQUE INDEX pg_dist_partition_logical_relid_index -ON citusdb.pg_dist_partition using btree(logicalrelid); -ALTER TABLE citusdb.pg_dist_partition SET SCHEMA pg_catalog; +ON citus.pg_dist_partition using btree(logicalrelid); +ALTER TABLE citus.pg_dist_partition SET SCHEMA pg_catalog; -CREATE TABLE citusdb.pg_dist_shard( +CREATE TABLE citus.pg_dist_shard( logicalrelid oid NOT NULL, shardid int8 NOT NULL, shardstorage "char" NOT NULL, @@ -46,12 +46,12 @@ CREATE TABLE citusdb.pg_dist_shard( shardmaxvalue text ); CREATE UNIQUE INDEX pg_dist_shard_shardid_index -ON citusdb.pg_dist_shard using btree(shardid); +ON citus.pg_dist_shard using btree(shardid); CREATE INDEX pg_dist_shard_logical_relid_index -ON citusdb.pg_dist_shard using btree(logicalrelid); -ALTER TABLE citusdb.pg_dist_shard SET SCHEMA pg_catalog; +ON citus.pg_dist_shard using btree(logicalrelid); +ALTER TABLE citus.pg_dist_shard SET SCHEMA pg_catalog; -CREATE TABLE citusdb.pg_dist_shard_placement( +CREATE TABLE citus.pg_dist_shard_placement( shardid int8 NOT NULL, shardstate int4 NOT NULL, shardlength int8 NOT NULL, @@ -59,40 +59,40 @@ CREATE TABLE citusdb.pg_dist_shard_placement( nodeport int8 NOT NULL ) WITH oids; CREATE UNIQUE INDEX pg_dist_shard_placement_oid_index -ON citusdb.pg_dist_shard_placement using btree(oid); +ON citus.pg_dist_shard_placement using btree(oid); CREATE INDEX pg_dist_shard_placement_shardid_index -ON citusdb.pg_dist_shard_placement using btree(shardid); +ON citus.pg_dist_shard_placement using btree(shardid); CREATE INDEX pg_dist_shard_placement_nodeid_index -ON citusdb.pg_dist_shard_placement using btree(nodename, nodeport); -ALTER TABLE citusdb.pg_dist_shard_placement SET SCHEMA pg_catalog; +ON citus.pg_dist_shard_placement using btree(nodename, nodeport); +ALTER TABLE citus.pg_dist_shard_placement SET SCHEMA pg_catalog; /***************************************************************************** - * CitusDB sequences + * Citus sequences *****************************************************************************/ /* * Unternal sequence to generate 64-bit shard ids. These identifiers are then * used to identify shards in the distributed database. */ -CREATE SEQUENCE citusdb.pg_dist_shardid_seq +CREATE SEQUENCE citus.pg_dist_shardid_seq MINVALUE 102008 NO CYCLE; -ALTER SEQUENCE citusdb.pg_dist_shardid_seq SET SCHEMA pg_catalog; +ALTER SEQUENCE citus.pg_dist_shardid_seq SET SCHEMA pg_catalog; /* * internal sequence to generate 32-bit jobIds. These identifiers are then * used to identify jobs in the distributed database; and they wrap at 32-bits * to allow for slave nodes to independently execute their distributed jobs. */ -CREATE SEQUENCE citusdb.pg_dist_jobid_seq +CREATE SEQUENCE citus.pg_dist_jobid_seq MINVALUE 2 /* first jobId reserved for clean up jobs */ MAXVALUE 4294967296; -ALTER SEQUENCE citusdb.pg_dist_jobid_seq SET SCHEMA pg_catalog; +ALTER SEQUENCE citus.pg_dist_jobid_seq SET SCHEMA pg_catalog; /***************************************************************************** - * CitusDB functions + * Citus functions *****************************************************************************/ /* For backward compatibility and ease of use create functions et al. in pg_catalog */ @@ -182,13 +182,13 @@ COMMENT ON FUNCTION master_get_round_robin_candidate_nodes(shard_id bigint) CREATE FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, - distribution_method citusdb.distribution_type) + distribution_method citus.distribution_type) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_create_distributed_table$$; COMMENT ON FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, - distribution_method citusdb.distribution_type) + distribution_method citus.distribution_type) IS 'define the table distribution functions'; -- define shard creation function for hash-partitioned tables @@ -323,7 +323,7 @@ COMMENT ON FUNCTION worker_append_table_to_shard(text, text, text, integer) /* trigger functions */ -CREATE OR REPLACE FUNCTION citusdb_drop_trigger() +CREATE OR REPLACE FUNCTION citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog @@ -349,7 +349,7 @@ BEGIN END LOOP; END; $cdbdt$; -COMMENT ON FUNCTION citusdb_drop_trigger() +COMMENT ON FUNCTION citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; CREATE FUNCTION master_dist_partition_cache_invalidate() @@ -369,21 +369,21 @@ COMMENT ON FUNCTION master_dist_shard_cache_invalidate() /* internal functions, not user accessible */ -CREATE FUNCTION citusdb_extradata_container(INTERNAL) +CREATE FUNCTION citus_extradata_container(INTERNAL) RETURNS void LANGUAGE C - AS 'MODULE_PATHNAME', $$citusdb_extradata_container$$; -COMMENT ON FUNCTION pg_catalog.citusdb_extradata_container(INTERNAL) + AS 'MODULE_PATHNAME', $$citus_extradata_container$$; +COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; /***************************************************************************** - * CitusDB triggers + * Citus triggers *****************************************************************************/ -CREATE EVENT TRIGGER citusdb_cascade_to_partition +CREATE EVENT TRIGGER citus_cascade_to_partition ON SQL_DROP - EXECUTE PROCEDURE citusdb_drop_trigger(); + EXECUTE PROCEDURE citus_drop_trigger(); CREATE TRIGGER dist_partition_cache_invalidate AFTER INSERT OR UPDATE OR DELETE @@ -397,7 +397,7 @@ CREATE TRIGGER dist_shard_cache_invalidate /***************************************************************************** - * CitusDB aggregates + * Citus aggregates *****************************************************************************/ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) diff --git a/src/backend/distributed/citusdb.control b/src/backend/distributed/citusdb.control deleted file mode 100644 index ae36339b5..000000000 --- a/src/backend/distributed/citusdb.control +++ /dev/null @@ -1,6 +0,0 @@ -# CitusDB extension -comment = 'CitusDB distributed database' -default_version = '5.0' -module_pathname = '$libdir/citusdb' -relocatable = false -schema = pg_catalog diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 7eabbf4fc..9ae1a7762 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -165,7 +165,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) * * Similarly, do not allow UNIQUE constraint and/or PRIMARY KEY if it does not * include partition column. This check is important for two reasons. First, - * currently CitusDB does not enforce uniqueness constraint on multiple shards. + * currently Citus does not enforce uniqueness constraint on multiple shards. * Second, INSERT INTO .. ON CONFLICT (i.e., UPSERT) queries can be executed with no * further check for constraints. */ @@ -191,17 +191,17 @@ master_create_distributed_table(PG_FUNCTION_ARGS) } /* - * CitusDB cannot enforce uniqueness constraints with overlapping shards. Thus, + * Citus cannot enforce uniqueness constraints with overlapping shards. Thus, * emit a warning for unique indexes on append partitioned tables. */ if (distributionMethod == DISTRIBUTE_BY_APPEND) { ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("table \"%s\" has a unique constraint", - distributedRelationName), - errdetail("Unique constraints and primary keys on " - "append-partitioned tables cannot be enforced."), - errhint("Consider using hash partitioning."))); + errmsg("table \"%s\" has a unique constraint", + distributedRelationName), + errdetail("Unique constraints and primary keys on " + "append-partitioned tables cannot be enforced."), + errhint("Consider using hash partitioning."))); } attributeCount = indexInfo->ii_NumIndexAttrs; @@ -262,7 +262,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) * necessary for a distributed relation in addition to the preexisting ones * for a normal relation. * - * We create one dependency from the (now distributed) relation to the citusdb + * We create one dependency from the (now distributed) relation to the citus * extension to prevent the extension from being dropped while distributed * tables exist. Furthermore a dependency from pg_dist_partition's * distribution clause to the underlying columns is created, but it's marked @@ -281,7 +281,7 @@ RecordDistributedRelationDependencies(Oid distributedRelationId, Node *distribut relationAddr.objectSubId = 0; citusExtensionAddr.classId = ExtensionRelationId; - citusExtensionAddr.objectId = get_extension_oid("citusdb", false); + citusExtensionAddr.objectId = get_extension_oid("citus", false); citusExtensionAddr.objectSubId = 0; /* dependency from table entry to extension */ @@ -294,10 +294,10 @@ RecordDistributedRelationDependencies(Oid distributedRelationId, Node *distribut /* - * LookupDistributionMethod maps the oids of citusdb.distribution_type enum + * LookupDistributionMethod maps the oids of citus.distribution_type enum * values to pg_dist_partition.partmethod values. * - * The passed in oid has to belong to a value of citusdb.distribution_type. + * The passed in oid has to belong to a value of citus.distribution_type. */ static char LookupDistributionMethod(Oid distributionMethodOid) diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index 0ab90d0ef..6e0d78136 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -136,7 +136,7 @@ static File FileOpenForTransmit(const char *filename, int fileFlags, int fileMode) { File fileDesc = -1; - int fileStated = -1; + int fileStated = -1; struct stat fileStat; fileStated = stat(filename, &fileStat); @@ -145,7 +145,7 @@ FileOpenForTransmit(const char *filename, int fileFlags, int fileMode) if (S_ISDIR(fileStat.st_mode)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is a directory", filename))); + errmsg("\"%s\" is a directory", filename))); } } @@ -270,18 +270,28 @@ ReceiveCopyData(StringInfo copyData) switch (messageType) { - case 'd': /* CopyData */ + case 'd': /* CopyData */ + { copyDone = false; break; - case 'c': /* CopyDone */ + } + + case 'c': /* CopyDone */ + { copyDone = true; break; - case 'f': /* CopyFail */ + } + + case 'f': /* CopyFail */ + { ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), errmsg("COPY data failed: %s", pq_getmsgstring(copyData)))); break; - case 'H': /* Flush */ - case 'S': /* Sync */ + } + + case 'H': /* Flush */ + case 'S': /* Sync */ + { /* * Ignore Flush/Sync for the convenience of client libraries (such * as libpq) that may send those without noticing that the command @@ -289,11 +299,15 @@ ReceiveCopyData(StringInfo copyData) */ copyDone = false; break; + } + default: + { ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("unexpected message type 0x%02X during COPY data", messageType))); break; + } } return copyDone; diff --git a/src/backend/distributed/executor/multi_client_executor.c b/src/backend/distributed/executor/multi_client_executor.c index e6b4ceef7..57ab6c1b9 100644 --- a/src/backend/distributed/executor/multi_client_executor.c +++ b/src/backend/distributed/executor/multi_client_executor.c @@ -301,7 +301,7 @@ MultiClientCancel(int32 connectionId) if (cancelSent == 0) { ereport(WARNING, (errmsg("could not issue cancel request"), - errdetail("Client error: %s", errorBuffer))); + errdetail("Client error: %s", errorBuffer))); canceled = false; } @@ -348,7 +348,7 @@ MultiClientResultStatus(int32 connectionId) } else { - ereport(WARNING, (errmsg("could not consume data from worker node"))); + ereport(WARNING, (errmsg("could not consume data from worker node"))); resultStatus = CLIENT_RESULT_UNAVAILABLE; } @@ -589,7 +589,7 @@ MultiClientCopyData(int32 connectionId, int32 fileDescriptor) while (receiveLength > 0) { /* received copy data; append these data to file */ - int appended = -1; + int appended = -1; errno = 0; appended = write(fileDescriptor, receiveBuffer, receiveLength); @@ -706,7 +706,7 @@ ClientConnectionReady(PGconn *connection, PostgresPollingStatusType pollingStatu fd_set readFileDescriptorSet; fd_set writeFileDescriptorSet; fd_set exceptionFileDescriptorSet; - struct timeval immediateTimeout = {0, 0}; + struct timeval immediateTimeout = { 0, 0 }; int connectionFileDescriptor = PQsocket(connection); FD_ZERO(&readFileDescriptorSet); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 25fd12640..145abb4d5 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -157,7 +157,6 @@ multi_ExecutorStart(QueryDesc *queryDesc, int eflags) queryDesc->plannedstmt = masterSelectPlan; eflags |= EXEC_FLAG_CITUS_MASTER_SELECT; } - } /* if the execution is not done for router executor, drop into standard executor */ @@ -253,7 +252,7 @@ multi_ExecutorEnd(QueryDesc *queryDesc) RangeTblEntry *rangeTableEntry = linitial(planStatement->rtable); Oid masterTableRelid = rangeTableEntry->relid; - ObjectAddress masterTableObject = {InvalidOid, InvalidOid, 0}; + ObjectAddress masterTableObject = { InvalidOid, InvalidOid, 0 }; masterTableObject.classId = RelationRelationId; masterTableObject.objectId = masterTableRelid; diff --git a/src/backend/distributed/executor/multi_real_time_executor.c b/src/backend/distributed/executor/multi_real_time_executor.c index 77436612c..e3050f64b 100644 --- a/src/backend/distributed/executor/multi_real_time_executor.c +++ b/src/backend/distributed/executor/multi_real_time_executor.c @@ -89,7 +89,7 @@ MultiRealTimeExecute(Job *job) } /* loop around until all tasks complete, one task fails, or user cancels */ - while ( !(allTasksCompleted || taskFailed || QueryCancelPending) ) + while (!(allTasksCompleted || taskFailed || QueryCancelPending)) { uint32 taskCount = list_length(taskList); uint32 completedTaskCount = 0; @@ -230,333 +230,338 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution) switch (currentStatus) { - case EXEC_TASK_CONNECT_START: - { - int32 connectionId = INVALID_CONNECTION_ID; - char *nodeDatabase = NULL; - - /* we use the same database name on the master and worker nodes */ - nodeDatabase = get_database_name(MyDatabaseId); - - connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); - connectionIdArray[currentIndex] = connectionId; - - /* if valid, poll the connection until the connection is initiated */ - if (connectionId != INVALID_CONNECTION_ID) + case EXEC_TASK_CONNECT_START: { - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; - taskExecution->connectPollCount = 0; - connectAction = CONNECT_ACTION_OPENED; - } - else - { - AdjustStateForFailure(taskExecution); - } + int32 connectionId = INVALID_CONNECTION_ID; + char *nodeDatabase = NULL; - break; - } + /* we use the same database name on the master and worker nodes */ + nodeDatabase = get_database_name(MyDatabaseId); - case EXEC_TASK_CONNECT_POLL: - { - int32 connectionId = connectionIdArray[currentIndex]; - ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); + connectionIdArray[currentIndex] = connectionId; - /* - * If the connection is established, we reset the data fetch counter and - * change our status to data fetching. - */ - if (pollStatus == CLIENT_CONNECTION_READY) - { - taskExecution->dataFetchTaskIndex = -1; - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; - } - else if (pollStatus == CLIENT_CONNECTION_BUSY) - { - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; - } - else if (pollStatus == CLIENT_CONNECTION_BAD) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - /* now check if we have been trying to connect for too long */ - taskExecution->connectPollCount++; - if (pollStatus == CLIENT_CONNECTION_BUSY) - { - uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; - uint32 currentCount = taskExecution->connectPollCount; - if (currentCount >= maxCount) + /* if valid, poll the connection until the connection is initiated */ + if (connectionId != INVALID_CONNECTION_ID) { - ereport(WARNING, (errmsg("could not establish asynchronous connection " - "after %u ms", REMOTE_NODE_CONNECT_TIMEOUT))); + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; + taskExecution->connectPollCount = 0; + connectAction = CONNECT_ACTION_OPENED; + } + else + { + AdjustStateForFailure(taskExecution); + } + break; + } + + case EXEC_TASK_CONNECT_POLL: + { + int32 connectionId = connectionIdArray[currentIndex]; + ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + + /* + * If the connection is established, we reset the data fetch counter and + * change our status to data fetching. + */ + if (pollStatus == CLIENT_CONNECTION_READY) + { + taskExecution->dataFetchTaskIndex = -1; + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; + } + else if (pollStatus == CLIENT_CONNECTION_BUSY) + { + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; + } + else if (pollStatus == CLIENT_CONNECTION_BAD) + { taskStatusArray[currentIndex] = EXEC_TASK_FAILED; } - } - break; - } - - case EXEC_TASK_FAILED: - { - /* - * On task failure, we close the connection. We also reset our execution - * status assuming that we might fail on all other worker nodes and come - * back to this failed node. In that case, we will retry the same fetch - * and compute task(s) on this node again. - */ - int32 connectionId = connectionIdArray[currentIndex]; - MultiClientDisconnect(connectionId); - connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; - connectAction = CONNECT_ACTION_CLOSED; - - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START; - - /* try next worker node */ - AdjustStateForFailure(taskExecution); - - break; - } - - case EXEC_FETCH_TASK_LOOP: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskCount = list_length(dataFetchTaskList); - - /* move to the next data fetch task */ - taskExecution->dataFetchTaskIndex++; - - if (taskExecution->dataFetchTaskIndex < dataFetchTaskCount) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_START; - } - else - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; - } - - break; - } - - case EXEC_FETCH_TASK_START: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex; - Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, dataFetchTaskIndex); - - char *dataFetchQuery = dataFetchTask->queryString; - int32 connectionId = connectionIdArray[currentIndex]; - - bool querySent = MultiClientSendQuery(connectionId, dataFetchQuery); - if (querySent) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - } - else - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - break; - } - - case EXEC_FETCH_TASK_RUNNING: - { - int32 connectionId = connectionIdArray[currentIndex]; - ResultStatus resultStatus = MultiClientResultStatus(connectionId); - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - - /* check if query results are in progress or unavailable */ - if (resultStatus == CLIENT_RESULT_BUSY) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - break; - } - else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - break; - } - - Assert(resultStatus == CLIENT_RESULT_READY); - - /* - * If the query executed successfully, loop onto the next data fetch - * task. Else if the query failed, try data fetching on another node. - */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_DONE) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; - } - else if (queryStatus == CLIENT_QUERY_FAILED) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - else - { - ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); - } - - break; - } - - case EXEC_COMPUTE_TASK_START: - { - int32 connectionId = connectionIdArray[currentIndex]; - bool querySent = false; - - /* construct new query to copy query results to stdout */ - char *queryString = task->queryString; - StringInfo computeTaskQuery = makeStringInfo(); - if (BinaryMasterCopyFormat) - { - appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY, queryString); - } - else - { - appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT, queryString); - } - - querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data); - if (querySent) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; - } - else - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - break; - } - - case EXEC_COMPUTE_TASK_RUNNING: - { - int32 connectionId = connectionIdArray[currentIndex]; - ResultStatus resultStatus = MultiClientResultStatus(connectionId); - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - - /* check if query results are in progress or unavailable */ - if (resultStatus == CLIENT_RESULT_BUSY) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; - break; - } - else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - break; - } - - Assert(resultStatus == CLIENT_RESULT_READY); - - /* check if our request to copy query results has been acknowledged */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_COPY) - { - StringInfo jobDirectoryName = JobDirectoryName(task->jobId); - StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); - - char *filename = taskFilename->data; - int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - int fileMode = (S_IRUSR | S_IWUSR); - - int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); - if (fileDescriptor >= 0) + /* now check if we have been trying to connect for too long */ + taskExecution->connectPollCount++; + if (pollStatus == CLIENT_CONNECTION_BUSY) + { + uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; + uint32 currentCount = taskExecution->connectPollCount; + if (currentCount >= maxCount) + { + ereport(WARNING, (errmsg("could not establish asynchronous " + "connection after %u ms", + REMOTE_NODE_CONNECT_TIMEOUT))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + + break; + } + + case EXEC_TASK_FAILED: + { + /* + * On task failure, we close the connection. We also reset our execution + * status assuming that we might fail on all other worker nodes and come + * back to this failed node. In that case, we will retry the same fetch + * and compute task(s) on this node again. + */ + int32 connectionId = connectionIdArray[currentIndex]; + MultiClientDisconnect(connectionId); + connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; + connectAction = CONNECT_ACTION_CLOSED; + + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START; + + /* try next worker node */ + AdjustStateForFailure(taskExecution); + + break; + } + + case EXEC_FETCH_TASK_LOOP: + { + List *dataFetchTaskList = task->dependedTaskList; + int32 dataFetchTaskCount = list_length(dataFetchTaskList); + + /* move to the next data fetch task */ + taskExecution->dataFetchTaskIndex++; + + if (taskExecution->dataFetchTaskIndex < dataFetchTaskCount) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_START; + } + else + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; + } + + break; + } + + case EXEC_FETCH_TASK_START: + { + List *dataFetchTaskList = task->dependedTaskList; + int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex; + Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, + dataFetchTaskIndex); + + char *dataFetchQuery = dataFetchTask->queryString; + int32 connectionId = connectionIdArray[currentIndex]; + + bool querySent = MultiClientSendQuery(connectionId, dataFetchQuery); + if (querySent) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; + } + else + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + + break; + } + + case EXEC_FETCH_TASK_RUNNING: + { + int32 connectionId = connectionIdArray[currentIndex]; + ResultStatus resultStatus = MultiClientResultStatus(connectionId); + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + + /* check if query results are in progress or unavailable */ + if (resultStatus == CLIENT_RESULT_BUSY) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; + break; + } + else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + break; + } + + Assert(resultStatus == CLIENT_RESULT_READY); + + /* + * If the query executed successfully, loop onto the next data fetch + * task. Else if the query failed, try data fetching on another node. + */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_DONE) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; + } + else if (queryStatus == CLIENT_QUERY_FAILED) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + else + { + ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + } + + break; + } + + case EXEC_COMPUTE_TASK_START: + { + int32 connectionId = connectionIdArray[currentIndex]; + bool querySent = false; + + /* construct new query to copy query results to stdout */ + char *queryString = task->queryString; + StringInfo computeTaskQuery = makeStringInfo(); + if (BinaryMasterCopyFormat) + { + appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY, + queryString); + } + else + { + appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT, + queryString); + } + + querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data); + if (querySent) + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; + } + else + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + + break; + } + + case EXEC_COMPUTE_TASK_RUNNING: + { + int32 connectionId = connectionIdArray[currentIndex]; + ResultStatus resultStatus = MultiClientResultStatus(connectionId); + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + + /* check if query results are in progress or unavailable */ + if (resultStatus == CLIENT_RESULT_BUSY) + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; + break; + } + else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + break; + } + + Assert(resultStatus == CLIENT_RESULT_READY); + + /* check if our request to copy query results has been acknowledged */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_COPY) + { + StringInfo jobDirectoryName = JobDirectoryName(task->jobId); + StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); + + char *filename = taskFilename->data; + int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); + int fileMode = (S_IRUSR | S_IWUSR); + + int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); + if (fileDescriptor >= 0) + { + /* + * All files inside the job directory get automatically cleaned + * up on transaction commit or abort. + */ + fileDescriptorArray[currentIndex] = fileDescriptor; + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + filename))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + else if (queryStatus == CLIENT_QUERY_FAILED) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + else + { + ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + } + + break; + } + + case EXEC_COMPUTE_TASK_COPYING: + { + int32 connectionId = connectionIdArray[currentIndex]; + int32 fileDesc = fileDescriptorArray[currentIndex]; + int closed = -1; + + /* copy data from worker node, and write to local file */ + CopyStatus copyStatus = MultiClientCopyData(connectionId, fileDesc); + + /* if worker node will continue to send more data, keep reading */ + if (copyStatus == CLIENT_COPY_MORE) { - /* - * All files inside the job directory get automatically cleaned - * up on transaction commit or abort. - */ - fileDescriptorArray[currentIndex] = fileDescriptor; taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; } - else + else if (copyStatus == CLIENT_COPY_DONE) { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", filename))); + closed = close(fileDesc); + fileDescriptorArray[currentIndex] = -1; + if (closed >= 0) + { + taskStatusArray[currentIndex] = EXEC_TASK_DONE; + + /* we are done executing; we no longer need the connection */ + MultiClientDisconnect(connectionId); + connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; + connectAction = CONNECT_ACTION_CLOSED; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copied file: %m"))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + else if (copyStatus == CLIENT_COPY_FAILED) + { taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + + closed = close(fileDesc); + fileDescriptorArray[currentIndex] = -1; + + if (closed < 0) + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copy file: %m"))); + } } + + break; } - else if (queryStatus == CLIENT_QUERY_FAILED) + + case EXEC_TASK_DONE: { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + /* we are done with this task's execution */ + break; } - else + + default: { - ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid execution status: %d", currentStatus))); + break; } - - break; - } - - case EXEC_COMPUTE_TASK_COPYING: - { - int32 connectionId = connectionIdArray[currentIndex]; - int32 fileDesc = fileDescriptorArray[currentIndex]; - int closed = -1; - - /* copy data from worker node, and write to local file */ - CopyStatus copyStatus = MultiClientCopyData(connectionId, fileDesc); - - /* if worker node will continue to send more data, keep reading */ - if (copyStatus == CLIENT_COPY_MORE) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; - } - else if (copyStatus == CLIENT_COPY_DONE) - { - closed = close(fileDesc); - fileDescriptorArray[currentIndex] = -1; - - if (closed >= 0) - { - taskStatusArray[currentIndex] = EXEC_TASK_DONE; - - /* we are done executing; we no longer need the connection */ - MultiClientDisconnect(connectionId); - connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; - connectAction = CONNECT_ACTION_CLOSED; - } - else - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copied file: %m"))); - - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - } - else if (copyStatus == CLIENT_COPY_FAILED) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - - closed = close(fileDesc); - fileDescriptorArray[currentIndex] = -1; - - if (closed < 0) - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copy file: %m"))); - } - } - - break; - } - - case EXEC_TASK_DONE: - { - /* we are done with this task's execution */ - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid execution status: %d", currentStatus))); - break; - } } return connectAction; diff --git a/src/backend/distributed/executor/multi_router_executor.c b/src/backend/distributed/executor/multi_router_executor.c index 5a6cdd06e..29fdcbfee 100644 --- a/src/backend/distributed/executor/multi_router_executor.c +++ b/src/backend/distributed/executor/multi_router_executor.c @@ -80,6 +80,7 @@ RouterExecutorStart(QueryDesc *queryDesc, int eflags, Task *task) queryDesc->estate = executorState; #if (PG_VERSION_NUM < 90500) + /* make sure that upsertQuery is false for versions that UPSERT is not available */ Assert(task->upsertQuery == false); #endif @@ -153,7 +154,7 @@ CommutativityRuleToLockMode(CmdType commandType, bool upsertQuery) static void AcquireExecutorShardLock(Task *task, LOCKMODE lockMode) { - int64 shardId = task->shardId; + int64 shardId = task->anchorShardId; LockShardResource(shardId, lockMode); } @@ -177,14 +178,14 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas if (!ScanDirectionIsForward(direction)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("scan directions other than forward scans " - "are unsupported"))); + errmsg("scan directions other than forward scans " + "are unsupported"))); } if (count != 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("fetching rows from a query using a cursor " - "is unsupported"))); + errmsg("fetching rows from a query using a cursor " + "is unsupported"))); } oldcontext = MemoryContextSwitchTo(estate->es_query_cxt); @@ -210,7 +211,7 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas else { ereport(ERROR, (errmsg("unrecognized operation code: %d", - (int) operation))); + (int) operation))); } if (queryDesc->totaltime != NULL) @@ -219,9 +220,9 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas } MemoryContextSwitchTo(oldcontext); - } + /* * ExecuteDistributedModify is the main entry point for modifying distributed * tables. A distributed modification is successful if any placement of the @@ -250,7 +251,7 @@ ExecuteDistributedModify(Task *task) Assert(taskPlacement->shardState == FILE_FINALIZED); - connection = GetConnection(nodeName, nodePort); + connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { failedPlacementList = lappend(failedPlacementList, taskPlacement); @@ -383,7 +384,7 @@ ExecuteTaskAndStoreResults(Task *task, TupleDesc tupleDescriptor, bool queryOK = false; bool storedOK = false; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { continue; @@ -532,9 +533,10 @@ StoreQueryResult(PGconn *connection, TupleDesc tupleDescriptor, return true; } + /* -* RouterExecutorFinish cleans up after a distributed execution. -*/ + * RouterExecutorFinish cleans up after a distributed execution. + */ void RouterExecutorFinish(QueryDesc *queryDesc) { diff --git a/src/backend/distributed/executor/multi_server_executor.c b/src/backend/distributed/executor/multi_server_executor.c index 1f143778d..22b958989 100644 --- a/src/backend/distributed/executor/multi_server_executor.c +++ b/src/backend/distributed/executor/multi_server_executor.c @@ -73,7 +73,7 @@ JobExecutorType(MultiPlan *multiPlan) ereport(WARNING, (errmsg("this query uses more connections than the " "configured max_connections limit"), errhint("Consider increasing max_connections or setting " - "citusdb.task_executor_type to " + "citus.task_executor_type to " "\"task-tracker\"."))); } @@ -88,7 +88,7 @@ JobExecutorType(MultiPlan *multiPlan) ereport(WARNING, (errmsg("this query uses more file descriptors than the " "configured max_files_per_process limit"), errhint("Consider increasing max_files_per_process or " - "setting citusdb.task_executor_type to " + "setting citus.task_executor_type to " "\"task-tracker\"."))); } @@ -96,7 +96,7 @@ JobExecutorType(MultiPlan *multiPlan) if (dependedJobCount > 0) { ereport(ERROR, (errmsg("cannot use real time executor with repartition jobs"), - errhint("Set citusdb.task_executor_type to " + errhint("Set citus.task_executor_type to " "\"task-tracker\"."))); } } @@ -119,7 +119,7 @@ JobExecutorType(MultiPlan *multiPlan) if (dependedJobCount > 0) { ereport(ERROR, (errmsg("cannot use router executor with repartition jobs"), - errhint("Set citusdb.task_executor_type to " + errhint("Set citus.task_executor_type to " "\"task-tracker\"."))); } @@ -128,7 +128,7 @@ JobExecutorType(MultiPlan *multiPlan) { ereport(ERROR, (errmsg("cannot use router executor with queries that " "hit multiple shards"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -138,7 +138,7 @@ JobExecutorType(MultiPlan *multiPlan) if (list_length(workerDependentTaskList) > 0) { ereport(ERROR, (errmsg("cannot use router executor with JOINs"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -146,7 +146,7 @@ JobExecutorType(MultiPlan *multiPlan) if (masterQuery != NULL && list_length(masterQuery->sortClause) > 0) { ereport(ERROR, (errmsg("cannot use router executor with ORDER BY clauses"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -158,7 +158,7 @@ JobExecutorType(MultiPlan *multiPlan) if (masterQueryHasAggregates) { ereport(ERROR, (errmsg("cannot use router executor with aggregates"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } } @@ -173,7 +173,7 @@ JobExecutorType(MultiPlan *multiPlan) * Every task requires 2 FDs, one file and one connection. Some FDs are taken by * the VFD pool and there is currently no way to reclaim these before opening a * connection. We therefore assume some FDs to be reserved for VFDs, based on - * observing a typical size of the pool on a CitusDB master. + * observing a typical size of the pool on a Citus master. */ int MaxMasterConnectionCount(void) @@ -303,13 +303,13 @@ AdjustStateForFailure(TaskExecution *taskExecution) if (taskExecution->currentNodeIndex < maxNodeIndex) { - taskExecution->currentNodeIndex++; /* try next worker node */ + taskExecution->currentNodeIndex++; /* try next worker node */ } else { taskExecution->currentNodeIndex = 0; /* go back to the first worker node */ } - taskExecution->dataFetchTaskIndex = -1; /* reset data fetch counter */ - taskExecution->failureCount++; /* record failure */ + taskExecution->dataFetchTaskIndex = -1; /* reset data fetch counter */ + taskExecution->failureCount++; /* record failure */ } diff --git a/src/backend/distributed/executor/multi_task_tracker_executor.c b/src/backend/distributed/executor/multi_task_tracker_executor.c index f67c82271..14c26aef9 100644 --- a/src/backend/distributed/executor/multi_task_tracker_executor.c +++ b/src/backend/distributed/executor/multi_task_tracker_executor.c @@ -38,10 +38,9 @@ int MaxAssignTaskBatchSize = 64; /* maximum number of tasks to assign per round /* TaskMapKey is used as a key in task hash */ typedef struct TaskMapKey { - TaskType taskType; - uint64 jobId; - uint32 taskId; - + TaskType taskType; + uint64 jobId; + uint32 taskId; } TaskMapKey; @@ -51,9 +50,8 @@ typedef struct TaskMapKey */ typedef struct TaskMapEntry { - TaskMapKey key; - Task *task; - + TaskMapKey key; + Task *task; } TaskMapEntry; @@ -83,7 +81,8 @@ static TaskTracker * TrackerHashLookup(HTAB *trackerHash, const char *nodeName, static TaskExecStatus ManageTaskExecution(TaskTracker *taskTracker, TaskTracker *sourceTaskTracker, Task *task, TaskExecution *taskExecution); -static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker, Task *task, +static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker, + Task *task, TaskExecution *taskExecution); static bool TaskExecutionsCompleted(List *taskList); static StringInfo MapFetchTaskQueryString(Task *mapFetchTask, Task *mapTask); @@ -194,8 +193,8 @@ MultiTaskTrackerExecute(Job *job) TrackerHashConnect(transmitTrackerHash); /* loop around until all tasks complete, one task fails, or user cancels */ - while ( !(allTasksCompleted || taskFailed || taskTransmitFailed || - clusterFailed || QueryCancelPending) ) + while (!(allTasksCompleted || taskFailed || taskTransmitFailed || + clusterFailed || QueryCancelPending)) { TaskTracker *taskTracker = NULL; TaskTracker *transmitTracker = NULL; @@ -493,8 +492,8 @@ TaskAndExecutionList(List *jobTaskList) */ if (!dependendTaskInHash) { - dependendTaskInHash = TaskHashEnter(taskHash, dependendTask); - taskQueue = lappend(taskQueue, dependendTaskInHash); + dependendTaskInHash = TaskHashEnter(taskHash, dependendTask); + taskQueue = lappend(taskQueue, dependendTaskInHash); } /* update dependedTaskList element to the one which is in the hash */ @@ -557,7 +556,7 @@ TaskHashEnter(HTAB *taskHash, Task *task) if (handleFound) { ereport(ERROR, (errmsg("multiple entries for task: \"%d:%ld:%d\"", - task->taskType, task->jobId, task->taskId))); + task->taskType, task->jobId, task->taskId))); } /* save the pointer to the original task in the hash */ @@ -820,82 +819,84 @@ TrackerConnectPoll(TaskTracker *taskTracker) { switch (taskTracker->trackerStatus) { - case TRACKER_CONNECT_START: - { - char *nodeName = taskTracker->workerName; - uint32 nodePort = taskTracker->workerPort; - char *nodeDatabase = get_database_name(MyDatabaseId); - - int32 connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); - if (connectionId != INVALID_CONNECTION_ID) + case TRACKER_CONNECT_START: { - taskTracker->connectionId = connectionId; - taskTracker->trackerStatus = TRACKER_CONNECT_POLL; - } - else - { - taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; - } + char *nodeName = taskTracker->workerName; + uint32 nodePort = taskTracker->workerPort; + char *nodeDatabase = get_database_name(MyDatabaseId); - break; - } - - case TRACKER_CONNECT_POLL: - { - int32 connectionId = taskTracker->connectionId; - - ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); - if (pollStatus == CLIENT_CONNECTION_READY) - { - taskTracker->trackerStatus = TRACKER_CONNECTED; - } - else if (pollStatus == CLIENT_CONNECTION_BUSY) - { - taskTracker->trackerStatus = TRACKER_CONNECT_POLL; - } - else if (pollStatus == CLIENT_CONNECTION_BAD) - { - taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; - - MultiClientDisconnect(connectionId); - taskTracker->connectionId = INVALID_CONNECTION_ID; - } - - /* now check if we have been trying to connect for too long */ - taskTracker->connectPollCount++; - if (pollStatus == CLIENT_CONNECTION_BUSY) - { - uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; - uint32 currentCount = taskTracker->connectPollCount; - if (currentCount >= maxCount) + int32 connectionId = MultiClientConnectStart(nodeName, nodePort, + nodeDatabase); + if (connectionId != INVALID_CONNECTION_ID) { - ereport(WARNING, (errmsg("could not establish asynchronous connection " - "after %u ms", REMOTE_NODE_CONNECT_TIMEOUT))); + taskTracker->connectionId = connectionId; + taskTracker->trackerStatus = TRACKER_CONNECT_POLL; + } + else + { + taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; + } + break; + } + + case TRACKER_CONNECT_POLL: + { + int32 connectionId = taskTracker->connectionId; + + ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + if (pollStatus == CLIENT_CONNECTION_READY) + { + taskTracker->trackerStatus = TRACKER_CONNECTED; + } + else if (pollStatus == CLIENT_CONNECTION_BUSY) + { + taskTracker->trackerStatus = TRACKER_CONNECT_POLL; + } + else if (pollStatus == CLIENT_CONNECTION_BAD) + { taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; MultiClientDisconnect(connectionId); taskTracker->connectionId = INVALID_CONNECTION_ID; } + + /* now check if we have been trying to connect for too long */ + taskTracker->connectPollCount++; + if (pollStatus == CLIENT_CONNECTION_BUSY) + { + uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; + uint32 currentCount = taskTracker->connectPollCount; + if (currentCount >= maxCount) + { + ereport(WARNING, (errmsg("could not establish asynchronous " + "connection after %u ms", + REMOTE_NODE_CONNECT_TIMEOUT))); + + taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; + + MultiClientDisconnect(connectionId); + taskTracker->connectionId = INVALID_CONNECTION_ID; + } + } + + break; } - break; - } + case TRACKER_CONNECTED: + case TRACKER_CONNECTION_FAILED: + { + /* if connected or failed to connect in previous pass, reset poll count */ + taskTracker->connectPollCount = 0; + break; + } - case TRACKER_CONNECTED: - case TRACKER_CONNECTION_FAILED: - { - /* if connected or failed to connect in previous pass, reset poll count */ - taskTracker->connectPollCount = 0; - break; - } - - default: - { - int trackerStatus = (int) taskTracker->trackerStatus; - ereport(FATAL, (errmsg("invalid task tracker status: %d", trackerStatus))); - break; - } + default: + { + int trackerStatus = (int) taskTracker->trackerStatus; + ereport(FATAL, (errmsg("invalid task tracker status: %d", trackerStatus))); + break; + } } return taskTracker->trackerStatus; @@ -1008,213 +1009,214 @@ ManageTaskExecution(TaskTracker *taskTracker, TaskTracker *sourceTaskTracker, switch (currentExecutionStatus) { - case EXEC_TASK_UNASSIGNED: - { - bool taskExecutionsCompleted = true; - TaskType taskType = TASK_TYPE_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(taskTracker); - if (!trackerHealthy) + case EXEC_TASK_UNASSIGNED: { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; - break; - } + bool taskExecutionsCompleted = true; + TaskType taskType = TASK_TYPE_INVALID_FIRST; - /* - * We first retrieve this task's downstream dependencies, and then check - * if these dependencies' executions have completed. - */ - taskExecutionsCompleted = TaskExecutionsCompleted(task->dependedTaskList); - if (!taskExecutionsCompleted) - { - nextExecutionStatus = EXEC_TASK_UNASSIGNED; - break; - } - - /* if map fetch task, create query string from completed map task */ - taskType = task->taskType; - if (taskType == MAP_OUTPUT_FETCH_TASK) - { - StringInfo mapFetchTaskQueryString = NULL; - Task *mapTask = (Task *) linitial(task->dependedTaskList); - TaskExecution *mapTaskExecution = mapTask->taskExecution; - - mapFetchTaskQueryString = MapFetchTaskQueryString(task, mapTask); - task->queryString = mapFetchTaskQueryString->data; - taskExecution->querySourceNodeIndex = mapTaskExecution->currentNodeIndex; - } - - /* - * We finally queue this task for execution. Note that we queue sql and - * other tasks slightly differently. - */ - if (taskType == SQL_TASK) - { - TrackerQueueSqlTask(taskTracker, task); - } - else - { - TrackerQueueTask(taskTracker, task); - } - - nextExecutionStatus = EXEC_TASK_QUEUED; - break; - } - - case EXEC_TASK_QUEUED: - { - TaskStatus remoteTaskStatus = TASK_STATUS_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(taskTracker); - if (!trackerHealthy) - { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; - break; - } - - remoteTaskStatus = TrackerTaskStatus(taskTracker, task); - if (remoteTaskStatus == TASK_SUCCEEDED) - { - nextExecutionStatus = EXEC_TASK_DONE; - } - else if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED || - remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) - { - nextExecutionStatus = EXEC_TASK_TRACKER_RETRY; - } - else if (remoteTaskStatus == TASK_PERMANENTLY_FAILED) - { - /* - * If a map output fetch task failed, we assume the problem lies with - * the map task (and the source task tracker it runs on). Otherwise, - * we assume the task tracker crashed, and fail over to the next task - * tracker. - */ - if (task->taskType == MAP_OUTPUT_FETCH_TASK) + bool trackerHealthy = TrackerHealthy(taskTracker); + if (!trackerHealthy) { - nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_RETRY; + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + break; + } + + /* + * We first retrieve this task's downstream dependencies, and then check + * if these dependencies' executions have completed. + */ + taskExecutionsCompleted = TaskExecutionsCompleted(task->dependedTaskList); + if (!taskExecutionsCompleted) + { + nextExecutionStatus = EXEC_TASK_UNASSIGNED; + break; + } + + /* if map fetch task, create query string from completed map task */ + taskType = task->taskType; + if (taskType == MAP_OUTPUT_FETCH_TASK) + { + StringInfo mapFetchTaskQueryString = NULL; + Task *mapTask = (Task *) linitial(task->dependedTaskList); + TaskExecution *mapTaskExecution = mapTask->taskExecution; + + mapFetchTaskQueryString = MapFetchTaskQueryString(task, mapTask); + task->queryString = mapFetchTaskQueryString->data; + taskExecution->querySourceNodeIndex = mapTaskExecution->currentNodeIndex; + } + + /* + * We finally queue this task for execution. Note that we queue sql and + * other tasks slightly differently. + */ + if (taskType == SQL_TASK) + { + TrackerQueueSqlTask(taskTracker, task); + } + else + { + TrackerQueueTask(taskTracker, task); + } + + nextExecutionStatus = EXEC_TASK_QUEUED; + break; + } + + case EXEC_TASK_QUEUED: + { + TaskStatus remoteTaskStatus = TASK_STATUS_INVALID_FIRST; + + bool trackerHealthy = TrackerHealthy(taskTracker); + if (!trackerHealthy) + { + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + break; + } + + remoteTaskStatus = TrackerTaskStatus(taskTracker, task); + if (remoteTaskStatus == TASK_SUCCEEDED) + { + nextExecutionStatus = EXEC_TASK_DONE; + } + else if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED || + remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + { + nextExecutionStatus = EXEC_TASK_TRACKER_RETRY; + } + else if (remoteTaskStatus == TASK_PERMANENTLY_FAILED) + { + /* + * If a map output fetch task failed, we assume the problem lies with + * the map task (and the source task tracker it runs on). Otherwise, + * we assume the task tracker crashed, and fail over to the next task + * tracker. + */ + if (task->taskType == MAP_OUTPUT_FETCH_TASK) + { + nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_RETRY; + } + else + { + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + } + } + else + { + /* assume task is still in progress */ + nextExecutionStatus = EXEC_TASK_QUEUED; + } + + break; + } + + case EXEC_TASK_TRACKER_RETRY: + { + bool trackerHealthy = false; + bool trackerConnectionUp = false; + + /* + * This case statement usually handles connection related issues. Some + * edge cases however, like a user sending a SIGTERM to the worker node, + * keep the connection open but disallow task assignments. We therefore + * need to track those as intermittent tracker failures here. + */ + trackerConnectionUp = TrackerConnectionUp(taskTracker); + if (trackerConnectionUp) + { + taskTracker->trackerFailureCount++; + } + + trackerHealthy = TrackerHealthy(taskTracker); + if (trackerHealthy) + { + TaskStatus remoteTaskStatus = TrackerTaskStatus(taskTracker, task); + if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED) + { + nextExecutionStatus = EXEC_TASK_UNASSIGNED; + } + else if (remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + { + nextExecutionStatus = EXEC_TASK_QUEUED; + } } else { nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; } - } - else - { - /* assume task is still in progress */ - nextExecutionStatus = EXEC_TASK_QUEUED; + + break; } - break; - } - - case EXEC_TASK_TRACKER_RETRY: - { - bool trackerHealthy = false; - bool trackerConnectionUp = false; - - /* - * This case statement usually handles connection related issues. Some - * edge cases however, like a user sending a SIGTERM to the worker node, - * keep the connection open but disallow task assignments. We therefore - * need to track those as intermittent tracker failures here. - */ - trackerConnectionUp = TrackerConnectionUp(taskTracker); - if (trackerConnectionUp) + case EXEC_SOURCE_TASK_TRACKER_RETRY: { - taskTracker->trackerFailureCount++; - } + Task *mapTask = (Task *) linitial(task->dependedTaskList); + TaskExecution *mapTaskExecution = mapTask->taskExecution; + uint32 sourceNodeIndex = mapTaskExecution->currentNodeIndex; - trackerHealthy = TrackerHealthy(taskTracker); - if (trackerHealthy) - { - TaskStatus remoteTaskStatus = TrackerTaskStatus(taskTracker, task); - if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED) + bool sourceTrackerHealthy = false; + Assert(sourceTaskTracker != NULL); + Assert(task->taskType == MAP_OUTPUT_FETCH_TASK); + + /* + * As this map fetch task was running, another map fetch that depends on + * another map task might have failed. We would have then reassigned the + * map task and potentially other map tasks in its constraint group. So + * this map fetch's source node might have changed underneath us. If it + * did, we don't want to record a failure for the new source tracker. + */ + if (taskExecution->querySourceNodeIndex == sourceNodeIndex) { + bool sourceTrackerConnectionUp = TrackerConnectionUp(sourceTaskTracker); + if (sourceTrackerConnectionUp) + { + sourceTaskTracker->trackerFailureCount++; + } + } + + sourceTrackerHealthy = TrackerHealthy(sourceTaskTracker); + if (sourceTrackerHealthy) + { + /* + * We change our status to unassigned. In that status, we queue an + * "update map fetch task" on the task tracker, and retry fetching + * the map task's output from the same source node. + */ nextExecutionStatus = EXEC_TASK_UNASSIGNED; } - else if (remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + else { - nextExecutionStatus = EXEC_TASK_QUEUED; + nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_FAILED; } - } - else - { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + + break; } - break; - } - - case EXEC_SOURCE_TASK_TRACKER_RETRY: - { - Task *mapTask = (Task *) linitial(task->dependedTaskList); - TaskExecution *mapTaskExecution = mapTask->taskExecution; - uint32 sourceNodeIndex = mapTaskExecution->currentNodeIndex; - - bool sourceTrackerHealthy = false; - Assert(sourceTaskTracker != NULL); - Assert(task->taskType == MAP_OUTPUT_FETCH_TASK); - - /* - * As this map fetch task was running, another map fetch that depends on - * another map task might have failed. We would have then reassigned the - * map task and potentially other map tasks in its constraint group. So - * this map fetch's source node might have changed underneath us. If it - * did, we don't want to record a failure for the new source tracker. - */ - if (taskExecution->querySourceNodeIndex == sourceNodeIndex) - { - bool sourceTrackerConnectionUp = TrackerConnectionUp(sourceTaskTracker); - if (sourceTrackerConnectionUp) - { - sourceTaskTracker->trackerFailureCount++; - } - } - - sourceTrackerHealthy = TrackerHealthy(sourceTaskTracker); - if (sourceTrackerHealthy) + case EXEC_TASK_TRACKER_FAILED: + case EXEC_SOURCE_TASK_TRACKER_FAILED: { /* - * We change our status to unassigned. In that status, we queue an - * "update map fetch task" on the task tracker, and retry fetching - * the map task's output from the same source node. + * These two cases exist to signal to the caller that we failed. In both + * cases, the caller is responsible for reassigning task(s) and running + * the appropriate recovery logic. */ nextExecutionStatus = EXEC_TASK_UNASSIGNED; + break; } - else + + case EXEC_TASK_DONE: { - nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_FAILED; + /* we are done with this task's execution */ + nextExecutionStatus = EXEC_TASK_DONE; + break; } - break; - } - - case EXEC_TASK_TRACKER_FAILED: - case EXEC_SOURCE_TASK_TRACKER_FAILED: - { - /* - * These two cases exist to signal to the caller that we failed. In both - * cases, the caller is responsible for reassigning task(s) and running - * the appropriate recovery logic. - */ - nextExecutionStatus = EXEC_TASK_UNASSIGNED; - break; - } - - case EXEC_TASK_DONE: - { - /* we are done with this task's execution */ - nextExecutionStatus = EXEC_TASK_DONE; - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid execution status: %d", currentExecutionStatus))); - break; - } + default: + { + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid execution status: %d", + currentExecutionStatus))); + break; + } } /* update task execution's status for most recent task tracker */ @@ -1247,225 +1249,227 @@ ManageTransmitExecution(TaskTracker *transmitTracker, switch (currentTransmitStatus) { - case EXEC_TRANSMIT_UNASSIGNED: - { - TaskExecStatus *taskStatusArray = taskExecution->taskStatusArray; - TaskExecStatus currentExecutionStatus = taskStatusArray[currentNodeIndex]; - bool trackerHealthy = false; - - /* if top level task's in progress, nothing to do */ - if (currentExecutionStatus != EXEC_TASK_DONE) + case EXEC_TRANSMIT_UNASSIGNED: { - nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; - break; - } + TaskExecStatus *taskStatusArray = taskExecution->taskStatusArray; + TaskExecStatus currentExecutionStatus = taskStatusArray[currentNodeIndex]; + bool trackerHealthy = false; - trackerHealthy = TrackerHealthy(transmitTracker); - if (!trackerHealthy) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; - break; - } + /* if top level task's in progress, nothing to do */ + if (currentExecutionStatus != EXEC_TASK_DONE) + { + nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; + break; + } - TrackerQueueFileTransmit(transmitTracker, task); - nextTransmitStatus = EXEC_TRANSMIT_QUEUED; - break; - } + trackerHealthy = TrackerHealthy(transmitTracker); + if (!trackerHealthy) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + break; + } - case EXEC_TRANSMIT_QUEUED: - { - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - int32 connectionId = INVALID_CONNECTION_ID; - TaskStatus taskStatus = TASK_STATUS_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(transmitTracker); - if (!trackerHealthy) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; - break; - } - - taskStatus = TrackerTaskStatus(transmitTracker, task); - if (taskStatus == TASK_FILE_TRANSMIT_QUEUED) - { - /* remain in queued status until tracker assigns this task */ + TrackerQueueFileTransmit(transmitTracker, task); nextTransmitStatus = EXEC_TRANSMIT_QUEUED; break; } - else if (taskStatus == TASK_CLIENT_SIDE_TRANSMIT_FAILED) + + case EXEC_TRANSMIT_QUEUED: { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; - break; - } + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + int32 connectionId = INVALID_CONNECTION_ID; + TaskStatus taskStatus = TASK_STATUS_INVALID_FIRST; - /* the open connection belongs to this task */ - connectionId = TransmitTrackerConnectionId(transmitTracker, task); - Assert(connectionId != INVALID_CONNECTION_ID); - Assert(taskStatus == TASK_ASSIGNED); - - /* start copy protocol */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_COPY) - { - StringInfo jobDirectoryName = JobDirectoryName(task->jobId); - StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); - - char *filename = taskFilename->data; - int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - int fileMode = (S_IRUSR | S_IWUSR); - - int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); - if (fileDescriptor >= 0) + bool trackerHealthy = TrackerHealthy(transmitTracker); + if (!trackerHealthy) { - /* - * All files inside the job directory get automatically cleaned - * up on transaction commit or abort. - */ - fileDescriptorArray[currentNodeIndex] = fileDescriptor; - nextTransmitStatus = EXEC_TRANSMIT_COPYING; + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + break; + } + + taskStatus = TrackerTaskStatus(transmitTracker, task); + if (taskStatus == TASK_FILE_TRANSMIT_QUEUED) + { + /* remain in queued status until tracker assigns this task */ + nextTransmitStatus = EXEC_TRANSMIT_QUEUED; + break; + } + else if (taskStatus == TASK_CLIENT_SIDE_TRANSMIT_FAILED) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + break; + } + + /* the open connection belongs to this task */ + connectionId = TransmitTrackerConnectionId(transmitTracker, task); + Assert(connectionId != INVALID_CONNECTION_ID); + Assert(taskStatus == TASK_ASSIGNED); + + /* start copy protocol */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_COPY) + { + StringInfo jobDirectoryName = JobDirectoryName(task->jobId); + StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); + + char *filename = taskFilename->data; + int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); + int fileMode = (S_IRUSR | S_IWUSR); + + int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); + if (fileDescriptor >= 0) + { + /* + * All files inside the job directory get automatically cleaned + * up on transaction commit or abort. + */ + fileDescriptorArray[currentNodeIndex] = fileDescriptor; + nextTransmitStatus = EXEC_TRANSMIT_COPYING; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + filename))); + + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + } } else { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", filename))); - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; } - } - else - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + + /* + * We use task tracker logic to manage file transmits as well, but that + * abstraction starts to leak after we drop into the copy protocol. To + * make our task tracker logic work, we need to "void" the tracker's + * connection if the transmit task failed in here. + */ + if (nextTransmitStatus == EXEC_TRANSMIT_TRACKER_RETRY) + { + transmitTracker->connectionBusy = false; + transmitTracker->connectionBusyOnTask = NULL; + } + + break; } - /* - * We use task tracker logic to manage file transmits as well, but that - * abstraction starts to leak after we drop into the copy protocol. To - * make our task tracker logic work, we need to "void" the tracker's - * connection if the transmit task failed in here. - */ - if (nextTransmitStatus == EXEC_TRANSMIT_TRACKER_RETRY) + case EXEC_TRANSMIT_COPYING: { + int32 fileDescriptor = fileDescriptorArray[currentNodeIndex]; + CopyStatus copyStatus = CLIENT_INVALID_COPY; + int closed = -1; + + /* the open connection belongs to this task */ + int32 connectionId = TransmitTrackerConnectionId(transmitTracker, task); + Assert(connectionId != INVALID_CONNECTION_ID); + + copyStatus = MultiClientCopyData(connectionId, fileDescriptor); + if (copyStatus == CLIENT_COPY_MORE) + { + /* worker node continues to send more data, keep reading */ + nextTransmitStatus = EXEC_TRANSMIT_COPYING; + break; + } + + /* we are done copying data */ + if (copyStatus == CLIENT_COPY_DONE) + { + closed = close(fileDescriptor); + fileDescriptorArray[currentNodeIndex] = -1; + + if (closed >= 0) + { + nextTransmitStatus = EXEC_TRANSMIT_DONE; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copied file: %m"))); + + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + } + } + else if (copyStatus == CLIENT_COPY_FAILED) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + + closed = close(fileDescriptor); + fileDescriptorArray[currentNodeIndex] = -1; + + if (closed < 0) + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copy file: %m"))); + } + } + + /* + * We use task tracker logic to manage file transmits as well, but that + * abstraction leaks after we drop into the copy protocol. To make it + * work, we reset transmit tracker's connection for next file transmit. + */ transmitTracker->connectionBusy = false; transmitTracker->connectionBusyOnTask = NULL; - } - break; - } - - case EXEC_TRANSMIT_COPYING: - { - int32 fileDescriptor = fileDescriptorArray[currentNodeIndex]; - CopyStatus copyStatus = CLIENT_INVALID_COPY; - int closed = -1; - - /* the open connection belongs to this task */ - int32 connectionId = TransmitTrackerConnectionId(transmitTracker, task); - Assert(connectionId != INVALID_CONNECTION_ID); - - copyStatus = MultiClientCopyData(connectionId, fileDescriptor); - if (copyStatus == CLIENT_COPY_MORE) - { - /* worker node continues to send more data, keep reading */ - nextTransmitStatus = EXEC_TRANSMIT_COPYING; break; } - /* we are done copying data */ - if (copyStatus == CLIENT_COPY_DONE) + case EXEC_TRANSMIT_TRACKER_RETRY: { - closed = close(fileDescriptor); - fileDescriptorArray[currentNodeIndex] = -1; + bool trackerHealthy = false; + bool trackerConnectionUp = false; - if (closed >= 0) + /* + * The task tracker proxy handles connection errors. On the off chance + * that our connection is still up and the transmit tracker misbehaved, + * we capture this as an intermittent tracker failure. + */ + trackerConnectionUp = TrackerConnectionUp(transmitTracker); + if (trackerConnectionUp) { - nextTransmitStatus = EXEC_TRANSMIT_DONE; + transmitTracker->trackerFailureCount++; + } + + trackerHealthy = TrackerHealthy(transmitTracker); + if (trackerHealthy) + { + nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; } else { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copied file: %m"))); - - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; } - } - else if (copyStatus == CLIENT_COPY_FAILED) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; - closed = close(fileDescriptor); - fileDescriptorArray[currentNodeIndex] = -1; - - if (closed < 0) - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copy file: %m"))); - } + break; } - /* - * We use task tracker logic to manage file transmits as well, but that - * abstraction leaks after we drop into the copy protocol. To make it - * work, we reset transmit tracker's connection for next file transmit. - */ - transmitTracker->connectionBusy = false; - transmitTracker->connectionBusyOnTask = NULL; - - break; - } - - case EXEC_TRANSMIT_TRACKER_RETRY: - { - bool trackerHealthy = false; - bool trackerConnectionUp = false; - - /* - * The task tracker proxy handles connection errors. On the off chance - * that our connection is still up and the transmit tracker misbehaved, - * we capture this as an intermittent tracker failure. - */ - trackerConnectionUp = TrackerConnectionUp(transmitTracker); - if (trackerConnectionUp) - { - transmitTracker->trackerFailureCount++; - } - - trackerHealthy = TrackerHealthy(transmitTracker); - if (trackerHealthy) + case EXEC_TRANSMIT_TRACKER_FAILED: { + /* + * This case exists to signal to the caller that we failed. The caller + * is now responsible for reassigning the transmit task (and downstream + * SQL task dependencies) and running the appropriate recovery logic. + */ nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; + break; } - else + + case EXEC_TRANSMIT_DONE: { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + /* we are done with fetching task results to the master node */ + nextTransmitStatus = EXEC_TRANSMIT_DONE; + break; } - break; - } - - case EXEC_TRANSMIT_TRACKER_FAILED: - { - /* - * This case exists to signal to the caller that we failed. The caller - * is now responsible for reassigning the transmit task (and downstream - * SQL task dependencies) and running the appropriate recovery logic. - */ - nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; - break; - } - - case EXEC_TRANSMIT_DONE: - { - /* we are done with fetching task results to the master node */ - nextTransmitStatus = EXEC_TRANSMIT_DONE; - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid transmit status: %d", currentTransmitStatus))); - break; - } + default: + { + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid transmit status: %d", + currentTransmitStatus))); + break; + } } /* update file transmit status for most recent transmit tracker */ @@ -2317,7 +2321,7 @@ AssignQueuedTasks(TaskTracker *taskTracker) { StringInfo taskAssignmentQuery = taskState->taskAssignmentQuery; - if(taskAssignmentCount > 0) + if (taskAssignmentCount > 0) { appendStringInfo(multiAssignQuery, ";"); } @@ -2336,7 +2340,7 @@ AssignQueuedTasks(TaskTracker *taskTracker) taskState = (TrackerTaskState *) hash_seq_search(&status); } - if(taskAssignmentCount > 0) + if (taskAssignmentCount > 0) { void *queryResult = NULL; int rowCount = 0; @@ -2833,7 +2837,8 @@ TrackerHashCleanupJob(HTAB *taskTrackerHash, Task *jobCleanupTask) if (queryStatus == CLIENT_QUERY_DONE) { ereport(DEBUG4, (errmsg("completed cleanup query for job " UINT64_FORMAT - " on node \"%s:%u\"", jobId, nodeName, nodePort))); + " on node \"%s:%u\"", jobId, nodeName, + nodePort))); /* clear connection for future cleanup queries */ taskTracker->connectionBusy = false; diff --git a/src/backend/distributed/executor/multi_utility.c b/src/backend/distributed/executor/multi_utility.c index ccd05c021..be4777085 100644 --- a/src/backend/distributed/executor/multi_utility.c +++ b/src/backend/distributed/executor/multi_utility.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * multi_utility.c - * CitusDB utility hook and related functionality. + * Citus utility hook and related functionality. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -39,9 +39,9 @@ */ struct DropRelationCallbackState { - char relkind; - Oid heapOid; - bool concurrent; + char relkind; + Oid heapOid; + bool concurrent; }; @@ -76,7 +76,7 @@ static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid ol /* - * Utility for handling citusdb specific concerns around utility statements. + * Utility for handling citus specific concerns around utility statements. * * There's two basic types of concerns here: * 1) Intercept utility statements that run after distributed query @@ -168,32 +168,32 @@ multi_ProcessUtility(Node *parsetree, /* * Inform the user about potential caveats. * - * To prevent failures in aborted transactions, CitusDBHasBeenLoaded() needs + * To prevent failures in aborted transactions, CitusHasBeenLoaded() needs * to be the second condition. See RelationIdGetRelation() which is called - * by CitusDBHasBeenLoaded(). + * by CitusHasBeenLoaded(). */ - if (IsA(parsetree, CreatedbStmt) && CitusDBHasBeenLoaded()) + if (IsA(parsetree, CreatedbStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB partially supports CREATE DATABASE for " + ereport(NOTICE, (errmsg("Citus partially supports CREATE DATABASE for " "distributed databases"), - errdetail("CitusDB does not propagate CREATE DATABASE " + errdetail("Citus does not propagate CREATE DATABASE " "command to workers"), errhint("You can manually create a database and its " "extensions on workers."))); } - else if (IsA(parsetree, CreateSchemaStmt) && CitusDBHasBeenLoaded()) + else if (IsA(parsetree, CreateSchemaStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB partially supports CREATE SCHEMA " + ereport(NOTICE, (errmsg("Citus partially supports CREATE SCHEMA " "for distributed databases"), errdetail("schema usage in joins and in some UDFs " - "provided by CitusDB are not supported yet"))); + "provided by Citus are not supported yet"))); } - else if (IsA(parsetree, CreateRoleStmt) && CitusDBHasBeenLoaded()) + else if (IsA(parsetree, CreateRoleStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB does not support CREATE ROLE/USER " - "for distributed databases"), - errdetail("Multiple roles are currently supported " - "only for local tables"))); + ereport(NOTICE, (errmsg("Citus does not support CREATE ROLE/USER " + "for distributed databases"), + errdetail("Multiple roles are currently supported " + "only for local tables"))); } /* now drop into standard process utility */ @@ -204,7 +204,7 @@ multi_ProcessUtility(Node *parsetree, /* * WarnIfDropCitusExtension prints a WARNING if dropStatement includes dropping - * citusdb extension. + * citus extension. */ static void WarnIfDropCitusExtension(DropStmt *dropStatement) @@ -218,8 +218,8 @@ WarnIfDropCitusExtension(DropStmt *dropStatement) List *objectNameList = lfirst(dropStatementObject); char *objectName = NameListToString(objectNameList); - /* we're only concerned with the citusdb extension */ - if (strncmp("citusdb", objectName, NAMEDATALEN) == 0) + /* we're only concerned with the citus extension */ + if (strncmp("citus", objectName, NAMEDATALEN) == 0) { /* * Warn the user about the possibility of invalid cache. Also, see @@ -296,7 +296,7 @@ VerifyTransmitStmt(CopyStmt *copyStatement) /* - * ProcessCopyStmt handles CitusDB specific concerns for COPY like supporting + * ProcessCopyStmt handles Citus specific concerns for COPY like supporting * COPYing from distributed tables and preventing unsupported actions. */ static Node * @@ -757,7 +757,7 @@ IsAlterTableRenameStmt(RenameStmt *renameStmt) isAlterTableRenameStmt = true; } -#if (PG_VERSION_NUM >=90500) +#if (PG_VERSION_NUM >= 90500) else if (renameStmt->renameType == OBJECT_TABCONSTRAINT) { isAlterTableRenameStmt = true; @@ -905,8 +905,9 @@ ExecuteCommandOnWorkerShards(Oid relationId, const char *commandString, } else { - ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT " on " - "node %s:%d", shardId, workerName, workerPort))); + ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT + " on node %s:%d", shardId, workerName, + workerPort))); } isFirstPlacement = false; @@ -988,6 +989,7 @@ AllFinalizedPlacementsAccessible(Oid relationId) static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg) { + /* *INDENT-OFF* */ HeapTuple tuple; struct DropRelationCallbackState *state; char relkind; @@ -1022,10 +1024,8 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi classform = (Form_pg_class) GETSTRUCT(tuple); if (classform->relkind != relkind) - { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", rel->relname))); - } /* Allow DROP to either table owner or schema owner */ if (!pg_class_ownercheck(relOid, GetUserId()) && @@ -1054,4 +1054,5 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi if (OidIsValid(state->heapOid)) LockRelationOid(state->heapOid, heap_lockmode); } + /* *INDENT-ON* */ } diff --git a/src/backend/distributed/master/master_create_shards.c b/src/backend/distributed/master/master_create_shards.c index 0617d8d59..9adc0e21c 100644 --- a/src/backend/distributed/master/master_create_shards.c +++ b/src/backend/distributed/master/master_create_shards.c @@ -185,7 +185,7 @@ master_create_worker_shards(PG_FUNCTION_ARGS) LockShardDistributionMetadata(shardId, ExclusiveLock); CreateShardPlacements(shardId, ddlCommandList, workerNodeList, - roundRobinNodeIndex, replicationFactor); + roundRobinNodeIndex, replicationFactor); InsertShardRow(distributedTableId, shardId, shardStorageType, minHashTokenText, maxHashTokenText); diff --git a/src/backend/distributed/master/master_delete_protocol.c b/src/backend/distributed/master/master_delete_protocol.c index b046360a6..ced5c630b 100644 --- a/src/backend/distributed/master/master_delete_protocol.c +++ b/src/backend/distributed/master/master_delete_protocol.c @@ -126,9 +126,9 @@ master_apply_delete_command(PG_FUNCTION_ARGS) if ((partitionMethod == DISTRIBUTE_BY_HASH) && (deleteCriteria != NULL)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot delete from distributed table"), - errdetail("Delete statements on hash-partitioned tables " - "with where clause is not supported"))); + errmsg("cannot delete from distributed table"), + errdetail("Delete statements on hash-partitioned tables " + "with where clause is not supported"))); } CheckDeleteCriteria(deleteCriteria); @@ -149,8 +149,8 @@ master_apply_delete_command(PG_FUNCTION_ARGS) else { deletableShardIntervalList = ShardsMatchingDeleteCriteria(relationId, - shardIntervalList, - deleteCriteria); + shardIntervalList, + deleteCriteria); } droppedShardCount = DropShards(relationId, schemaName, relationName, @@ -222,7 +222,7 @@ DropShards(Oid relationId, char *schemaName, char *relationName, { List *shardPlacementList = NIL; List *droppedPlacementList = NIL; - List *lingeringPlacementList= NIL; + List *lingeringPlacementList = NIL; ListCell *shardPlacementCell = NULL; ListCell *droppedPlacementCell = NULL; ListCell *lingeringPlacementCell = NULL; @@ -251,7 +251,8 @@ DropShards(Oid relationId, char *schemaName, char *relationName, shardPlacementList = ShardPlacementList(shardId); foreach(shardPlacementCell, shardPlacementList) { - ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell); + ShardPlacement *shardPlacement = + (ShardPlacement *) lfirst(shardPlacementCell); char *workerName = shardPlacement->nodeName; uint32 workerPort = shardPlacement->nodePort; bool dropSuccessful = false; @@ -260,15 +261,18 @@ DropShards(Oid relationId, char *schemaName, char *relationName, char storageType = shardInterval->storageType; if (storageType == SHARD_STORAGE_TABLE) { - appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, quotedShardName); + appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, + quotedShardName); } else if (storageType == SHARD_STORAGE_COLUMNAR || storageType == SHARD_STORAGE_FOREIGN) { - appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, quotedShardName); + appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, + quotedShardName); } - dropSuccessful = ExecuteRemoteCommand(workerName, workerPort, workerDropQuery); + dropSuccessful = ExecuteRemoteCommand(workerName, workerPort, + workerDropQuery); if (dropSuccessful) { droppedPlacementList = lappend(droppedPlacementList, shardPlacement); @@ -312,12 +316,13 @@ DropShards(Oid relationId, char *schemaName, char *relationName, if (QueryCancelPending) { - ereport(WARNING, (errmsg("cancel requests are ignored during shard deletion"))); + ereport(WARNING, (errmsg("cancel requests are ignored during shard " + "deletion"))); QueryCancelPending = false; } RESUME_INTERRUPTS(); - } + } droppedShardCount = list_length(deletableShardIntervalList); @@ -343,7 +348,7 @@ CheckTableCount(Query *deleteQuery) static void CheckDeleteCriteria(Node *deleteCriteria) { - bool simpleOpExpression = true; + bool simpleOpExpression = true; if (deleteCriteria == NULL) { @@ -372,7 +377,7 @@ CheckDeleteCriteria(Node *deleteCriteria) } else { - simpleOpExpression = false; + simpleOpExpression = false; } if (!simpleOpExpression) @@ -384,15 +389,15 @@ CheckDeleteCriteria(Node *deleteCriteria) } - /* - * CheckPartitionColumn checks that the given where clause is based only on the - * partition key of the given relation id. - */ +/* + * CheckPartitionColumn checks that the given where clause is based only on the + * partition key of the given relation id. + */ static void CheckPartitionColumn(Oid relationId, Node *whereClause) { Var *partitionColumn = PartitionKey(relationId); - ListCell *columnCell = NULL; + ListCell *columnCell = NULL; List *columnList = pull_var_clause_default(whereClause); foreach(columnCell, columnList) @@ -418,7 +423,7 @@ CheckPartitionColumn(Oid relationId, Node *whereClause) */ static List * ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, - Node *deleteCriteria) + Node *deleteCriteria) { List *dropShardIntervalList = NIL; List *deleteCriteriaList = NIL; diff --git a/src/backend/distributed/master/master_metadata_utility.c b/src/backend/distributed/master/master_metadata_utility.c index 18ac2bce0..dd452dcec 100644 --- a/src/backend/distributed/master/master_metadata_utility.c +++ b/src/backend/distributed/master/master_metadata_utility.c @@ -219,7 +219,7 @@ ShardLength(uint64 shardId) if (shardPlacementList == NIL) { ereport(ERROR, (errmsg("could not find length of shard " UINT64_FORMAT, shardId), - errdetail("Could not find any shard placements for the shard."))); + errdetail("Could not find any shard placements for the shard."))); } else { diff --git a/src/backend/distributed/master/master_node_protocol.c b/src/backend/distributed/master/master_node_protocol.c index fc9413666..064756d77 100644 --- a/src/backend/distributed/master/master_node_protocol.c +++ b/src/backend/distributed/master/master_node_protocol.c @@ -49,7 +49,7 @@ /* Shard related configuration */ int ShardReplicationFactor = 2; /* desired replication factor for shards */ -int ShardMaxSize = 1048576; /* maximum size in KB one shard can grow to */ +int ShardMaxSize = 1048576; /* maximum size in KB one shard can grow to */ int ShardPlacementPolicy = SHARD_PLACEMENT_ROUND_ROBIN; @@ -210,7 +210,7 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) tableDDLEventCell = list_head(tableDDLEventList); functionContext->user_fctx = tableDDLEventCell; - + MemoryContextSwitchTo(oldContext); } @@ -226,8 +226,8 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) if (tableDDLEventCell != NULL) { char *ddlStatement = (char *) lfirst(tableDDLEventCell); - text *ddlStatementText = cstring_to_text(ddlStatement); - + text *ddlStatementText = cstring_to_text(ddlStatement); + functionContext->user_fctx = lnext(tableDDLEventCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(ddlStatementText)); @@ -252,7 +252,7 @@ Datum master_get_new_shardid(PG_FUNCTION_ARGS) { text *sequenceName = cstring_to_text(SHARDID_SEQUENCE_NAME); - Oid sequenceId = ResolveRelationId(sequenceName); + Oid sequenceId = ResolveRelationId(sequenceName); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); /* generate new and unique shardId from sequence */ @@ -281,7 +281,7 @@ master_get_local_first_candidate_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; TupleDesc tupleDescriptor = NULL; uint32 liveNodeCount = 0; bool hasOid = false; @@ -396,7 +396,7 @@ master_get_round_robin_candidate_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; TupleDesc tupleDescriptor = NULL; List *workerNodeList = NIL; TypeFuncClass resultTypeClass = 0; @@ -477,7 +477,7 @@ master_get_active_worker_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; List *workerNodeList = NIL; uint32 workerNodeCount = 0; TupleDesc tupleDescriptor = NULL; @@ -567,7 +567,7 @@ GetTableDDLEvents(Oid relationId) Relation pgIndex = NULL; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; @@ -599,13 +599,13 @@ GetTableDDLEvents(Oid relationId) /* fetch table schema and column option definitions */ tableSchemaDef = pg_get_tableschemadef_string(relationId); tableColumnOptionsDef = pg_get_tablecolumnoptionsdef_string(relationId); - + tableDDLEventList = lappend(tableDDLEventList, tableSchemaDef); if (tableColumnOptionsDef != NULL) { tableDDLEventList = lappend(tableDDLEventList, tableColumnOptionsDef); } - + /* open system catalog and scan all indexes that belong to this table */ pgIndex = heap_open(IndexRelationId, AccessShareLock); @@ -660,7 +660,7 @@ GetTableDDLEvents(Oid relationId) { statementDef = pg_get_indexdef_string(indexId); } - + /* append found constraint or index definition to the list */ tableDDLEventList = lappend(tableDDLEventList, statementDef); @@ -695,8 +695,8 @@ hostname_client_addr(void) Port *port = MyProcPort; char *remoteHost = NULL; int remoteHostLen = NI_MAXHOST; - int flags = NI_NAMEREQD; /* require fully qualified hostname */ - int nameFound = 0; + int flags = NI_NAMEREQD; /* require fully qualified hostname */ + int nameFound = 0; if (port == NULL) { @@ -709,10 +709,15 @@ hostname_client_addr(void) #ifdef HAVE_IPV6 case AF_INET6: #endif - break; + { + break; + } + default: + { ereport(ERROR, (errmsg("invalid address family in connection"))); break; + } } remoteHost = palloc0(remoteHostLen); diff --git a/src/backend/distributed/master/master_stage_protocol.c b/src/backend/distributed/master/master_stage_protocol.c index 1c13237cc..921d09e23 100644 --- a/src/backend/distributed/master/master_stage_protocol.c +++ b/src/backend/distributed/master/master_stage_protocol.c @@ -45,7 +45,8 @@ static bool WorkerCreateShard(char *nodeName, uint32 nodePort, static bool WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardName, uint64 *shardLength, text **shardMinValue, text **shardMaxValue); -static uint64 WorkerTableSize(char *nodeName, uint32 nodePort, char *tableName); +static uint64 WorkerTableSize(char *nodeName, uint32 nodePort, Oid relationId, + char *tableName); static StringInfo WorkerPartitionValue(char *nodeName, uint32 nodePort, Oid relationId, char *shardName, char *selectQuery); @@ -77,23 +78,22 @@ master_create_empty_shard(PG_FUNCTION_ARGS) List *candidateNodeList = NIL; text *nullMinValue = NULL; text *nullMaxValue = NULL; - char tableType = 0; char partitionMethod = 0; + char storageType = SHARD_STORAGE_TABLE; Oid relationId = ResolveRelationId(relationNameText); CheckDistributedTable(relationId); - tableType = get_rel_relkind(relationId); - if (tableType != RELKIND_RELATION) + if (CStoreTable(relationId)) { - ereport(ERROR, (errmsg("relation \"%s\" is not a regular table", relationName))); + storageType = SHARD_STORAGE_COLUMNAR; } partitionMethod = PartitionMethod(relationId); if (partitionMethod == DISTRIBUTE_BY_HASH) { ereport(ERROR, (errmsg("relation \"%s\" is a hash partitioned table", - relationName), + relationName), errdetail("We currently don't support creating shards " "on hash-partitioned tables"))); } @@ -128,9 +128,9 @@ master_create_empty_shard(PG_FUNCTION_ARGS) } CreateShardPlacements(shardId, ddlEventList, candidateNodeList, 0, - ShardReplicationFactor); + ShardReplicationFactor); - InsertShardRow(relationId, shardId, SHARD_STORAGE_TABLE, nullMinValue, nullMaxValue); + InsertShardRow(relationId, shardId, storageType, nullMinValue, nullMaxValue); PG_RETURN_INT64(shardId); } @@ -171,9 +171,10 @@ master_append_table_to_shard(PG_FUNCTION_ARGS) ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; + bool cstoreTable = CStoreTable(relationId); char storageType = shardInterval->storageType; - if (storageType != SHARD_STORAGE_TABLE) + if (storageType != SHARD_STORAGE_TABLE && !cstoreTable) { ereport(ERROR, (errmsg("cannot append to shardId " UINT64_FORMAT, shardId), errdetail("The underlying shard is not a regular table"))); @@ -361,7 +362,7 @@ CheckDistributedTable(Oid relationId) */ void CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, - int workerStartIndex, int replicationFactor) + int workerStartIndex, int replicationFactor) { int attemptCount = replicationFactor; int workerNodeCount = list_length(workerNodeList); @@ -393,7 +394,7 @@ CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, else { ereport(WARNING, (errmsg("could not create shard on \"%s:%u\"", - nodeName, nodePort))); + nodeName, nodePort))); } if (placementsCreated >= replicationFactor) @@ -406,7 +407,7 @@ CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, if (placementsCreated < replicationFactor) { ereport(ERROR, (errmsg("could only create %u of %u of required shard replicas", - placementsCreated, replicationFactor))); + placementsCreated, replicationFactor))); } } @@ -457,7 +458,7 @@ WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardNam PG_TRY(); { - uint64 tableSize = WorkerTableSize(nodeName, nodePort, shardName); + uint64 tableSize = WorkerTableSize(nodeName, nodePort, relationId, shardName); StringInfo minValue = WorkerPartitionValue(nodeName, nodePort, relationId, shardName, SHARD_MIN_VALUE_QUERY); StringInfo maxValue = WorkerPartitionValue(nodeName, nodePort, relationId, @@ -479,18 +480,27 @@ WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardNam /* * WorkerTableSize queries the worker node to extract the disk space used by the - * given relation. The function assumes the relation represents a regular table. + * given relation. The function assumes the relation represents a regular table or + * a cstore_fdw table. */ static uint64 -WorkerTableSize(char *nodeName, uint32 nodePort, char *tableName) +WorkerTableSize(char *nodeName, uint32 nodePort, Oid relationId, char *tableName) { uint64 tableSize = 0; List *queryResultList = NIL; StringInfo tableSizeString = NULL; char *tableSizeStringEnd = NULL; - + bool cstoreTable = CStoreTable(relationId); StringInfo tableSizeQuery = makeStringInfo(); - appendStringInfo(tableSizeQuery, SHARD_TABLE_SIZE_QUERY, tableName); + + if (cstoreTable) + { + appendStringInfo(tableSizeQuery, SHARD_CSTORE_TABLE_SIZE_QUERY, tableName); + } + else + { + appendStringInfo(tableSizeQuery, SHARD_TABLE_SIZE_QUERY, tableName); + } queryResultList = ExecuteRemoteQuery(nodeName, nodePort, tableSizeQuery); if (queryResultList == NIL) diff --git a/src/backend/distributed/planner/modify_planner.c b/src/backend/distributed/planner/modify_planner.c index 15a08ea16..4484e53bb 100644 --- a/src/backend/distributed/planner/modify_planner.c +++ b/src/backend/distributed/planner/modify_planner.c @@ -121,16 +121,6 @@ ErrorIfQueryNotSupported(Query *queryTree) Assert(commandType == CMD_INSERT || commandType == CMD_UPDATE || commandType == CMD_DELETE); - if (!(partitionMethod == DISTRIBUTE_BY_HASH || - partitionMethod == DISTRIBUTE_BY_RANGE)) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot perform distributed planning for the given" - " modification"), - errdetail("Only hash- or range-partitioned tables may be the " - "target of distributed modifications"))); - } - /* * Reject subqueries which are in SELECT or WHERE clause. * Queries which include subqueries in FROM clauses are rejected below. @@ -403,6 +393,7 @@ DistributedModifyTask(Query *query) query->onConflict = RebuildOnConflict(relationId, query->onConflict); } #else + /* always set to false for PG_VERSION_NUM < 90500 */ upsertQuery = false; #endif @@ -424,6 +415,7 @@ DistributedModifyTask(Query *query) #if (PG_VERSION_NUM >= 90500) + /* * RebuildOnConflict rebuilds OnConflictExpr for correct deparsing. The function * makes WHERE clause elements explicit and filters dropped columns @@ -443,7 +435,7 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) /* Convert onConflictWhere qualifiers to an explicitly and'd clause */ updatedOnConflict->onConflictWhere = - (Node *) make_ands_explicit((List *) onConflictWhere); + (Node *) make_ands_explicit((List *) onConflictWhere); /* * Here we handle dropped columns on the distributed table. onConflictSet @@ -458,7 +450,7 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) foreach(targetEntryCell, onConflictSet) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); - FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno -1]; + FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno - 1]; /* skip dropped columns */ if (tableAttribute->attisdropped) @@ -478,6 +470,8 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) return updatedOnConflict; } + + #endif diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 35e0ca79b..d6f36a42d 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_explain.c - * CitusDB explain support. + * Citus explain support. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -45,7 +45,7 @@ MultiExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, if (localQuery) { PlannedStmt *plan = NULL; - instr_time planstart; + instr_time planstart; instr_time planduration; INSTR_TIME_SET_CURRENT(planstart); diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index e8cc68f07..9878bf49c 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -33,18 +33,18 @@ /* Config variables managed via guc.c */ -int LargeTableShardCount = 4; /* shard counts for a large table */ +int LargeTableShardCount = 4; /* shard counts for a large table */ bool LogMultiJoinOrder = false; /* print join order as a debugging aid */ /* Function pointer type definition for join rule evaluation functions */ -typedef JoinOrderNode * (*RuleEvalFunction) (JoinOrderNode *currentJoinNode, - TableEntry *candidateTable, - List *candidateShardList, - List *applicableJoinClauses, - JoinType joinType); +typedef JoinOrderNode *(*RuleEvalFunction) (JoinOrderNode *currentJoinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, + JoinType joinType); -static char * RuleNameArray[JOIN_RULE_LAST] = {0}; /* ordered join rule names */ -static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */ +static char *RuleNameArray[JOIN_RULE_LAST] = { 0 }; /* ordered join rule names */ +static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ @@ -54,7 +54,8 @@ static bool JoinExprListWalker(Node *node, List **joinList); static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex); static List * MergeShardIntervals(List *leftShardIntervalList, List *rightShardIntervalList, JoinType joinType); -static bool ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList); +static bool ShardIntervalsMatch(List *leftShardIntervalList, + List *rightShardIntervalList); static List * LoadSortedShardIntervalList(Oid relationId); static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClauseList); @@ -68,31 +69,41 @@ static List * TableEntryListDifference(List *lhsTableList, List *rhsTableList); static TableEntry * FindTableEntry(List *tableEntryList, uint32 tableId); /* Local functions forward declarations for join evaluations */ -static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, - TableEntry *candidateTable, List *candidateShardList, +static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, + JoinOrderNode *currentJoinNode, + TableEntry *candidateTable, + List *candidateShardList, List *joinClauseList, JoinType joinType); static List * RangeTableIdList(List *tableList); static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType); static char * JoinRuleName(JoinRuleType ruleType); static JoinOrderNode * BroadcastJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * LocalJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *candidateShardList, List *applicableJoinClauses, JoinType joinType); static bool JoinOnColumns(Var *currentPartitioncolumn, Var *candidatePartitionColumn, List *joinClauseList); -static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType joinRuleType, - Var *partitionColumn, char partitionMethod); +static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType + joinRuleType, Var *partitionColumn, + char partitionMethod); /* @@ -106,7 +117,7 @@ List * FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) { List *joinList = NIL; - ListCell * joinCell = NULL; + ListCell *joinCell = NULL; List *joinWhereClauseList = NIL; List *joinOrderList = NIL; List *joinedTableList = NIL; @@ -175,7 +186,7 @@ FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) "query"), errdetail("Cannot perform outer joins with broadcast " "joins of more than 1 shard"), - errhint("Set citusdb.large_table_shard_count to 1"))); + errhint("Set citus.large_table_shard_count to 1"))); } } else if (nextJoinNode->joinRuleType == LOCAL_PARTITION_JOIN) @@ -199,7 +210,6 @@ FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) "query"), errdetail("Shards of relations in outer join queries " "must have 1-to-1 shard partitioning"))); - } } else @@ -439,7 +449,7 @@ MergeShardIntervals(List *leftShardIntervalList, List *rightShardIntervalList, bool nextMaxSmaller = comparisonResult > 0; if ((shardUnion && nextMaxLarger) || - (!shardUnion && nextMaxSmaller) ) + (!shardUnion && nextMaxSmaller)) { newShardInterval->maxValue = datumCopy(nextMax, typeByValue, typeLen); } @@ -586,7 +596,8 @@ ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList) nextRightIntervalCell = lnext(rightShardIntervalCell); if (nextRightIntervalCell != NULL) { - ShardInterval *nextRightInterval = (ShardInterval *) lfirst(nextRightIntervalCell); + ShardInterval *nextRightInterval = + (ShardInterval *) lfirst(nextRightIntervalCell); shardIntervalsIntersect = ShardIntervalsOverlap(leftInterval, nextRightInterval); if (shardIntervalsIntersect) @@ -730,7 +741,7 @@ JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClause * BestJoinOrder takes in a list of candidate join orders, and determines the * best join order among these candidates. The function uses two heuristics for * this. First, the function chooses join orders that have the fewest number of - * join operators that cause large data transfers. Second, the function chooses + * join operators that cause large data transfers. Second, the function chooses * join orders where large data transfers occur later in the execution. */ static List * @@ -1009,7 +1020,7 @@ EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, uint32 candidateTableId = 0; List *joinedTableIdList = NIL; List *applicableJoinClauses = NIL; - uint32 lowestValidIndex = JOIN_RULE_INVALID_FIRST + 1; + uint32 lowestValidIndex = JOIN_RULE_INVALID_FIRST + 1; uint32 highestValidIndex = JOIN_RULE_LAST - 1; uint32 ruleIndex = 0; @@ -1028,11 +1039,11 @@ EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, JoinRuleType ruleType = (JoinRuleType) ruleIndex; RuleEvalFunction ruleEvalFunction = JoinRuleEvalFunction(ruleType); - nextJoinNode = (*ruleEvalFunction) (currentJoinNode, - candidateTable, - candidateShardList, - applicableJoinClauses, - joinType); + nextJoinNode = (*ruleEvalFunction)(currentJoinNode, + candidateTable, + candidateShardList, + applicableJoinClauses, + joinType); /* break after finding the first join rule that applies */ if (nextJoinNode != NULL) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 3aebfcb37..787937079 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -91,7 +91,8 @@ static void ParentSetNewChild(MultiNode *parentNode, MultiNode *oldChildNode, /* Local functions forward declarations for aggregate expressions */ static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode, - MultiExtendedOp *masterNode, MultiExtendedOp *workerNode); + MultiExtendedOp *masterNode, + MultiExtendedOp *workerNode); static void TransformSubqueryNode(MultiTable *subqueryNode); static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode); static Node * MasterAggregateMutator(Node *originalNode, AttrNumber *columnId); @@ -117,7 +118,8 @@ static void ErrorIfUnsupportedArrayAggregate(Aggref *arrayAggregateExpression); static void ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, MultiNode *logicalPlanNode); static Var * AggregateDistinctColumn(Aggref *aggregateExpression); -static bool TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode, +static bool TablePartitioningSupportsDistinct(List *tableNodeList, + MultiExtendedOp *opNode, Var *distinctColumn); static bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column); @@ -257,6 +259,7 @@ MultiLogicalPlanOptimize(MultiTreeRoot *multiLogicalPlan) MultiTable *tableNode = (MultiTable *) lfirst(tableNodeCell); if (tableNode->relationId == SUBQUERY_RELATION_ID) { + ErrorIfContainsUnsupportedAggregate((MultiNode *) tableNode); TransformSubqueryNode(tableNode); } } @@ -488,7 +491,7 @@ AddressProjectSpecialConditions(MultiProject *projectNode) /* * We check if we need to include any child columns in the project node to - * address the following special conditions. + * address the following special conditions. * * SNC1: project node must include child node's projected columns, or * SNC2: project node must include child node's partition column, or @@ -637,7 +640,7 @@ Commutative(MultiUnaryNode *parentNode, MultiUnaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* we cannot be commutative with non-query operators */ if (childNodeTag == T_MultiTreeRoot || childNodeTag == T_MultiTable) @@ -692,7 +695,7 @@ Distributive(MultiUnaryNode *parentNode, MultiBinaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* special condition checks for partition operator are not implemented */ Assert(parentNodeTag != T_MultiPartition); @@ -751,7 +754,7 @@ Factorizable(MultiBinaryNode *parentNode, MultiUnaryNode *childNode) { PullUpStatus pullUpStatus = PULL_UP_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* * The following nodes are factorizable with their parents, but we don't @@ -1220,7 +1223,7 @@ MasterExtendedOpNode(MultiExtendedOp *originalOpNode) bool hasAggregates = contain_agg_clause((Node *) originalExpression); if (hasAggregates) { - Node *newNode = MasterAggregateMutator((Node*) originalExpression, + Node *newNode = MasterAggregateMutator((Node *) originalExpression, &columnId); newExpression = (Expr *) newNode; } @@ -1826,7 +1829,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate) static AggregateType GetAggregateType(Oid aggFunctionId) { - char *aggregateProcName = NULL; + char *aggregateProcName = NULL; uint32 aggregateCount = 0; uint32 aggregateIndex = 0; bool found = false; @@ -1980,22 +1983,30 @@ CountDistinctHashFunctionName(Oid argumentType) switch (argumentType) { case INT4OID: + { hashFunctionName = pstrdup(HLL_HASH_INTEGER_FUNC_NAME); break; + } case INT8OID: + { hashFunctionName = pstrdup(HLL_HASH_BIGINT_FUNC_NAME); break; + } case TEXTOID: case BPCHAROID: case VARCHAROID: + { hashFunctionName = pstrdup(HLL_HASH_TEXT_FUNC_NAME); break; + } default: + { hashFunctionName = pstrdup(HLL_HASH_ANY_FUNC_NAME); break; + } } return hashFunctionName; @@ -2135,8 +2146,9 @@ ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, bool distinctSupported = true; List *repartitionNodeList = NIL; Var *distinctColumn = NULL; - - AggregateType aggregateType = GetAggregateType(aggregateExpression->aggfnoid); + List *multiTableNodeList = NIL; + ListCell *multiTableNodeCell = NULL; + AggregateType aggregateType = AGGREGATE_INVALID_FIRST; /* check if logical plan includes a subquery */ List *subqueryMultiTableList = SubqueryMultiTableList(logicalPlanNode); @@ -2147,7 +2159,20 @@ ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, errdetail("distinct in the outermost query is unsupported"))); } + multiTableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); + foreach(multiTableNodeCell, multiTableNodeList) + { + MultiTable *multiTable = (MultiTable *) lfirst(multiTableNodeCell); + if (multiTable->relationId == SUBQUERY_RELATION_ID) + { + ereport(ERROR, (errmsg("cannot compute count (distinct)"), + errdetail("Subqueries with aggregate (distinct) are " + "not supported yet"))); + } + } + /* if we have a count(distinct), and distinct approximation is enabled */ + aggregateType = GetAggregateType(aggregateExpression->aggfnoid); if (aggregateType == AGGREGATE_COUNT && CountDistinctErrorRate != DISABLE_DISTINCT_APPROXIMATION) { @@ -2479,7 +2504,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) if (subqueryTree->setOperations) { SetOperationStmt *setOperationStatement = - (SetOperationStmt *) subqueryTree->setOperations; + (SetOperationStmt *) subqueryTree->setOperations; if (setOperationStatement->op == SETOP_UNION) { @@ -2563,7 +2588,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) List *joinTreeTableIndexList = NIL; uint32 joiningTableCount = 0; - ExtractRangeTableIndexWalker((Node*) subqueryTree->jointree, + ExtractRangeTableIndexWalker((Node *) subqueryTree->jointree, &joinTreeTableIndexList); joiningTableCount = list_length(joinTreeTableIndexList); @@ -2587,7 +2612,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) List *distinctTargetEntryList = GroupTargetEntryList(distinctClauseList, targetEntryList); bool distinctOnPartitionColumn = - TargetListOnPartitionColumn(subqueryTree, distinctTargetEntryList); + TargetListOnPartitionColumn(subqueryTree, distinctTargetEntryList); if (!distinctOnPartitionColumn) { preconditionsSatisfied = false; @@ -2609,7 +2634,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) foreach(rangeTableEntryCell, subqueryEntryList) { RangeTblEntry *rangeTableEntry = - (RangeTblEntry *) lfirst(rangeTableEntryCell); + (RangeTblEntry *) lfirst(rangeTableEntryCell); Query *innerSubquery = rangeTableEntry->subquery; ErrorIfCannotPushdownSubquery(innerSubquery, outerQueryHasLimit); @@ -2639,7 +2664,7 @@ ErrorIfUnsupportedTableCombination(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -2768,7 +2793,7 @@ ErrorIfUnsupportedUnionQuery(Query *unionQuery) leftQueryOnPartitionColumn = TargetListOnPartitionColumn(leftQuery, leftQuery->targetList); rightQueryOnPartitionColumn = TargetListOnPartitionColumn(rightQuery, - rightQuery->targetList); + rightQuery->targetList); if (!(leftQueryOnPartitionColumn && rightQueryOnPartitionColumn)) { @@ -2807,7 +2832,7 @@ GroupTargetEntryList(List *groupClauseList, List *targetEntryList) { SortGroupClause *groupClause = (SortGroupClause *) lfirst(groupClauseCell); TargetEntry *groupTargetEntry = - get_sortgroupclause_tle(groupClause, targetEntryList); + get_sortgroupclause_tle(groupClause, targetEntryList); groupTargetEntryList = lappend(groupTargetEntryList, groupTargetEntry); } @@ -2890,7 +2915,7 @@ IsPartitionColumnRecursive(Expr *columnExpression, Query *query) else if (IsA(columnExpression, FieldSelect)) { FieldSelect *compositeField = (FieldSelect *) columnExpression; - Expr *fieldExpression = compositeField->arg; + Expr *fieldExpression = compositeField->arg; if (IsA(fieldExpression, Var)) { @@ -2909,7 +2934,7 @@ IsPartitionColumnRecursive(Expr *columnExpression, Query *query) return false; } - rangeTableEntryIndex = candidateColumn->varno - 1; + rangeTableEntryIndex = candidateColumn->varno - 1; rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_RELATION) @@ -2980,7 +3005,7 @@ CompositeFieldRecursive(Expr *expression, Query *query) return NULL; } - rangeTableEntryIndex = candidateColumn->varno - 1; + rangeTableEntryIndex = candidateColumn->varno - 1; rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_SUBQUERY) @@ -3019,7 +3044,7 @@ FullCompositeFieldList(List *compositeFieldList) uint32 fieldIndex = 0; ListCell *fieldSelectCell = NULL; - foreach (fieldSelectCell, compositeFieldList) + foreach(fieldSelectCell, compositeFieldList) { FieldSelect *fieldSelect = (FieldSelect *) lfirst(fieldSelectCell); uint32 compositeFieldIndex = 0; @@ -3226,9 +3251,10 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery) if (outerColumnIsPartitionColumn && localColumnIsPartitionColumn) { FieldSelect *outerCompositeField = - CompositeFieldRecursive(outerQueryExpression, parentQuery); + CompositeFieldRecursive(outerQueryExpression, parentQuery); FieldSelect *localCompositeField = - CompositeFieldRecursive(localQueryExpression, lateralQuery); + CompositeFieldRecursive(localQueryExpression, lateralQuery); + /* * If partition colums are composite fields, add them to list to * check later if all composite fields are used. @@ -3251,12 +3277,12 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery) } /* check composite fields */ - if(!supportedLateralQuery) + if (!supportedLateralQuery) { bool outerFullCompositeFieldList = - FullCompositeFieldList(outerCompositeFieldList); + FullCompositeFieldList(outerCompositeFieldList); bool localFullCompositeFieldList = - FullCompositeFieldList(localCompositeFieldList); + FullCompositeFieldList(localCompositeFieldList); if (outerFullCompositeFieldList && localFullCompositeFieldList) { @@ -3301,15 +3327,15 @@ JoinOnPartitionColumn(Query *query) if (isLeftColumnPartitionColumn && isRightColumnPartitionColumn) { FieldSelect *leftCompositeField = - CompositeFieldRecursive(leftArgument, query); + CompositeFieldRecursive(leftArgument, query); FieldSelect *rightCompositeField = - CompositeFieldRecursive(rightArgument, query); + CompositeFieldRecursive(rightArgument, query); /* * If partition colums are composite fields, add them to list to * check later if all composite fields are used. */ - if(leftCompositeField && rightCompositeField) + if (leftCompositeField && rightCompositeField) { leftCompositeFieldList = lappend(leftCompositeFieldList, leftCompositeField); @@ -3318,7 +3344,7 @@ JoinOnPartitionColumn(Query *query) } /* if both sides are not composite fields, they are normal columns */ - if(!(leftCompositeField && rightCompositeField)) + if (!(leftCompositeField && rightCompositeField)) { joinOnPartitionColumn = true; break; @@ -3327,12 +3353,12 @@ JoinOnPartitionColumn(Query *query) } /* check composite fields */ - if(!joinOnPartitionColumn) + if (!joinOnPartitionColumn) { bool leftFullCompositeFieldList = - FullCompositeFieldList(leftCompositeFieldList); + FullCompositeFieldList(leftCompositeFieldList); bool rightFullCompositeFieldList = - FullCompositeFieldList(rightCompositeFieldList); + FullCompositeFieldList(rightCompositeFieldList); if (leftFullCompositeFieldList && rightFullCompositeFieldList) { @@ -3409,7 +3435,7 @@ ErrorIfUnsupportedShardDistribution(Query *query) /* check if this table has 1-1 shard partitioning with first table */ coPartitionedTables = CoPartitionedTables(firstShardIntervalList, - currentShardIntervalList); + currentShardIntervalList); if (!coPartitionedTables) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -3437,7 +3463,7 @@ RelationIdList(Query *query) foreach(tableEntryCell, tableEntryList) { - TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); + TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); Oid relationId = tableEntry->relationId; relationIdList = list_append_unique_oid(relationIdList, relationId); @@ -3617,7 +3643,7 @@ ExtractQueryWalker(Node *node, List **queryList) Query *query = (Query *) node; (*queryList) = lappend(*queryList, query); - walkerResult = query_tree_walker(query, ExtractQueryWalker, queryList, + walkerResult = query_tree_walker(query, ExtractQueryWalker, queryList, QTW_EXAMINE_RTES); } @@ -3641,7 +3667,7 @@ LeafQuery(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -3725,7 +3751,7 @@ PartitionColumnOpExpressionList(Query *query) } else if (IsA(leftArgument, Const) && IsA(leftArgument, Var)) { - candidatePartitionColumn = (Var *) rightArgument; + candidatePartitionColumn = (Var *) rightArgument; } else { diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index 8f866c074..a9edcfc39 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_logical_planner.c - * + * * Routines for constructing a logical plan tree from the given Query tree * structure. This new logical plan is based on multi-relational algebra rules. * @@ -39,11 +39,11 @@ bool SubqueryPushdown = false; /* is subquery pushdown enabled */ /* Function pointer type definition for apply join rule functions */ -typedef MultiNode * (*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode, - Var *partitionColumn, JoinType joinType, - List *joinClauses); +typedef MultiNode *(*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode, + Var *partitionColumn, JoinType joinType, + List *joinClauses); -static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */ +static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ static MultiNode * MultiPlanTree(Query *queryTree); @@ -157,7 +157,7 @@ SubqueryEntryList(Query *queryTree) * only walk over range table entries at this level and do not recurse into * subqueries. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -285,6 +285,7 @@ MultiPlanTree(Query *queryTree) else { bool hasOuterJoin = false; + /* * We calculate the join order using the list of tables in the query and * the join clauses between them. Note that this function owns the table @@ -465,6 +466,7 @@ ErrorIfQueryNotSupported(Query *queryTree) #if (PG_VERSION_NUM >= 90500) + /* HasTablesample returns tree if the query contains tablesample */ static bool HasTablesample(Query *queryTree) @@ -485,6 +487,8 @@ HasTablesample(Query *queryTree) return hasTablesample; } + + #endif @@ -529,7 +533,8 @@ HasUnsupportedJoinWalker(Node *node, void *context) * ErrorIfSubqueryNotSupported checks that we can perform distributed planning for * the given subquery. */ -static void ErrorIfSubqueryNotSupported(Query *subqueryTree) +static void +ErrorIfSubqueryNotSupported(Query *subqueryTree) { char *errorDetail = NULL; bool preconditionsSatisfied = true; @@ -587,7 +592,6 @@ HasOuterJoin(Query *queryTree) static bool HasOuterJoinWalker(Node *node, void *context) { - bool hasOuterJoin = false; if (node == NULL) { @@ -657,7 +661,7 @@ HasComplexRangeTableType(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -675,7 +679,7 @@ HasComplexRangeTableType(Query *queryTree) * subquery. */ if (rangeTableEntry->rtekind != RTE_RELATION && - rangeTableEntry->rtekind != RTE_SUBQUERY) + rangeTableEntry->rtekind != RTE_SUBQUERY) { hasComplexRangeTableType = true; } @@ -966,7 +970,7 @@ TableEntryList(List *rangeTableList) foreach(rangeTableCell, rangeTableList) { - RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); + RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->rtekind == RTE_RELATION) { @@ -1178,8 +1182,8 @@ IsSelectClause(Node *clause) /* we currently consider the following nodes as select clauses */ NodeTag nodeTag = nodeTag(clause); - if ( !(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr || - nodeTag == T_NullTest || nodeTag == T_BooleanTest) ) + if (!(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr || + nodeTag == T_NullTest || nodeTag == T_BooleanTest)) { return false; } @@ -1317,9 +1321,9 @@ UnaryOperator(MultiNode *node) { bool unaryOperator = false; - if (CitusIsA(node, MultiTreeRoot) || CitusIsA(node, MultiTable) || - CitusIsA(node, MultiCollect) || CitusIsA(node, MultiSelect) || - CitusIsA(node, MultiProject) || CitusIsA(node, MultiPartition) || + if (CitusIsA(node, MultiTreeRoot) || CitusIsA(node, MultiTable) || + CitusIsA(node, MultiCollect) || CitusIsA(node, MultiSelect) || + CitusIsA(node, MultiProject) || CitusIsA(node, MultiPartition) || CitusIsA(node, MultiExtendedOp)) { unaryOperator = true; @@ -1403,7 +1407,7 @@ FindNodesOfType(MultiNode *node, int type) } else if (BinaryOperator(node)) { - MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; + MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; MultiNode *rightChildNode = ((MultiBinaryNode *) node)->rightChildNode; List *leftChildNodeList = FindNodesOfType(leftChildNode, type); @@ -1533,9 +1537,9 @@ ExtractRangeTableEntryWalker(Node *node, List **rangeTableList) List * pull_var_clause_default(Node *node) { - List *columnList = pull_var_clause(node, PVC_RECURSE_AGGREGATES, - PVC_REJECT_PLACEHOLDERS); - return columnList; + List *columnList = pull_var_clause(node, PVC_RECURSE_AGGREGATES, + PVC_REJECT_PLACEHOLDERS); + return columnList; } @@ -1552,7 +1556,7 @@ ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, MultiNode *multiNode = NULL; List *applicableJoinClauses = NIL; - List *leftTableIdList = OutputTableIdList(leftNode); + List *leftTableIdList = OutputTableIdList(leftNode); List *rightTableIdList = OutputTableIdList(rightNode); int rightTableIdCount = 0; uint32 rightTableId = 0; @@ -1567,8 +1571,8 @@ ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, /* call the join rule application function to create the new join node */ ruleApplyFunction = JoinRuleApplyFunction(ruleType); - multiNode = (*ruleApplyFunction) (leftNode, rightNode, partitionColumn, - joinType, applicableJoinClauses); + multiNode = (*ruleApplyFunction)(leftNode, rightNode, partitionColumn, + joinType, applicableJoinClauses); if (joinType != JOIN_INNER && CitusIsA(multiNode, MultiJoin)) { @@ -1918,7 +1922,7 @@ ErrorIfSubqueryJoin(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); joiningRangeTableCount = list_length(joinTreeTableIndexList); if (joiningRangeTableCount > 1) diff --git a/src/backend/distributed/planner/multi_master_planner.c b/src/backend/distributed/planner/multi_master_planner.c index 88fda53a3..b328e2e7c 100644 --- a/src/backend/distributed/planner/multi_master_planner.c +++ b/src/backend/distributed/planner/multi_master_planner.c @@ -122,7 +122,7 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) AggStrategy aggregateStrategy = AGG_PLAIN; AggClauseCosts aggregateCosts; AttrNumber *groupColumnIdArray = NULL; - List *aggregateTargetList = NIL; + List *aggregateTargetList = NIL; List *groupColumnList = NIL; List *columnList = NIL; ListCell *columnCell = NULL; @@ -168,13 +168,13 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) /* finally create the plan */ #if (PG_VERSION_NUM >= 90500) - aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, - &aggregateCosts, groupColumnCount, groupColumnIdArray, - groupColumnOpArray, NIL, rowEstimate, subPlan); + aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, + &aggregateCosts, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, NIL, rowEstimate, subPlan); #else - aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, - &aggregateCosts, groupColumnCount, groupColumnIdArray, - groupColumnOpArray, rowEstimate, subPlan); + aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, + &aggregateCosts, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, rowEstimate, subPlan); #endif return aggregatePlan; @@ -211,7 +211,7 @@ BuildSelectStatement(Query *masterQuery, char *masterTableName, rangeTableEntry = copyObject(queryRangeTableEntry); rangeTableEntry->rtekind = RTE_RELATION; rangeTableEntry->eref = makeAlias(masterTableName, NIL); - rangeTableEntry->relid = 0; /* to be filled in exec_Start */ + rangeTableEntry->relid = 0; /* to be filled in exec_Start */ rangeTableEntry->inh = false; rangeTableEntry->inFromCl = true; @@ -220,7 +220,7 @@ BuildSelectStatement(Query *masterQuery, char *masterTableName, /* (2) build and initialize sequential scan node */ sequentialScan = makeNode(SeqScan); - sequentialScan->scanrelid = 1; /* always one */ + sequentialScan->scanrelid = 1; /* always one */ /* (3) add an aggregation plan if needed */ if (masterQuery->hasAggs || masterQuery->groupClause) diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 1b4172d3a..b34c36b4c 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -138,7 +138,7 @@ static OpExpr * MakeOpExpressionWithZeroConst(void); static List * BuildRestrictInfoList(List *qualList); static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery, List *dependedJobList); -static JoinSequenceNode * JoinSequenceArray(List * rangeTableFragmentsList, +static JoinSequenceNode * JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *dependedJobList); static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependedJobList); static void CheckJoinBetweenColumns(OpExpr *joinClause); @@ -155,7 +155,8 @@ static StringInfo DatumArrayString(Datum *datumArray, uint32 datumCount, Oid dat static Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType, char *queryString); static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList); -static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment); +static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, + RangeTableFragment *fragment); static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId); static List * PruneSqlTaskDependencies(List *sqlTaskList); static List * AssignTaskList(List *sqlTaskList); @@ -167,7 +168,7 @@ static Task * GreedyAssignTask(WorkerNode *workerNode, List *taskList, static List * RoundRobinAssignTaskList(List *taskList); static List * RoundRobinReorder(Task *task, List *placementList); static List * ReorderAndAssignTaskList(List *taskList, - List * (*reorderFunction) (Task *, List *)); + List * (*reorderFunction)(Task *, List *)); static int CompareTasksByShardId(const void *leftElement, const void *rightElement); static List * ActiveShardPlacementLists(List *taskList); static List * ActivePlacementList(List *placementList); @@ -309,6 +310,7 @@ BuildJobTree(MultiTreeRoot *multiTree) partitionKey, partitionType, baseRelationId, JOIN_MAP_MERGE_JOB); + /* reset depended job list */ loopDependedJobList = NIL; loopDependedJobList = list_make1(mapMergeJob); @@ -538,7 +540,7 @@ BuildJobQuery(MultiNode *multiNode, List *dependedJobList) * If we are building this query on a repartitioned subquery job then we * don't need to update column attributes. */ - if(dependedJobList != NIL) + if (dependedJobList != NIL) { Job *job = (Job *) linitial(dependedJobList); if (CitusIsA(job, MapMergeJob)) @@ -628,10 +630,10 @@ BuildJobQuery(MultiNode *multiNode, List *dependedJobList) jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; - jobQuery->sortClause = sortClauseList; + jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; - jobQuery->limitCount = limitCount; + jobQuery->limitCount = limitCount; jobQuery->hasAggs = contain_agg_clause((Node *) targetList); return jobQuery; @@ -718,10 +720,10 @@ BuildReduceQuery(MultiExtendedOp *extendedOpNode, List *dependedJobList) reduceQuery->rtable = derivedRangeTableList; reduceQuery->targetList = targetList; reduceQuery->jointree = joinTree; - reduceQuery->sortClause = extendedOpNode->sortClauseList; + reduceQuery->sortClause = extendedOpNode->sortClauseList; reduceQuery->groupClause = extendedOpNode->groupClauseList; reduceQuery->limitOffset = extendedOpNode->limitOffset; - reduceQuery->limitCount = extendedOpNode->limitCount; + reduceQuery->limitCount = extendedOpNode->limitCount; reduceQuery->hasAggs = contain_agg_clause((Node *) targetList); return reduceQuery; @@ -754,7 +756,7 @@ BaseRangeTableList(MultiNode *multiNode) */ MultiTable *multiTable = (MultiTable *) multiNode; if (multiTable->relationId != SUBQUERY_RELATION_ID && - multiTable->relationId != HEAP_ANALYTICS_SUBQUERY_RELATION_ID) + multiTable->relationId != HEAP_ANALYTICS_SUBQUERY_RELATION_ID) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->inFromCl = true; @@ -870,7 +872,7 @@ TargetEntryList(List *expressionList) Expr *expression = (Expr *) lfirst(expressionCell); TargetEntry *targetEntry = makeTargetEntry(expression, - list_length(targetEntryList)+1, + list_length(targetEntryList) + 1, NULL, false); targetEntryList = lappend(targetEntryList, targetEntry); } @@ -1044,7 +1046,7 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList /* fix the column attributes in ON (...) clauses */ columnList = pull_var_clause_default((Node *) joinNode->joinClauseList); - foreach (columnCell, columnList) + foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); UpdateColumnAttributes(column, *rangeTableList, dependedJobList); @@ -1093,7 +1095,8 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList uint32 columnCount = (uint32) list_length(dependedTargetList); List *columnNameList = DerivedColumnNameList(columnCount, dependedJob->jobId); - RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, columnNameList, + RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, + columnNameList, tableIdList); RangeTblRef *rangeTableRef = makeNode(RangeTblRef); @@ -1246,7 +1249,7 @@ ExtractColumns(RangeTblEntry *rangeTableEntry, int rangeTableId, List *dependedJ else if (rangeTableKind == CITUS_RTE_RELATION) { /* - * For distributed tables, we construct a regular table RTE to call + * For distributed tables, we construct a regular table RTE to call * expandRTE, which will extract columns from the distributed table * schema. */ @@ -1405,10 +1408,10 @@ BuildSubqueryJobQuery(MultiNode *multiNode) jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; - jobQuery->sortClause = sortClauseList; + jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; - jobQuery->limitCount = limitCount; + jobQuery->limitCount = limitCount; jobQuery->hasAggs = contain_agg_clause((Node *) targetList); return jobQuery; @@ -1646,7 +1649,7 @@ static uint64 UniqueJobId(void) { text *sequenceName = cstring_to_text(JOBID_SEQUENCE_NAME); - Oid sequenceId = ResolveRelationId(sequenceName); + Oid sequenceId = ResolveRelationId(sequenceName); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); /* generate new and unique jobId from sequence */ @@ -1747,7 +1750,7 @@ HashPartitionCount(void) uint32 nodeCount = WorkerGetLiveNodeCount(); double maxReduceTasksPerNode = MaxRunningTasksPerNode / 2.0; - uint32 partitionCount = (uint32) rint(nodeCount * maxReduceTasksPerNode); + uint32 partitionCount = (uint32) rint(nodeCount * maxReduceTasksPerNode); return partitionCount; } @@ -1864,8 +1867,9 @@ SplitPointObject(ShardInterval **shardIntervalArray, uint32 shardIntervalCount) return splitPointObject; } + /* ------------------------------------------------------------ - * Functions that relate to building and assigning tasks follow + * Functions that relate to building and assigning tasks follow * ------------------------------------------------------------ */ @@ -1986,7 +1990,7 @@ SubquerySqlTaskList(Job *job) ListCell *rangeTableCell = NULL; ListCell *queryCell = NULL; Node *whereClauseTree = NULL; - uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ + uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint32 anchorRangeTableId = 0; uint32 rangeTableIndex = 0; const uint32 fragmentSize = sizeof(RangeTableFragment); @@ -2036,10 +2040,10 @@ SubquerySqlTaskList(Job *job) if (opExpressionList != NIL) { Var *partitionColumn = PartitionColumn(relationId, tableId); - List *whereClauseList = ReplaceColumnsInOpExpressionList(opExpressionList, + List *whereClauseList = ReplaceColumnsInOpExpressionList(opExpressionList, partitionColumn); finalShardIntervalList = PruneShardList(relationId, tableId, whereClauseList, - shardIntervalList); + shardIntervalList); } else { @@ -2146,7 +2150,7 @@ static List * SqlTaskList(Job *job) { List *sqlTaskList = NIL; - uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ + uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint64 jobId = job->jobId; bool anchorRangeTableBasedAssignment = false; uint32 anchorRangeTableId = 0; @@ -2472,8 +2476,8 @@ RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, List *shardIntervalList = LoadShardIntervalList(relationId); List *prunedShardIntervalList = PruneShardList(relationId, tableId, - whereClauseList, - shardIntervalList); + whereClauseList, + shardIntervalList); /* * If we prune all shards for one table, query results will be empty. @@ -2548,7 +2552,7 @@ RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, */ List * PruneShardList(Oid relationId, Index tableId, List *whereClauseList, - List *shardIntervalList) + List *shardIntervalList) { List *remainingShardList = NIL; ListCell *shardIntervalCell = NULL; @@ -2653,7 +2657,7 @@ MakeOpExpression(Var *variable, int16 strategyNumber) Oid accessMethodId = BTREE_AM_OID; Oid operatorId = InvalidOid; Oid operatorClassInputType = InvalidOid; - Const *constantValue = NULL; + Const *constantValue = NULL; OpExpr *expression = NULL; char typeType = 0; @@ -2679,7 +2683,7 @@ MakeOpExpression(Var *variable, int16 strategyNumber) /* Now make the expression with the given variable and a null constant */ expression = (OpExpr *) make_opclause(operatorId, InvalidOid, /* no result type yet */ - false, /* no return set */ + false, /* no return set */ (Expr *) variable, (Expr *) constantValue, InvalidOid, collationId); @@ -2808,6 +2812,10 @@ SimpleOpExpression(Expr *clause) return false; /* not a binary opclause */ } + /* strip coercions before doing check */ + leftOperand = strip_implicit_coercions(leftOperand); + rightOperand = strip_implicit_coercions(rightOperand); + if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) { constantClause = (Const *) rightOperand; @@ -2896,7 +2904,7 @@ HashableClauseMutator(Node *originalNode, Var *partitionColumn) * If this node is not hashable, continue walking down the expression tree * to find and hash clauses which are eligible. */ - if(newNode == NULL) + if (newNode == NULL) { newNode = expression_tree_mutator(originalNode, HashableClauseMutator, (void *) partitionColumn); @@ -2919,6 +2927,10 @@ OpExpressionContainsColumn(OpExpr *operatorExpression, Var *partitionColumn) Node *rightOperand = get_rightop((Expr *) operatorExpression); Var *column = NULL; + /* strip coercions before doing check */ + leftOperand = strip_implicit_coercions(leftOperand); + rightOperand = strip_implicit_coercions(rightOperand); + if (IsA(leftOperand, Var)) { column = (Var *) leftOperand; @@ -3037,7 +3049,7 @@ MakeInt4Constant(Datum constantValue) bool constantIsNull = false; bool constantByValue = true; - Const *int4Constant = makeConst(constantType, constantTypeMode, constantCollationId, + Const *int4Constant = makeConst(constantType, constantTypeMode, constantCollationId, constantLength, constantValue, constantIsNull, constantByValue); return int4Constant; @@ -3094,7 +3106,7 @@ UpdateConstraint(Node *baseConstraint, ShardInterval *shardInterval) Node *greaterThanExpr = (Node *) lsecond(andExpr->args); Node *minNode = get_rightop((Expr *) greaterThanExpr); /* right op */ - Node *maxNode = get_rightop((Expr *) lessThanExpr); /* right op */ + Node *maxNode = get_rightop((Expr *) lessThanExpr); /* right op */ Const *minConstant = NULL; Const *maxConstant = NULL; @@ -3265,7 +3277,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended joinSequenceArray[joinedTableCount].joiningRangeTableId = NON_PRUNABLE_JOIN; joinedTableCount++; - foreach (joinExprCell, joinExprList) + foreach(joinExprCell, joinExprList) { JoinExpr *joinExpr = (JoinExpr *) lfirst(joinExprCell); JoinType joinType = joinExpr->jointype; @@ -3339,7 +3351,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended if (IS_OUTER_JOIN(joinType)) { int innerRangeTableId = 0; - List * tableFragments = NIL; + List *tableFragments = NIL; int fragmentCount = 0; if (joinType == JOIN_RIGHT) @@ -3492,7 +3504,7 @@ FindRangeTableFragmentsList(List *rangeTableFragmentsList, int tableId) if (tableFragments != NIL) { RangeTableFragment *tableFragment = - (RangeTableFragment*) linitial(tableFragments); + (RangeTableFragment *) linitial(tableFragments); if (tableFragment->rangeTableId == tableId) { foundTableFragments = tableFragments; @@ -3698,7 +3710,7 @@ UniqueFragmentList(List *fragmentList) foreach(uniqueFragmentCell, uniqueFragmentList) { RangeTableFragment *uniqueFragment = - (RangeTableFragment *) lfirst(uniqueFragmentCell); + (RangeTableFragment *) lfirst(uniqueFragmentCell); uint64 *uniqueShardId = uniqueFragment->fragmentReference; if (*shardId == *uniqueShardId) @@ -4038,6 +4050,7 @@ FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment) return alias; } + /* * AnchorShardId walks over each fragment in the given fragment list, finds the * fragment that corresponds to the given anchor range tableId, and returns this @@ -4352,7 +4365,7 @@ MergeTaskList(MapMergeJob *mapMergeJob, List *mapTaskList, uint32 taskIdIndex) StringInfo intermediateTableQueryString = IntermediateTableQueryString(jobId, taskIdIndex, reduceQuery); - StringInfo mergeAndRunQueryString= makeStringInfo(); + StringInfo mergeAndRunQueryString = makeStringInfo(); appendStringInfo(mergeAndRunQueryString, MERGE_FILES_AND_RUN_QUERY_COMMAND, jobId, taskIdIndex, mergeTableQueryString->data, intermediateTableQueryString->data); @@ -4678,7 +4691,7 @@ TaskListAppendUnique(List *list, Task *task) List * TaskListConcatUnique(List *list1, List *list2) { - ListCell *taskCell = NULL; + ListCell *taskCell = NULL; foreach(taskCell, list2) { @@ -4952,7 +4965,7 @@ List * FirstReplicaAssignTaskList(List *taskList) { /* No additional reordering need take place for this algorithm */ - List * (*reorderFunction)(Task *, List *) = NULL; + List *(*reorderFunction)(Task *, List *) = NULL; taskList = ReorderAndAssignTaskList(taskList, reorderFunction); @@ -4976,6 +4989,7 @@ RoundRobinAssignTaskList(List *taskList) return taskList; } + /* * RoundRobinReorder implements the core of the round-robin assignment policy. * It takes a task and placement list and rotates a copy of the placement list @@ -5108,7 +5122,8 @@ ActiveShardPlacementLists(List *taskList) List *activeShardPlacementList = ActivePlacementList(shardPlacementList); /* sort shard placements by their insertion time */ - activeShardPlacementList = SortList(activeShardPlacementList, CompareShardPlacements); + activeShardPlacementList = SortList(activeShardPlacementList, + CompareShardPlacements); shardPlacementLists = lappend(shardPlacementLists, activeShardPlacementList); } @@ -5249,7 +5264,8 @@ AssignDualHashTaskList(List *taskList) uint32 replicaIndex = 0; for (replicaIndex = 0; replicaIndex < ShardReplicationFactor; replicaIndex++) { - uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + replicaIndex; + uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + + replicaIndex; uint32 assignmentIndex = assignmentOffset % workerNodeCount; WorkerNode *workerNode = list_nth(workerNodeList, assignmentIndex); diff --git a/src/backend/distributed/planner/multi_planner.c b/src/backend/distributed/planner/multi_planner.c index 93d05817e..28780fbdf 100644 --- a/src/backend/distributed/planner/multi_planner.c +++ b/src/backend/distributed/planner/multi_planner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_planner.c - * General CitusDB planner code. + * General Citus planner code. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -45,26 +45,17 @@ multi_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) PlannedStmt *result = NULL; /* - * First call into standard planner. This is required because the CitusDB + * First call into standard planner. This is required because the Citus * planner relies on parse tree transformations made by postgres' planner. */ result = standard_planner(parse, cursorOptions, boundParams); if (NeedsDistributedPlanning(parse)) { - MemoryContext oldcontext = NULL; - MultiPlan *physicalPlan = NULL; - - /* Switch to top level message context */ - oldcontext = MemoryContextSwitchTo(MessageContext); - - physicalPlan = CreatePhysicalPlan(parse); + MultiPlan *physicalPlan = CreatePhysicalPlan(parse); /* store required data into the planned statement */ result = MultiQueryContainerNode(result, physicalPlan); - - /* Now switch back to original context */ - MemoryContextSwitchTo(oldcontext); } return result; @@ -99,7 +90,7 @@ CreatePhysicalPlan(Query *parse) /* * This check is here to make it likely that all node types used in - * CitusDB are dumpable. Explain can dump logical and physical plans + * Citus are dumpable. Explain can dump logical and physical plans * using the extended outfuncs infrastructure, but it's infeasible to * test most plans. MultiQueryContainerNode always serializes the * physical plan, so there's no need to check that separately. @@ -141,7 +132,7 @@ HasCitusToplevelNode(PlannedStmt *result) * yet. Directly return false, part of the required infrastructure for * further checks might not be present. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return false; } @@ -268,7 +259,7 @@ GetMultiPlanString(PlannedStmt *result) if (list_length(fauxFuncExpr->args) != 1) { ereport(ERROR, (errmsg("unexpected number of function arguments to " - "citusdb_extradata_container"))); + "citus_extradata_container"))); } multiPlanData = (Const *) linitial(fauxFuncExpr->args); diff --git a/src/backend/distributed/relay/relay_event_utility.c b/src/backend/distributed/relay/relay_event_utility.c index 926f4f25e..49a9f8d34 100644 --- a/src/backend/distributed/relay/relay_event_utility.c +++ b/src/backend/distributed/relay/relay_event_utility.c @@ -35,7 +35,7 @@ /* Local functions forward declarations */ static bool TypeAddIndexConstraint(const AlterTableCmd *command); -static bool TypeDropIndexConstraint(const AlterTableCmd *command, +static bool TypeDropIndexConstraint(const AlterTableCmd *command, const RangeVar *relation, uint64 shardId); static void AppendShardIdToConstraintName(AlterTableCmd *command, uint64 shardId); @@ -67,7 +67,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) AppendShardIdToName(sequenceName, shardId); break; } - + case T_AlterTableStmt: { /* @@ -79,7 +79,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) AlterTableStmt *alterTableStmt = (AlterTableStmt *) parseTree; char **relationName = &(alterTableStmt->relation->relname); - RangeVar *relation = alterTableStmt->relation; /* for constraints */ + RangeVar *relation = alterTableStmt->relation; /* for constraints */ List *commandList = alterTableStmt->cmds; ListCell *commandCell = NULL; @@ -179,15 +179,15 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) objectType == OBJECT_INDEX || objectType == OBJECT_FOREIGN_TABLE || objectType == OBJECT_FOREIGN_SERVER) { - List *relationNameList = NULL; - int relationNameListLength = 0; + List *relationNameList = NULL; + int relationNameListLength = 0; Value *relationNameValue = NULL; - char **relationName = NULL; + char **relationName = NULL; uint32 dropCount = list_length(dropStmt->objects); if (dropCount > 1) { - ereport(ERROR, + ereport(ERROR, (errmsg("cannot extend name for multiple drop objects"))); } @@ -205,19 +205,30 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) switch (relationNameListLength) { case 1: + { relationNameValue = linitial(relationNameList); break; + } + case 2: + { relationNameValue = lsecond(relationNameList); break; + } + case 3: + { relationNameValue = lthird(relationNameList); break; + } + default: + { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("improper relation name: \"%s\"", NameListToString(relationNameList)))); break; + } } relationName = &(relationNameValue->val.str); @@ -304,7 +315,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) { RenameStmt *renameStmt = (RenameStmt *) parseTree; ObjectType objectType = renameStmt->renameType; - + if (objectType == OBJECT_TABLE || objectType == OBJECT_SEQUENCE || objectType == OBJECT_INDEX) { @@ -335,7 +346,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) * We currently do not support truncate statements. This is * primarily because truncates allow implicit modifications to * sequences through table column dependencies. As we have not - * determined our dependency model for sequences, we error here. + * determined our dependency model for sequences, we error here. */ ereport(ERROR, (errmsg("cannot extend name for truncate statement"))); break; @@ -384,18 +395,18 @@ TypeAddIndexConstraint(const AlterTableCmd *command) * associated with an index. */ static bool -TypeDropIndexConstraint(const AlterTableCmd *command, +TypeDropIndexConstraint(const AlterTableCmd *command, const RangeVar *relation, uint64 shardId) { Relation pgConstraint = NULL; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; char *searchedConstraintName = NULL; - bool indexConstraint = false; - Oid relationId = InvalidOid; + bool indexConstraint = false; + Oid relationId = InvalidOid; bool failOK = true; if (command->subtype != AT_DropConstraint) @@ -423,8 +434,8 @@ TypeDropIndexConstraint(const AlterTableCmd *command, ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); - - scanDescriptor = systable_beginscan(pgConstraint, + + scanDescriptor = systable_beginscan(pgConstraint, ConstraintRelidIndexId, true, /* indexOK */ NULL, scanKeyCount, scanKey); @@ -433,7 +444,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); char *constraintName = NameStr(constraintForm->conname); - + if (strncmp(constraintName, searchedConstraintName, NAMEDATALEN) == 0) { /* we found the constraint, now check if it is for an index */ @@ -442,7 +453,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, { indexConstraint = true; } - + break; } @@ -451,7 +462,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, systable_endscan(scanDescriptor); heap_close(pgConstraint, AccessShareLock); - + pfree(searchedConstraintName); return indexConstraint; @@ -489,10 +500,10 @@ AppendShardIdToConstraintName(AlterTableCmd *command, uint64 shardId) void AppendShardIdToName(char **name, uint64 shardId) { - char extendedName[NAMEDATALEN]; + char extendedName[NAMEDATALEN]; uint32 extendedNameLength = 0; - snprintf(extendedName, NAMEDATALEN, "%s%c" UINT64_FORMAT, + snprintf(extendedName, NAMEDATALEN, "%s%c" UINT64_FORMAT, (*name), SHARD_NAME_SEPARATOR, shardId); /* diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 344665eb0..332319b2a 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * shared_library_init.c - * Initialize CitusDB extension + * Initialize Citus extension * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -48,23 +48,23 @@ static void NormalizeWorkerListPath(void); /* GUC enum definitions */ static const struct config_enum_entry task_assignment_policy_options[] = { - {"greedy", TASK_ASSIGNMENT_GREEDY, false}, - {"first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false}, - {"round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false}, - {NULL, 0, false} + { "greedy", TASK_ASSIGNMENT_GREEDY, false }, + { "first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false }, + { "round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false }, + { NULL, 0, false } }; static const struct config_enum_entry task_executor_type_options[] = { - {"real-time", MULTI_EXECUTOR_REAL_TIME, false}, - {"task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false}, - {"router", MULTI_EXECUTOR_ROUTER, false}, - {NULL, 0, false} + { "real-time", MULTI_EXECUTOR_REAL_TIME, false }, + { "task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false }, + { "router", MULTI_EXECUTOR_ROUTER, false }, + { NULL, 0, false } }; static const struct config_enum_entry shard_placement_policy_options[] = { - {"local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false}, - {"round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false}, - {NULL, 0, false} + { "local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false }, + { "round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false }, + { NULL, 0, false } }; @@ -74,8 +74,8 @@ _PG_init(void) { if (!process_shared_preload_libraries_in_progress) { - ereport(ERROR, (errmsg("CitusDB can only be loaded via shared_preload_libraries"), - errhint("Add citusdb to shared_preload_libraries."))); + ereport(ERROR, (errmsg("Citus can only be loaded via shared_preload_libraries"), + errhint("Add citus to shared_preload_libraries."))); } /* @@ -95,8 +95,8 @@ _PG_init(void) ExecutorEnd_hook != NULL || ProcessUtility_hook != NULL) { - ereport(ERROR, (errmsg("CitusDB has to be loaded first"), - errhint("Place citusdb at the beginning of " + ereport(ERROR, (errmsg("Citus has to be loaded first"), + errhint("Place citus at the beginning of " "shared_preload_libraries."))); } @@ -107,7 +107,7 @@ _PG_init(void) CreateRequiredDirectories(); /* - * Register CitusDB configuration variables. Do so before intercepting + * Register Citus configuration variables. Do so before intercepting * hooks or calling initialization functions, in case we want to do the * latter in a configuration dependent manner. */ @@ -137,7 +137,7 @@ _PG_init(void) /* - * CreateRequiredDirectories - Create directories required for CitusDB to + * CreateRequiredDirectories - Create directories required for Citus to * function. * * These used to be created by initdb, but that's not possible anymore. @@ -166,12 +166,12 @@ CreateRequiredDirectories(void) } -/* Register CitusDB configuration variables. */ +/* Register Citus configuration variables. */ static void RegisterCitusConfigVariables(void) { DefineCustomStringVariable( - "citusdb.worker_list_file", + "citus.worker_list_file", gettext_noop("Sets the server's \"worker_list\" configuration file."), NULL, &WorkerListFileName, @@ -182,7 +182,7 @@ RegisterCitusConfigVariables(void) NormalizeWorkerListPath(); DefineCustomBoolVariable( - "citusdb.binary_master_copy_format", + "citus.binary_master_copy_format", gettext_noop("Use the binary master copy format."), gettext_noop("When enabled, data is copied from workers to the master " "in PostgreSQL's binary serialization format."), @@ -193,7 +193,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.binary_worker_copy_format", + "citus.binary_worker_copy_format", gettext_noop("Use the binary worker copy format."), gettext_noop("When enabled, data is copied from workers to workers " "in PostgreSQL's binary serialization format when " @@ -205,10 +205,11 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.expire_cached_shards", - gettext_noop("Enables shard cache expiration if a shard's size on disk has changed. "), - gettext_noop("When appending to an existing shard, old data may still be cached on " - "other workers. This configuration entry activates automatic " + "citus.expire_cached_shards", + gettext_noop("Enables shard cache expiration if a shard's size on disk has " + "changed."), + gettext_noop("When appending to an existing shard, old data may still be cached " + "on other workers. This configuration entry activates automatic " "expiration, but should not be used with manual updates to shards."), &ExpireCachedShards, false, @@ -217,7 +218,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.subquery_pushdown", + "citus.subquery_pushdown", gettext_noop("Enables supported subquery pushdown to workers."), NULL, &SubqueryPushdown, @@ -227,7 +228,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.log_multi_join_order", + "citus.log_multi_join_order", gettext_noop("Logs the distributed join order to the server log."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, we print the distributed join order."), @@ -238,7 +239,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.explain_multi_logical_plan", + "citus.explain_multi_logical_plan", gettext_noop("Enables Explain to print out distributed logical plans."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, the Explain command prints out the optimized " @@ -250,7 +251,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.explain_multi_physical_plan", + "citus.explain_multi_physical_plan", gettext_noop("Enables Explain to print out distributed physical plans."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, the Explain command prints out the physical " @@ -262,7 +263,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.all_modifications_commutative", + "citus.all_modifications_commutative", gettext_noop("Bypasses commutativity checks when enabled"), NULL, &AllModificationsCommutative, @@ -272,7 +273,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.shard_replication_factor", + "citus.shard_replication_factor", gettext_noop("Sets the replication factor for shards."), gettext_noop("Shards are replicated across nodes according to this " "replication factor. Note that shards read this " @@ -285,7 +286,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.shard_max_size", + "citus.shard_max_size", gettext_noop("Sets the maximum size a shard will grow before it gets split."), gettext_noop("Shards store table and file data. When the source " "file's size for one shard exceeds this configuration " @@ -300,7 +301,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_worker_nodes_tracked", + "citus.max_worker_nodes_tracked", gettext_noop("Sets the maximum number of worker nodes that are tracked."), gettext_noop("Worker nodes' network locations, their membership and " "health status are tracked in a shared hash table on " @@ -314,7 +315,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.remote_task_check_interval", + "citus.remote_task_check_interval", gettext_noop("Sets the frequency at which we check job statuses."), gettext_noop("The master node assigns tasks to workers nodes, and " "then regularly checks with them about each task's " @@ -327,7 +328,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.task_tracker_delay", + "citus.task_tracker_delay", gettext_noop("Task tracker sleep time between task management rounds."), gettext_noop("The task tracker process wakes up regularly, walks over " "all tasks assigned to it, and schedules and executes these " @@ -341,7 +342,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_assign_task_batch_size", + "citus.max_assign_task_batch_size", gettext_noop("Sets the maximum number of tasks to assign per round."), gettext_noop("The master node synchronously assigns tasks to workers in " "batches. Bigger batches allow for faster task assignment, " @@ -355,7 +356,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_tracked_tasks_per_node", + "citus.max_tracked_tasks_per_node", gettext_noop("Sets the maximum number of tracked tasks per node."), gettext_noop("The task tracker processes keeps all assigned tasks in " "a shared hash table, and schedules and executes these " @@ -369,7 +370,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_running_tasks_per_node", + "citus.max_running_tasks_per_node", gettext_noop("Sets the maximum number of tasks to run concurrently per node."), gettext_noop("The task tracker process schedules and executes the tasks " "assigned to it as appropriate. This configuration value " @@ -382,7 +383,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.partition_buffer_size", + "citus.partition_buffer_size", gettext_noop("Sets the buffer size to use for partition operations."), gettext_noop("Worker nodes allow for table data to be repartitioned " "into multiple text files, much like Hadoop's Map " @@ -396,7 +397,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.large_table_shard_count", + "citus.large_table_shard_count", gettext_noop("The shard count threshold over which a table is considered large."), gettext_noop("A distributed table is considered to be large if it has " "more shards than the value specified here. This largeness " @@ -409,7 +410,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.limit_clause_row_fetch_count", + "citus.limit_clause_row_fetch_count", gettext_noop("Number of rows to fetch per task for limit clause optimization."), gettext_noop("Select queries get partitioned and executed as smaller " "tasks. In some cases, select queries with limit clauses " @@ -424,7 +425,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomRealVariable( - "citusdb.count_distinct_error_rate", + "citus.count_distinct_error_rate", gettext_noop("Desired error rate when calculating count(distinct) " "approximates using the postgresql-hll extension. " "0.0 disables approximations for count(distinct); 1.0 " @@ -437,14 +438,14 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.task_assignment_policy", + "citus.task_assignment_policy", gettext_noop("Sets the policy to use when assigning tasks to worker nodes."), gettext_noop("The master node assigns tasks to worker nodes based on shard " - "locations. This configuration value specifies the policy to " - "use when making these assignments. The greedy policy aims to " - "evenly distribute tasks across worker nodes, first-replica just " - "assigns tasks in the order shard placements were created, " - "and the round-robin policy assigns tasks to worker nodes in " + "locations. This configuration value specifies the policy to " + "use when making these assignments. The greedy policy aims to " + "evenly distribute tasks across worker nodes, first-replica just " + "assigns tasks in the order shard placements were created, " + "and the round-robin policy assigns tasks to worker nodes in " "a round-robin fashion."), &TaskAssignmentPolicy, TASK_ASSIGNMENT_GREEDY, @@ -454,7 +455,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.task_executor_type", + "citus.task_executor_type", gettext_noop("Sets the executor type to be used for distributed queries."), gettext_noop("The master node chooses between three different executor types " "when executing a distributed query. The router executor is " @@ -472,7 +473,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.shard_placement_policy", + "citus.shard_placement_policy", gettext_noop("Sets the policy to use when choosing nodes for shard placement."), gettext_noop("The master node chooses which worker nodes to place new shards " "on. This configuration value specifies the policy to use when " @@ -486,16 +487,14 @@ RegisterCitusConfigVariables(void) 0, NULL, NULL, NULL); - /* warn about config items in the citusdb namespace that are not registered above */ - EmitWarningsOnPlaceholders("citusdb"); - /* Also warn about citus namespace, as that's a very likely misspelling */ + /* warn about config items in the citus namespace that are not registered above */ EmitWarningsOnPlaceholders("citus"); } /* * NormalizeWorkerListPath converts the path configured via - * citusdb.worker_list_file into an absolute path, falling back to the default + * citus.worker_list_file into an absolute path, falling back to the default * value if necessary. The previous value of the config variable is * overwritten with the normalized value. * @@ -515,8 +514,10 @@ NormalizeWorkerListPath(void) { absoluteFileName = malloc(strlen(DataDir) + strlen(WORKER_LIST_FILENAME) + 2); if (absoluteFileName == NULL) + { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + } sprintf(absoluteFileName, "%s/%s", DataDir, WORKER_LIST_FILENAME); } @@ -525,11 +526,12 @@ NormalizeWorkerListPath(void) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("%s does not know where to find the \"worker_list_file\" " "configuration file.\n" - "This can be specified as \"citusdb.worker_list_file\" in " + "This can be specified as \"citus.worker_list_file\" in " "\"%s\", or by the -D invocation option, or by the PGDATA " "environment variable.\n", progname, ConfigFileName))); } - SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER, PGC_S_OVERRIDE); + SetConfigOption("citus.worker_list_file", absoluteFileName, PGC_POSTMASTER, + PGC_S_OVERRIDE); free(absoluteFileName); } diff --git a/src/backend/distributed/test/connection_cache.c b/src/backend/distributed/test/connection_cache.c index 25fcbf593..691020a55 100644 --- a/src/backend/distributed/test/connection_cache.c +++ b/src/backend/distributed/test/connection_cache.c @@ -2,7 +2,7 @@ * * test/src/connection_cache.c * - * This file contains functions to exercise CitusDB's connection hash + * This file contains functions to exercise Citus's connection hash * functionality for purposes of unit testing. * * Copyright (c) 2014-2015, Citus Data, Inc. @@ -48,7 +48,7 @@ initialize_remote_temp_table(PG_FUNCTION_ARGS) int32 nodePort = PG_GETARG_INT32(1); PGresult *result = NULL; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); @@ -79,7 +79,7 @@ count_remote_temp_table_rows(PG_FUNCTION_ARGS) Datum count = Int32GetDatum(-1); PGresult *result = NULL; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_DATUM(count); @@ -114,7 +114,7 @@ get_and_purge_connection(PG_FUNCTION_ARGS) char *nodeName = PG_GETARG_CSTRING(0); int32 nodePort = PG_GETARG_INT32(1); - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); @@ -136,7 +136,7 @@ set_connection_status_bad(PG_FUNCTION_ARGS) char *nodeName = PG_GETARG_CSTRING(0); int32 nodePort = PG_GETARG_INT32(1); - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); diff --git a/src/backend/distributed/test/create_shards.c b/src/backend/distributed/test/create_shards.c index e6b2b5762..eb8a4d8af 100644 --- a/src/backend/distributed/test/create_shards.c +++ b/src/backend/distributed/test/create_shards.c @@ -3,7 +3,7 @@ * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/distribution_metadata.c b/src/backend/distributed/test/distribution_metadata.c index f54359a19..83bb02c7c 100644 --- a/src/backend/distributed/test/distribution_metadata.c +++ b/src/backend/distributed/test/distribution_metadata.c @@ -3,7 +3,7 @@ * test/src/distribution_metadata.c * * This file contains functions to exercise distributed table metadata - * functionality within CitusDB. + * functionality within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index 883c77755..9409dde3b 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -116,9 +116,9 @@ FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses) #else static ForeignScan * -FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, - ForeignPath *best_path, List *tlist, List *scan_clauses, - Plan *outer_plan) +FakeGetForeignPlan(PlannerInfo * root, RelOptInfo * baserel, Oid foreigntableid, + ForeignPath * best_path, List * tlist, List * scan_clauses, + Plan * outer_plan) #endif { Index scan_relid = baserel->relid; @@ -129,7 +129,7 @@ FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, return make_foreignscan(tlist, scan_clauses, scan_relid, NIL, NIL); #else return make_foreignscan(tlist, scan_clauses, scan_relid, NIL, NIL, NIL, NIL, - outer_plan); + outer_plan); #endif } diff --git a/src/backend/distributed/test/generate_ddl_commands.c b/src/backend/distributed/test/generate_ddl_commands.c index 4cbc225cf..4b6ea04bc 100644 --- a/src/backend/distributed/test/generate_ddl_commands.c +++ b/src/backend/distributed/test/generate_ddl_commands.c @@ -3,7 +3,7 @@ * test/src/generate_ddl_commands.c * * This file contains functions to exercise DDL generation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index f722a70ec..b7edf661e 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -3,7 +3,7 @@ * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/test_helper_functions.c b/src/backend/distributed/test/test_helper_functions.c index bc463c4c6..28458d96d 100644 --- a/src/backend/distributed/test/test_helper_functions.c +++ b/src/backend/distributed/test/test_helper_functions.c @@ -2,7 +2,7 @@ * * test/src/test_helper_functions.c * - * This file contains helper functions used in many CitusDB tests. + * This file contains helper functions used in many Citus tests. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index b3858cbdf..eecf13f52 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -16,7 +16,7 @@ /* exports for SQL callable functions */ -PG_FUNCTION_INFO_V1(citusdb_extradata_container); +PG_FUNCTION_INFO_V1(citus_extradata_container); /* @@ -189,7 +189,7 @@ ExtractRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind *rteKind, if (list_length(fauxFuncExpr->args) != 4) { ereport(ERROR, (errmsg("unexpected number of function arguments to " - "citusdb_extradata_container"))); + "citus_extradata_container"))); return; } @@ -265,7 +265,7 @@ GetRangeTblKind(RangeTblEntry *rte) { CitusRTEKind rteKind = CITUS_RTE_RELATION /* invalid */; - switch(rte->rtekind) + switch (rte->rtekind) { /* directly rtekind if it's not possibly an extended RTE */ case RTE_RELATION: @@ -273,9 +273,13 @@ GetRangeTblKind(RangeTblEntry *rte) case RTE_JOIN: case RTE_VALUES: case RTE_CTE: + { rteKind = (CitusRTEKind) rte->rtekind; break; + } + case RTE_FUNCTION: + { /* * Extract extra data - correct even if a plain RTE_FUNCTION, not * an extended one, ExtractRangeTblExtraData handles that case @@ -283,6 +287,7 @@ GetRangeTblKind(RangeTblEntry *rte) */ ExtractRangeTblExtraData(rte, &rteKind, NULL, NULL, NULL); break; + } } return rteKind; @@ -290,13 +295,13 @@ GetRangeTblKind(RangeTblEntry *rte) /* - * citusdb_extradata_container is a placeholder function to store information - * needed by CitusDB in plain postgres node trees. Executor and other hooks + * citus_extradata_container is a placeholder function to store information + * needed by Citus in plain postgres node trees. Executor and other hooks * should always intercept statements containing calls to this function. It's * not actually SQL callable by the user because of an INTERNAL argument. */ Datum -citusdb_extradata_container(PG_FUNCTION_ARGS) +citus_extradata_container(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("not supposed to get here, did you cheat?"))); diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index bb620987a..babd10620 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_outfuncs.c - * Output functions for CitusDB tree nodes. + * Output functions for Citus tree nodes. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -9,7 +9,7 @@ * * NOTES * This is a wrapper around postgres' nodeToString() that additionally - * supports CitusDB node types. + * supports Citus node types. * * Keep as closely aligned with the upstream version as possible. * @@ -220,7 +220,7 @@ _outDatum(StringInfo str, Datum value, int typlen, bool typbyval) /***************************************************************************** - * Output routines for CitusDB node types + * Output routines for Citus node types *****************************************************************************/ static void diff --git a/src/backend/distributed/utils/citus_readfuncs_94.c b/src/backend/distributed/utils/citus_readfuncs_94.c index ace76bb3a..7cbf124c4 100644 --- a/src/backend/distributed/utils/citus_readfuncs_94.c +++ b/src/backend/distributed/utils/citus_readfuncs_94.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_readfuncs.c - * CitusDB adapted reader functions for Citus & Postgres tree nodes + * Citus adapted reader functions for Citus & Postgres tree nodes * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/backend/distributed/utils/citus_readfuncs_95.c b/src/backend/distributed/utils/citus_readfuncs_95.c index 84866f632..87b1d827a 100644 --- a/src/backend/distributed/utils/citus_readfuncs_95.c +++ b/src/backend/distributed/utils/citus_readfuncs_95.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_readfuncs.c - * CitusDB adapted reader functions for Citus & Postgres tree nodes + * Citus adapted reader functions for Citus & Postgres tree nodes * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/backend/distributed/utils/citus_ruleutils.c b/src/backend/distributed/utils/citus_ruleutils.c index ad1ac7332..dda9a86fc 100644 --- a/src/backend/distributed/utils/citus_ruleutils.c +++ b/src/backend/distributed/utils/citus_ruleutils.c @@ -102,6 +102,7 @@ pg_get_extensiondef_string(Oid tableRelationId) static Oid get_extension_schema(Oid ext_oid) { + /* *INDENT-OFF* */ Oid result; Relation rel; SysScanDesc scandesc; @@ -131,6 +132,7 @@ get_extension_schema(Oid ext_oid) heap_close(rel, AccessShareLock); return result; + /* *INDENT-ON* */ } @@ -186,7 +188,7 @@ AppendOptionListToString(StringInfo stringBuffer, List *optionList) foreach(optionCell, optionList) { - DefElem *option = (DefElem*) lfirst(optionCell); + DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; char *optionValue = defGetString(option); @@ -219,7 +221,7 @@ pg_get_tableschemadef_string(Oid tableRelationId) char relationKind = 0; TupleDesc tupleDescriptor = NULL; TupleConstr *tupleConstraints = NULL; - int attributeIndex = 0; + int attributeIndex = 0; bool firstAttributePrinted = false; AttrNumber defaultValueIndex = 0; AttrNumber constraintIndex = 0; @@ -447,21 +449,35 @@ pg_get_tablecolumnoptionsdef_string(Oid tableRelationId) switch (attributeForm->attstorage) { case 'p': + { storageName = "PLAIN"; break; + } + case 'e': + { storageName = "EXTERNAL"; break; + } + case 'm': + { storageName = "MAIN"; break; + } + case 'x': + { storageName = "EXTENDED"; break; + } + default: + { ereport(ERROR, (errmsg("unrecognized storage type: %c", attributeForm->attstorage))); break; + } } appendStringInfo(&statement, "ALTER COLUMN %s ", diff --git a/src/backend/distributed/utils/connection_cache.c b/src/backend/distributed/utils/connection_cache.c index 4281a4a59..a7391aa5b 100644 --- a/src/backend/distributed/utils/connection_cache.c +++ b/src/backend/distributed/utils/connection_cache.c @@ -32,7 +32,7 @@ /* * NodeConnectionHash is the connection hash itself. It begins uninitialized. - * The first call to GetConnection triggers hash creation. + * The first call to GetOrEstablishConnection triggers hash creation. */ static HTAB *NodeConnectionHash = NULL; @@ -44,10 +44,10 @@ static char * ConnectionGetOptionValue(PGconn *connection, char *optionKeyword); /* - * GetConnection returns a PGconn which can be used to execute queries on a - * remote PostgreSQL server. If no suitable connection to the specified node on - * the specified port yet exists, the function establishes a new connection and - * returns that. + * GetOrEstablishConnection returns a PGconn which can be used to execute + * queries on a remote PostgreSQL server. If no suitable connection to the + * specified node on the specified port yet exists, the function establishes + * a new connection and adds it to the connection cache before returning it. * * Returned connections are guaranteed to be in the CONNECTION_OK state. If the * requested connection cannot be established, or if it was previously created @@ -56,7 +56,7 @@ static char * ConnectionGetOptionValue(PGconn *connection, char *optionKeyword); * This function throws an error if a hostname over 255 characters is provided. */ PGconn * -GetConnection(char *nodeName, int32 nodePort) +GetOrEstablishConnection(char *nodeName, int32 nodePort) { PGconn *connection = NULL; NodeConnectionKey nodeConnectionKey; @@ -249,7 +249,7 @@ CreateNodeConnectionHash(void) info.hcxt = CacheMemoryContext; hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); - nodeConnectionHash = hash_create("citusdb connection cache", 32, &info, hashFlags); + nodeConnectionHash = hash_create("citus connection cache", 32, &info, hashFlags); return nodeConnectionHash; } @@ -257,7 +257,7 @@ CreateNodeConnectionHash(void) /* * ConnectToNode opens a connection to a remote PostgreSQL server. The function - * configures the connection's fallback application name to 'citusdb' and sets + * configures the connection's fallback application name to 'citus' and sets * the remote encoding to match the local one. This function requires that the * port be specified as a string for easier use with libpq functions. * @@ -277,7 +277,7 @@ ConnectToNode(char *nodeName, char *nodePort) "client_encoding", "connect_timeout", "dbname", NULL }; const char *valueArray[] = { - nodeName, nodePort, "citusdb", clientEncoding, + nodeName, nodePort, "citus", clientEncoding, CLIENT_CONNECT_TIMEOUT_SECONDS, dbname, NULL }; diff --git a/src/backend/distributed/utils/metadata_cache.c b/src/backend/distributed/utils/metadata_cache.c index ec034e2b6..6cb29cb97 100644 --- a/src/backend/distributed/utils/metadata_cache.c +++ b/src/backend/distributed/utils/metadata_cache.c @@ -51,10 +51,10 @@ static void InvalidateDistRelationCacheCallback(Datum argument, Oid relationId); static HeapTuple LookupDistPartitionTuple(Oid relationId); static List * LookupDistShardTuples(Oid relationId); static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, - Oid *intervalTypeId, int32 *intervalTypeMod); + Oid *intervalTypeId, int32 *intervalTypeMod); static ShardInterval * TupleToShardInterval(HeapTuple heapTuple, - TupleDesc tupleDescriptor, Oid intervalTypeId, - int32 intervalTypeMod); + TupleDesc tupleDescriptor, Oid intervalTypeId, + int32 intervalTypeMod); static void CachedRelationLookup(const char *relationName, Oid *cachedOid); @@ -77,7 +77,7 @@ IsDistributedTable(Oid relationId) * yet. As we can't do lookups in nonexistent tables, directly return * false. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return false; } @@ -87,6 +87,7 @@ IsDistributedTable(Oid relationId) return cacheEntry->isDistributedTable; } + /* * LoadShardInterval reads shard metadata for given shardId from pg_dist_shard, * and converts min/max values in these metadata to their properly typed datum @@ -98,7 +99,7 @@ LoadShardInterval(uint64 shardId) { ShardInterval *shardInterval; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; Form_pg_dist_shard shardForm = NULL; @@ -127,11 +128,11 @@ LoadShardInterval(uint64 shardId) partitionEntry = DistributedTableCacheEntry(shardForm->logicalrelid); GetPartitionTypeInputInfo(partitionEntry->partitionKeyString, - partitionEntry->partitionMethod, &intervalTypeId, - &intervalTypeMod); + partitionEntry->partitionMethod, &intervalTypeId, + &intervalTypeMod); shardInterval = TupleToShardInterval(heapTuple, tupleDescriptor, intervalTypeId, - intervalTypeMod); + intervalTypeMod); systable_endscan(scanDescriptor); heap_close(pgDistShard, AccessShareLock); @@ -139,6 +140,7 @@ LoadShardInterval(uint64 shardId) return shardInterval; } + /* * DistributedTableCacheEntry looks up a pg_dist_partition entry for a * relation. @@ -155,7 +157,7 @@ DistributedTableCacheEntry(Oid distributedRelationId) * yet. As we can't do lookups in nonexistent tables, directly return NULL * here. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return NULL; } @@ -239,19 +241,19 @@ LookupDistTableCacheEntry(Oid relationId) int32 intervalTypeMod = -1; GetPartitionTypeInputInfo(partitionKeyString, partitionMethod, &intervalTypeId, - &intervalTypeMod); + &intervalTypeMod); shardIntervalArray = MemoryContextAllocZero(CacheMemoryContext, - shardIntervalArrayLength * - sizeof(ShardInterval)); + shardIntervalArrayLength * + sizeof(ShardInterval)); foreach(distShardTupleCell, distShardTupleList) { HeapTuple shardTuple = lfirst(distShardTupleCell); ShardInterval *shardInterval = TupleToShardInterval(shardTuple, - distShardTupleDesc, - intervalTypeId, - intervalTypeMod); + distShardTupleDesc, + intervalTypeId, + intervalTypeMod); MemoryContext oldContext = MemoryContextSwitchTo(CacheMemoryContext); CopyShardInterval(shardInterval, &shardIntervalArray[arrayIndex]); @@ -292,7 +294,7 @@ LookupDistTableCacheEntry(Oid relationId) /* - * CitusDBHasBeenLoaded returns true if the citusdb extension has been created + * CitusHasBeenLoaded returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. The result is cached as this is called very frequently. * @@ -301,17 +303,17 @@ LookupDistTableCacheEntry(Oid relationId) * acceptable. */ bool -CitusDBHasBeenLoaded(void) +CitusHasBeenLoaded(void) { static bool extensionLoaded = false; - /* recheck presence until citusdb has been loaded */ + /* recheck presence until citus has been loaded */ if (!extensionLoaded) { bool extensionPresent = false; bool extensionScriptExecuted = true; - Oid extensionOid = get_extension_oid("citusdb", true); + Oid extensionOid = get_extension_oid("citus", true); if (extensionOid != InvalidOid) { extensionPresent = true; @@ -319,7 +321,7 @@ CitusDBHasBeenLoaded(void) if (extensionPresent) { - /* check if CitusDB extension objects are still being created */ + /* check if Citus extension objects are still being created */ if (creating_extension && CurrentExtensionObject == extensionOid) { extensionScriptExecuted = false; @@ -428,7 +430,7 @@ CitusExtraDataContainerFuncId(void) if (cachedOid == InvalidOid) { nameList = list_make2(makeString("pg_catalog"), - makeString("citusdb_extradata_container")); + makeString("citus_extradata_container")); cachedOid = LookupFuncName(nameList, 1, paramOids, false); } @@ -741,7 +743,7 @@ LookupDistShardTuples(Oid relationId) scanKey[0].sk_argument = ObjectIdGetDatum(relationId); scanDescriptor = systable_beginscan(pgDistShard, DistShardLogicalRelidIndexId(), true, - NULL, 1, scanKey); + NULL, 1, scanKey); currentShardTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(currentShardTuple)) @@ -765,7 +767,7 @@ LookupDistShardTuples(Oid relationId) */ static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, - Oid *intervalTypeId, int32 *intervalTypeMod) + Oid *intervalTypeId, int32 *intervalTypeMod) { *intervalTypeId = InvalidOid; *intervalTypeMod = -1; @@ -794,7 +796,7 @@ GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported table partition type: %c", - partitionMethod))); + partitionMethod))); } } } @@ -806,7 +808,7 @@ GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, */ static ShardInterval * TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid intervalTypeId, - int32 intervalTypeMod) + int32 intervalTypeMod) { ShardInterval *shardInterval = NULL; bool isNull = false; @@ -815,16 +817,16 @@ TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid interva Oid inputFunctionId = InvalidOid; Oid typeIoParam = InvalidOid; Datum relationIdDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_logicalrelid, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum shardIdDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardid, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum storageTypeDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardstorage, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum minValueTextDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardminvalue, - tupleDescriptor, &minValueNull); + tupleDescriptor, &minValueNull); Datum maxValueTextDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardmaxvalue, - tupleDescriptor, &maxValueNull); + tupleDescriptor, &maxValueNull); Oid relationId = DatumGetObjectId(relationIdDatum); int64 shardId = DatumGetInt64(shardIdDatum); @@ -845,7 +847,7 @@ TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid interva /* TODO: move this up the call stack to avoid per-tuple invocation? */ get_type_io_data(intervalTypeId, IOFunc_input, &intervalTypeLen, &intervalByVal, - &intervalAlign, &intervalDelim, &typeIoParam, &inputFunctionId); + &intervalAlign, &intervalDelim, &typeIoParam, &inputFunctionId); /* finally convert min/max values to their actual types */ minValue = OidInputFunctionCall(inputFunctionId, minValueString, diff --git a/src/backend/distributed/utils/multi_resowner.c b/src/backend/distributed/utils/multi_resowner.c index 80aecf7e5..380479a4d 100644 --- a/src/backend/distributed/utils/multi_resowner.c +++ b/src/backend/distributed/utils/multi_resowner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_resowner.c - * CitusDB resource owner integration + * Citus resource owner integration * * An extension can't directly add members to ResourceOwnerData. Instead we * have to use the resource owner callback mechanism. Right now it's @@ -22,7 +22,8 @@ #include "distributed/multi_resowner.h" -typedef struct JobDirectoryEntry { +typedef struct JobDirectoryEntry +{ ResourceOwner owner; uint64 jobId; } JobDirectoryEntry; @@ -44,8 +45,8 @@ MultiResourceOwnerReleaseCallback(ResourceReleasePhase phase, bool isTopLevel, void *arg) { - int lastJobIndex = NumRegisteredJobDirectories - 1; - int jobIndex = 0; + int lastJobIndex = NumRegisteredJobDirectories - 1; + int jobIndex = 0; if (phase == RESOURCE_RELEASE_AFTER_LOCKS) { @@ -79,7 +80,7 @@ MultiResourceOwnerReleaseCallback(ResourceReleasePhase phase, void ResourceOwnerEnlargeJobDirectories(ResourceOwner owner) { - int newMax = 0; + int newMax = 0; /* ensure callback is registered */ if (!RegisteredResownerCallback) @@ -91,15 +92,17 @@ ResourceOwnerEnlargeJobDirectories(ResourceOwner owner) if (RegisteredJobDirectories == NULL) { newMax = 16; - RegisteredJobDirectories = (JobDirectoryEntry *) - MemoryContextAlloc(TopMemoryContext, newMax * sizeof(JobDirectoryEntry)); + RegisteredJobDirectories = + (JobDirectoryEntry *) MemoryContextAlloc(TopMemoryContext, + newMax * sizeof(JobDirectoryEntry)); NumAllocatedJobDirectories = newMax; } else if (NumRegisteredJobDirectories + 1 > NumAllocatedJobDirectories) { newMax = NumAllocatedJobDirectories * 2; - RegisteredJobDirectories = (JobDirectoryEntry *) - repalloc(RegisteredJobDirectories, newMax * sizeof(JobDirectoryEntry)); + RegisteredJobDirectories = + (JobDirectoryEntry *) repalloc(RegisteredJobDirectories, + newMax * sizeof(JobDirectoryEntry)); NumAllocatedJobDirectories = newMax; } } @@ -123,8 +126,8 @@ ResourceOwnerRememberJobDirectory(ResourceOwner owner, uint64 jobId) void ResourceOwnerForgetJobDirectory(ResourceOwner owner, uint64 jobId) { - int lastJobIndex = NumRegisteredJobDirectories - 1; - int jobIndex = 0; + int lastJobIndex = NumRegisteredJobDirectories - 1; + int jobIndex = 0; for (jobIndex = lastJobIndex; jobIndex >= 0; jobIndex--) { @@ -135,7 +138,8 @@ ResourceOwnerForgetJobDirectory(ResourceOwner owner, uint64 jobId) /* move all later entries one up */ while (jobIndex < lastJobIndex) { - RegisteredJobDirectories[jobIndex] = RegisteredJobDirectories[jobIndex + 1]; + RegisteredJobDirectories[jobIndex] = + RegisteredJobDirectories[jobIndex + 1]; jobIndex++; } NumRegisteredJobDirectories = lastJobIndex; diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index a2552d46b..e42ab3d46 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * resource_lock.c - * Locking Infrastructure for CitusDB. + * Locking Infrastructure for Citus. * * To avoid introducing a new type of locktag - that then could not be * displayed by core functionality - we reuse advisory locks. If we'd just @@ -14,9 +14,10 @@ */ #include "postgres.h" - +#include "c.h" #include "miscadmin.h" +#include "distributed/relay_utility.h" #include "distributed/resource_lock.h" #include "storage/lmgr.h" @@ -30,7 +31,7 @@ void LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; @@ -64,10 +65,12 @@ LockRelationDistributionMetadata(Oid relationId, LOCKMODE lockMode) void LockShardResource(uint64 shardId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; + AssertArg(shardId != INVALID_SHARD_ID); + SET_LOCKTAG_SHARD_RESOURCE(tag, MyDatabaseId, shardId); (void) LockAcquire(&tag, lockmode, sessionLock, dontWait); @@ -78,7 +81,7 @@ LockShardResource(uint64 shardId, LOCKMODE lockmode) void UnlockShardResource(uint64 shardId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_SHARD_RESOURCE(tag, MyDatabaseId, shardId); @@ -95,7 +98,7 @@ UnlockShardResource(uint64 shardId, LOCKMODE lockmode) void LockJobResource(uint64 jobId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; @@ -109,7 +112,7 @@ LockJobResource(uint64 jobId, LOCKMODE lockmode) void UnlockJobResource(uint64 jobId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_JOB_RESOURCE(tag, MyDatabaseId, jobId); diff --git a/src/backend/distributed/worker/task_tracker.c b/src/backend/distributed/worker/task_tracker.c index b8a6bde6a..bc67f4d7f 100644 --- a/src/backend/distributed/worker/task_tracker.c +++ b/src/backend/distributed/worker/task_tracker.c @@ -50,7 +50,7 @@ #include "utils/memutils.h" -int TaskTrackerDelay = 200; /* process sleep interval in millisecs */ +int TaskTrackerDelay = 200; /* process sleep interval in millisecs */ int MaxRunningTasksPerNode = 16; /* max number of running tasks */ int MaxTrackedTasksPerNode = 1024; /* max number of tracked tasks */ WorkerTasksSharedStateData *WorkerTasksSharedState; /* shared memory state */ @@ -76,10 +76,10 @@ static void TrackerCleanupJobSchemas(void); static void TrackerCleanupConnections(HTAB *WorkerTasksHash); static void TrackerRegisterShutDown(HTAB *WorkerTasksHash); static void TrackerDelayLoop(void); -static List *SchedulableTaskList(HTAB *WorkerTasksHash); +static List * SchedulableTaskList(HTAB *WorkerTasksHash); static WorkerTask * SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash); static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash, - bool (*CriteriaFunction) (WorkerTask *)); + bool (*CriteriaFunction)(WorkerTask *)); static bool RunningTask(WorkerTask *workerTask); static bool SchedulableTask(WorkerTask *workerTask); static int CompareTasksByTime(const void *first, const void *second); @@ -240,7 +240,7 @@ TaskTrackerMain(Datum main_arg) /* * Reload worker membership file. For now we do that in the task * tracker because that's currently the only background worker in - * CitusDB. And only background workers allow us to safely + * Citus. And only background workers allow us to safely * register a SIGHUP handler. */ LoadWorkerNodeList(WorkerListFileName); @@ -295,7 +295,7 @@ WorkerTasksHashEnter(uint64 jobId, uint32 taskId) { ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of shared memory"), - errhint("Try increasing citusdb.max_tracked_tasks_per_node."))); + errhint("Try increasing citus.max_tracked_tasks_per_node."))); } /* check that we do not have the same task assigned twice to this node */ @@ -393,7 +393,7 @@ TrackerCleanupJobSchemas(void) /* * We create cleanup tasks since we can't remove schemas within the task * tracker process. We also assign high priorities to these tasks so - * that they get scheduled before everyone else. + * that they get scheduled before everyone else. */ cleanupTask = WorkerTasksHashEnter(jobId, taskIndex); cleanupTask->assignedAt = HIGH_PRIORITY_TASK_TIME; @@ -440,7 +440,7 @@ TrackerCleanupConnections(HTAB *WorkerTasksHash) currentTask->connectionId = INVALID_CONNECTION_ID; } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } } @@ -494,8 +494,9 @@ TrackerDelayLoop(void) } } + /* ------------------------------------------------------------ - * Signal handling and shared hash initialization functions follow + * Signal handling and shared hash initialization functions follow * ------------------------------------------------------------ */ @@ -503,7 +504,7 @@ TrackerDelayLoop(void) static void TrackerSigHupHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; got_SIGHUP = true; if (MyProc != NULL) @@ -519,7 +520,7 @@ TrackerSigHupHandler(SIGNAL_ARGS) static void TrackerShutdownHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; got_SIGTERM = true; if (MyProc != NULL) @@ -579,10 +580,10 @@ TaskTrackerShmemInit(void) LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); /* allocate struct containing task tracker related shared state */ - WorkerTasksSharedState = (WorkerTasksSharedStateData *) - ShmemInitStruct("Worker Task Control", - sizeof(WorkerTasksSharedStateData), - &alreadyInitialized); + WorkerTasksSharedState = + (WorkerTasksSharedStateData *) ShmemInitStruct("Worker Task Control", + sizeof(WorkerTasksSharedStateData), + &alreadyInitialized); if (!alreadyInitialized) { @@ -607,6 +608,7 @@ TaskTrackerShmemInit(void) } } + /* ------------------------------------------------------------ * Task scheduling and management functions follow * ------------------------------------------------------------ @@ -638,7 +640,7 @@ SchedulableTaskList(HTAB *WorkerTasksHash) schedulableTaskCount = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); if (schedulableTaskCount == 0) { - return NIL; /* we do not have any new tasks to schedule */ + return NIL; /* we do not have any new tasks to schedule */ } tasksToScheduleCount = MaxRunningTasksPerNode - runningTaskCount; @@ -653,7 +655,7 @@ SchedulableTaskList(HTAB *WorkerTasksHash) for (queueIndex = 0; queueIndex < tasksToScheduleCount; queueIndex++) { WorkerTask *schedulableTask = (WorkerTask *) palloc0(sizeof(WorkerTask)); - schedulableTask->jobId = schedulableTaskQueue[queueIndex].jobId; + schedulableTask->jobId = schedulableTaskQueue[queueIndex].jobId; schedulableTask->taskId = schedulableTaskQueue[queueIndex].taskId; schedulableTaskList = lappend(schedulableTaskList, schedulableTask); @@ -681,13 +683,13 @@ SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash) uint32 queueIndex = 0; /* our priority queue size equals to the number of schedulable tasks */ - queueSize = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); + queueSize = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); if (queueSize == 0) { return NULL; } - /* allocate an array of tasks for our priority queue */ + /* allocate an array of tasks for our priority queue */ priorityQueue = (WorkerTask *) palloc0(sizeof(WorkerTask) * queueSize); /* copy tasks in the shared hash to the priority queue */ @@ -719,7 +721,7 @@ SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash) /* Counts the number of tasks that match the given criteria function. */ static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash, - bool (*CriteriaFunction) (WorkerTask *)) + bool (*CriteriaFunction)(WorkerTask *)) { HASH_SEQ_STATUS status; WorkerTask *currentTask = NULL; @@ -730,13 +732,13 @@ CountTasksMatchingCriteria(HTAB *WorkerTasksHash, currentTask = (WorkerTask *) hash_seq_search(&status); while (currentTask != NULL) { - bool matchesCriteria = (*CriteriaFunction) (currentTask); + bool matchesCriteria = (*CriteriaFunction)(currentTask); if (matchesCriteria) { taskCount++; } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } return taskCount; @@ -775,7 +777,7 @@ SchedulableTask(WorkerTask *workerTask) static int CompareTasksByTime(const void *first, const void *second) { - WorkerTask *firstTask = (WorkerTask *) first; + WorkerTask *firstTask = (WorkerTask *) first; WorkerTask *secondTask = (WorkerTask *) second; /* tasks that are assigned earlier have higher priority */ @@ -893,7 +895,7 @@ ManageWorkerTask(WorkerTask *workerTask, HTAB *WorkerTasksHash) { case TASK_ASSIGNED: { - break; /* nothing to do until the task gets scheduled */ + break; /* nothing to do until the task gets scheduled */ } case TASK_SCHEDULED: diff --git a/src/backend/distributed/worker/task_tracker_protocol.c b/src/backend/distributed/worker/task_tracker_protocol.c index 84115f441..1b52be99a 100644 --- a/src/backend/distributed/worker/task_tracker_protocol.c +++ b/src/backend/distributed/worker/task_tracker_protocol.c @@ -57,7 +57,7 @@ task_tracker_assign_task(PG_FUNCTION_ARGS) { uint64 jobId = PG_GETARG_INT64(0); uint32 taskId = PG_GETARG_UINT32(1); - text *taskCallStringText = PG_GETARG_TEXT_P(2); + text *taskCallStringText = PG_GETARG_TEXT_P(2); StringInfo jobSchemaName = JobSchemaName(jobId); bool schemaExists = false; @@ -185,7 +185,7 @@ task_tracker_cleanup_job(PG_FUNCTION_ARGS) CleanupTask(currentTask); } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } LWLockRelease(WorkerTasksSharedState->taskHashLock); @@ -308,7 +308,7 @@ CreateTask(uint64 jobId, uint32 taskId, char *taskCallString) } -/* +/* * UpdateTask updates the call string text for an already existing task. Note * that this function expects the caller to hold an exclusive lock over the * shared hash. @@ -331,7 +331,7 @@ UpdateTask(WorkerTask *workerTask, char *taskCallString) if (taskStatus == TASK_SUCCEEDED || taskStatus == TASK_CANCEL_REQUESTED || taskStatus == TASK_CANCELED) { - ; /* nothing to do */ + /* nothing to do */ } else if (taskStatus == TASK_PERMANENTLY_FAILED) { diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 0e5b68a1d..17a1058b6 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -53,11 +53,14 @@ static void ReceiveResourceCleanup(int32 connectionId, const char *filename, static void DeleteFile(const char *filename); static void FetchTableCommon(text *tableName, uint64 remoteTableSize, ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction) (const char *, uint32, StringInfo)); + bool (*FetchTableFunction)(const char *, uint32, + StringInfo)); static uint64 LocalTableSize(Oid relationId); static uint64 ExtractShardId(StringInfo tableName); -static bool FetchRegularTable(const char *nodeName, uint32 nodePort, StringInfo tableName); -static bool FetchForeignTable(const char *nodeName, uint32 nodePort, StringInfo tableName); +static bool FetchRegularTable(const char *nodeName, uint32 nodePort, + StringInfo tableName); +static bool FetchForeignTable(const char *nodeName, uint32 nodePort, + StringInfo tableName); static List * TableDDLCommandList(const char *nodeName, uint32 nodePort, StringInfo tableName); static StringInfo ForeignFilePath(const char *nodeName, uint32 nodePort, @@ -85,7 +88,7 @@ worker_fetch_partition_file(PG_FUNCTION_ARGS) uint64 jobId = PG_GETARG_INT64(0); uint32 partitionTaskId = PG_GETARG_UINT32(1); uint32 partitionFileId = PG_GETARG_UINT32(2); - uint32 upstreamTaskId = PG_GETARG_UINT32(3); + uint32 upstreamTaskId = PG_GETARG_UINT32(3); text *nodeNameText = PG_GETARG_TEXT_P(4); uint32 nodePort = PG_GETARG_UINT32(5); char *nodeName = NULL; @@ -226,7 +229,7 @@ ReceiveRegularFile(const char *nodeName, uint32 nodePort, char filename[MAXPGPATH]; int closed = -1; const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - const int fileMode = (S_IRUSR | S_IWUSR); + const int fileMode = (S_IRUSR | S_IWUSR); QueryStatus queryStatus = CLIENT_INVALID_QUERY; int32 connectionId = INVALID_CONNECTION_ID; @@ -309,7 +312,7 @@ ReceiveRegularFile(const char *nodeName, uint32 nodePort, } else if (copyStatus == CLIENT_COPY_MORE) { - ; /* remote node will continue to send more data */ + /* remote node will continue to send more data */ } else { @@ -468,7 +471,7 @@ worker_fetch_foreign_file(PG_FUNCTION_ARGS) static void FetchTableCommon(text *tableNameText, uint64 remoteTableSize, ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction) (const char *, uint32, StringInfo)) + bool (*FetchTableFunction)(const char *, uint32, StringInfo)) { StringInfo tableName = NULL; char *tableNameCString = NULL; @@ -531,7 +534,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize, if (remoteTableSize > localTableSize) { /* table is not up to date, drop the table */ - ObjectAddress tableObject = {InvalidOid, InvalidOid, 0}; + ObjectAddress tableObject = { InvalidOid, InvalidOid, 0 }; tableObject.classId = RelationRelationId; tableObject.objectId = relationId; @@ -554,7 +557,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize, char *nodeName = TextDatumGetCString(nodeNameDatum); uint32 nodePort = DatumGetUInt32(nodePortDatum); - tableFetched = (*FetchTableFunction) (nodeName, nodePort, tableName); + tableFetched = (*FetchTableFunction)(nodeName, nodePort, tableName); nodeIndex++; } @@ -994,11 +997,10 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) StringInfo remoteCopyCommand = NULL; CopyStmt *localCopyCommand = NULL; RangeVar *localTable = NULL; - uint64 copiedRowCount = 0; uint64 shardId = INVALID_SHARD_ID; bool received = false; char *quotedTableName = NULL; - const char *queryString = NULL; + StringInfo queryString = NULL; const char *schemaName = NULL; /* copy remote table's data to this node */ @@ -1010,7 +1012,7 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) * the transaction for this function commits, this lock will automatically * be released. This ensures appends to a shard happen in a serial manner. */ - shardId = ExtractShardId(shardNameString); + shardId = ExtractShardId(shardNameString); LockShardResource(shardId, AccessExclusiveLock); localFilePath = makeStringInfo(); @@ -1032,8 +1034,13 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) localTable = makeRangeVar((char *) schemaName, shardNameString->data, -1); localCopyCommand = CopyStatement(localTable, localFilePath->data); - DoCopy(localCopyCommand, queryString, &copiedRowCount); - (void) copiedRowCount; + quotedTableName = quote_qualified_identifier(schemaName, shardNameString->data); + + queryString = makeStringInfo(); + appendStringInfo(queryString, COPY_IN_COMMAND, quotedTableName, localFilePath->data); + + ProcessUtility((Node *) localCopyCommand, queryString->data, + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); /* finally delete the temporary file we created */ DeleteFile(localFilePath->data); @@ -1049,7 +1056,7 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) static bool check_log_statement(List *statementList) { - ListCell *statementCell; + ListCell *statementCell; if (log_statement == LOGSTMT_NONE) { diff --git a/src/backend/distributed/worker/worker_file_access_protocol.c b/src/backend/distributed/worker/worker_file_access_protocol.c index 04deb2881..6c9eb18bd 100644 --- a/src/backend/distributed/worker/worker_file_access_protocol.c +++ b/src/backend/distributed/worker/worker_file_access_protocol.c @@ -40,22 +40,22 @@ worker_foreign_file_path(PG_FUNCTION_ARGS) ForeignTable *foreignTable = GetForeignTable(relationId); ListCell *optionCell = NULL; - foreach(optionCell, foreignTable->options) - { - DefElem *option = (DefElem *) lfirst(optionCell); + foreach(optionCell, foreignTable->options) + { + DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; - int compareResult = strncmp(optionName, FOREIGN_FILENAME_OPTION, MAXPGPATH); - if (compareResult == 0) - { - char *optionValue = defGetString(option); - foreignFilePath = cstring_to_text(optionValue); - break; - } - } + int compareResult = strncmp(optionName, FOREIGN_FILENAME_OPTION, MAXPGPATH); + if (compareResult == 0) + { + char *optionValue = defGetString(option); + foreignFilePath = cstring_to_text(optionValue); + break; + } + } /* check that we found the filename option */ - if (foreignFilePath == NULL) + if (foreignFilePath == NULL) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errmsg("could not find filename for foreign table: \"%s\"", diff --git a/src/backend/distributed/worker/worker_merge_protocol.c b/src/backend/distributed/worker/worker_merge_protocol.c index ee829e342..92afb1fb1 100644 --- a/src/backend/distributed/worker/worker_merge_protocol.c +++ b/src/backend/distributed/worker/worker_merge_protocol.c @@ -133,7 +133,7 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) const char *createMergeTableQuery = text_to_cstring(createMergeTableQueryText); const char *createIntermediateTableQuery = - text_to_cstring(createIntermediateTableQueryText); + text_to_cstring(createIntermediateTableQueryText); StringInfo taskDirectoryName = TaskDirectoryName(jobId, taskId); StringInfo jobSchemaName = JobSchemaName(jobId); @@ -170,14 +170,14 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) if (setSearchPathResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - setSearchPathString->data))); + setSearchPathString->data))); } createMergeTableResult = SPI_exec(createMergeTableQuery, 0); if (createMergeTableResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - createMergeTableQuery))); + createMergeTableQuery))); } appendStringInfo(mergeTableName, "%s%s", intermediateTableName->data, @@ -188,7 +188,7 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) if (createIntermediateTableResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - createIntermediateTableQuery))); + createIntermediateTableQuery))); } finished = SPI_finish(); @@ -256,8 +256,8 @@ JobSchemaName(uint64 jobId) */ #ifdef HAVE_INTTYPES_H StringInfo jobSchemaName = makeStringInfo(); - appendStringInfo(jobSchemaName, "%s%0*"PRIu64, - JOB_SCHEMA_PREFIX, MIN_JOB_DIRNAME_WIDTH, jobId); + appendStringInfo(jobSchemaName, "%s%0*" PRIu64, JOB_SCHEMA_PREFIX, + MIN_JOB_DIRNAME_WIDTH, jobId); #else StringInfo jobSchemaName = makeStringInfo(); appendStringInfo(jobSchemaName, "%s%0*llu", diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index c6578295b..9ad956ed6 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -59,7 +59,7 @@ static void FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite) static void FileOutputStreamFlush(FileOutputStream file); static void FilterAndPartitionTable(const char *filterQuery, const char *columnName, Oid columnType, - uint32 (*PartitionIdFunction) (Datum, const void *), + uint32 (*PartitionIdFunction)(Datum, const void *), const void *partitionIdContext, FileOutputStream *partitionFileArray, uint32 fileCount); @@ -105,7 +105,7 @@ worker_range_partition_table(PG_FUNCTION_ARGS) uint32 taskId = PG_GETARG_UINT32(1); text *filterQueryText = PG_GETARG_TEXT_P(2); text *partitionColumnText = PG_GETARG_TEXT_P(3); - Oid partitionColumnType = PG_GETARG_OID(4); + Oid partitionColumnType = PG_GETARG_OID(4); ArrayType *splitPointObject = PG_GETARG_ARRAYTYPE_P(5); const char *filterQuery = text_to_cstring(filterQueryText); @@ -125,7 +125,7 @@ worker_range_partition_table(PG_FUNCTION_ARGS) if (splitPointType != partitionColumnType) { ereport(ERROR, (errmsg("partition column type %u and split point type %u " - "do not match", partitionColumnType, splitPointType))); + "do not match", partitionColumnType, splitPointType))); } /* use column's type information to get the comparison function */ @@ -181,7 +181,7 @@ worker_hash_partition_table(PG_FUNCTION_ARGS) uint32 taskId = PG_GETARG_UINT32(1); text *filterQueryText = PG_GETARG_TEXT_P(2); text *partitionColumnText = PG_GETARG_TEXT_P(3); - Oid partitionColumnType = PG_GETARG_OID(4); + Oid partitionColumnType = PG_GETARG_OID(4); uint32 partitionCount = PG_GETARG_UINT32(5); const char *filterQuery = text_to_cstring(filterQueryText); @@ -463,7 +463,7 @@ JobDirectoryName(uint64 jobId) */ #ifdef HAVE_INTTYPES_H StringInfo jobDirectoryName = makeStringInfo(); - appendStringInfo(jobDirectoryName, "base/%s/%s%0*"PRIu64, + appendStringInfo(jobDirectoryName, "base/%s/%s%0*" PRIu64, PG_JOB_CACHE_DIR, JOB_DIRECTORY_PREFIX, MIN_JOB_DIRNAME_WIDTH, jobId); #else @@ -726,7 +726,7 @@ FileOutputStreamFlush(FileOutputStream file) static void FilterAndPartitionTable(const char *filterQuery, const char *partitionColumnName, Oid partitionColumnType, - uint32 (*PartitionIdFunction) (Datum, const void *), + uint32 (*PartitionIdFunction)(Datum, const void *), const void *partitionIdContext, FileOutputStream *partitionFileArray, uint32 fileCount) @@ -794,7 +794,7 @@ FilterAndPartitionTable(const char *filterQuery, FileOutputStream partitionFile = { 0, 0, 0 }; StringInfo rowText = NULL; Datum partitionKey = 0; - bool partitionKeyNull = false; + bool partitionKeyNull = false; uint32 partitionId = 0; partitionKey = SPI_getbinval(row, rowDescriptor, @@ -808,7 +808,7 @@ FilterAndPartitionTable(const char *filterQuery, */ if (!partitionKeyNull) { - partitionId = (*PartitionIdFunction) (partitionKey, partitionIdContext); + partitionId = (*PartitionIdFunction)(partitionKey, partitionIdContext); } else { @@ -926,7 +926,7 @@ InitRowOutputState(void) /* initialize defaults for printing null values */ char *nullPrint = pstrdup("\\N"); - int nullPrintLen = strlen(nullPrint); + int nullPrintLen = strlen(nullPrint); char *nullPrintClient = pg_server_to_any(nullPrint, nullPrintLen, fileEncoding); /* set default text output characters */ @@ -946,7 +946,7 @@ InitRowOutputState(void) } /* set up transcoding information and default text output characters */ - if ( (fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1) ) + if ((fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1)) { rowOutputState->need_transcoding = true; } @@ -1057,7 +1057,7 @@ OutputRow(HeapTuple row, TupleDesc rowDescriptor, CopySendString(rowOutputState, rowOutputState->null_print_client); } - lastColumn = ((columnIndex+1) == columnCount); + lastColumn = ((columnIndex + 1) == columnCount); if (!lastColumn) { CopySendChar(rowOutputState, rowOutputState->delim[0]); @@ -1094,9 +1094,9 @@ OutputBinaryHeaders(FileOutputStream *partitionFileArray, uint32 fileCount) { /* Generate header for a binary copy */ const int32 zero = 0; - FileOutputStream partitionFile = {0, 0, 0}; + FileOutputStream partitionFile = { 0, 0, 0 }; PartialCopyStateData headerOutputStateData; - PartialCopyState headerOutputState = (PartialCopyState) &headerOutputStateData; + PartialCopyState headerOutputState = (PartialCopyState) & headerOutputStateData; memset(headerOutputState, 0, sizeof(PartialCopyStateData)); headerOutputState->fe_msgbuf = makeStringInfo(); @@ -1128,9 +1128,9 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) { /* Generate footer for a binary copy */ int16 negative = -1; - FileOutputStream partitionFile = {0, 0, 0}; + FileOutputStream partitionFile = { 0, 0, 0 }; PartialCopyStateData footerOutputStateData; - PartialCopyState footerOutputState = (PartialCopyState) &footerOutputStateData; + PartialCopyState footerOutputState = (PartialCopyState) & footerOutputStateData; memset(footerOutputState, 0, sizeof(PartialCopyStateData)); footerOutputState->fe_msgbuf = makeStringInfo(); @@ -1143,6 +1143,7 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) } +/* *INDENT-OFF* */ /* Append data to the copy buffer in outputState */ static void CopySendData(PartialCopyState outputState, const void *databuf, int datasize) @@ -1282,6 +1283,7 @@ CopyAttributeOutText(PartialCopyState cstate, char *string) } +/* *INDENT-ON* */ /* Helper function to send pending copy output */ static inline void CopyFlushOutput(PartialCopyState cstate, char *start, char *pointer) @@ -1359,7 +1361,7 @@ RangePartitionId(Datum partitionValue, const void *context) currentLength = currentLength - halfLength - 1; } } - + return firstIndex; } diff --git a/src/bin/csql/Makefile b/src/bin/csql/Makefile index cbfe7c01f..2c1ae79af 100644 --- a/src/bin/csql/Makefile +++ b/src/bin/csql/Makefile @@ -9,12 +9,12 @@ # #------------------------------------------------------------------------- -citusdb_subdir = src/bin/csql -citusdb_top_builddir = ../../.. +citus_subdir = src/bin/csql +citus_top_builddir = ../../.. PROGRAM = csql -PGFILEDESC = "csql - the CitusDB interactive terminal" +PGFILEDESC = "csql - the Citus interactive terminal" PGAPPICON=win32 OBJS =command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ @@ -26,7 +26,7 @@ OBJS =command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ PG_LIBS = $(libpq) -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global override CPPFLAGS += -I$(libpq_srcdir) -I$(top_srcdir)/src/bin/csql diff --git a/src/bin/csql/copy_options.c b/src/bin/csql/copy_options.c index 357ae726c..ac0c4d8eb 100644 --- a/src/bin/csql/copy_options.c +++ b/src/bin/csql/copy_options.c @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * copy_options.c * Routines for parsing copy and stage meta commands. * @@ -16,7 +16,22 @@ #include "stringutils.h" -/* Concatenates "more" onto "var", and frees the original value of *var. */ +/* *INDENT-OFF* */ +void +free_copy_options(copy_options * ptr) +{ + if (!ptr) + return; + free(ptr->before_tofrom); + free(ptr->after_tofrom); + free(ptr->file); + free(ptr->tableName); + free(ptr->columnList); + free(ptr); +} + + +/* concatenate "more" onto "var", freeing the original value of *var */ static void xstrcat(char **var, const char *more) { @@ -210,21 +225,9 @@ error: return NULL; } +/* *INDENT-ON* */ /* Frees copy options. */ -void -free_copy_options(copy_options * ptr) -{ - if (!ptr) - return; - free(ptr->before_tofrom); - free(ptr->after_tofrom); - free(ptr->file); - free(ptr->tableName); - free(ptr->columnList); - free(ptr); -} - /* * ParseStageOptions takes the given copy options, parses the additional options diff --git a/src/bin/csql/copy_options.h b/src/bin/csql/copy_options.h index 07a3aeb09..ad91fefbd 100644 --- a/src/bin/csql/copy_options.h +++ b/src/bin/csql/copy_options.h @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * copy_options.h * Shared declarations for parsing copy and stage meta-commands. The stage * meta-command borrows from copy's syntax, but does not yet support @@ -46,7 +46,7 @@ typedef struct copy_options bool psql_inout; /* true = use psql stdin/stdout */ bool from; /* true = FROM, false = TO */ - char *tableName; /* table name to stage data to */ + char *tableName; /* table name to stage data to */ char *columnList; /* optional column list used in staging */ } copy_options; diff --git a/src/bin/csql/help.c b/src/bin/csql/help.c index 10304564e..b5c2e405e 100644 --- a/src/bin/csql/help.c +++ b/src/bin/csql/help.c @@ -67,7 +67,7 @@ usage(unsigned short int pager) output = PageOutput(59, pager ? &(pset.popt.topt) : NULL); - printf(_("csql is the CitusDB interactive terminal.\n\n")); + printf(_("csql is the Citus interactive terminal.\n\n")); fprintf(output, _("Usage:\n")); printf(_(" csql [OPTION]... [DBNAME [USERNAME]]\n\n")); diff --git a/src/bin/csql/mainloop.c b/src/bin/csql/mainloop.c index 507dd5986..65070e643 100644 --- a/src/bin/csql/mainloop.c +++ b/src/bin/csql/mainloop.c @@ -200,7 +200,7 @@ MainLoop(FILE *source) (line[4] == '\0' || line[4] == ';' || isspace((unsigned char) line[4]))) { free(line); - puts(_("You are using csql, the command-line interface to CitusDB.")); + puts(_("You are using csql, the command-line interface to Citus.")); printf(_("Type: \\copyright for distribution terms\n" " \\h for help with SQL commands\n" " \\? for help with csql commands\n" diff --git a/src/bin/csql/stage.c b/src/bin/csql/stage.c index 1863b8bcb..9e5d96f7b 100644 --- a/src/bin/csql/stage.c +++ b/src/bin/csql/stage.c @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * stage.c * Helper routines to execute the csql meta-command \stage. These routines * communicate with the master and worker nodes; and create new shards and @@ -26,7 +26,8 @@ static bool FileSize(char *filename, uint64 *fileSize); static PGconn * ConnectToWorkerNode(const char *nodeName, uint32 nodePort, const char *nodeDatabase); -static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection, const char *remoteCommand, +static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection, + const char *remoteCommand, const char **parameterValues, int parameterCount); static TableMetadata * InitTableMetadata(const char *tableName); static ShardMetadata * InitShardMetadata(int shardPlacementPolicy); @@ -41,7 +42,8 @@ static uint64 GetValueUint64(const PGresult *result, int rowNumber, int columnNu static bool MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata); static bool MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata); static bool MasterGetNewShardId(ShardMetadata *shardMetadata); -static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy); +static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata, + int shardPlacementPolicy); static bool MasterInsertShardRow(uint32 logicalRelid, char storageType, const ShardMetadata *shardMetadata); static bool MasterInsertPlacementRows(const ShardMetadata *shardMetadata); @@ -62,7 +64,8 @@ static bool ApplyShardDDLCommand(PGconn *workerNode, uint64 shardId, const char static bool TransmitTableData(PGconn *workerNode, uint64 shardId, uint64 shardMaxSize, copy_options *stageOptions, uint64 currentFileOffset, uint64 *nextFileOffset); -static bool TransmitFile(PGconn *workerNode, const char *localPath, const char *remotePath); +static bool TransmitFile(PGconn *workerNode, const char *localPath, + const char *remotePath); static bool FileStreamOK(const copy_options *stageOptions); static PQExpBuffer CreateCopyQueryString(const char *tableName, const char *columnList, const char *afterToFrom); @@ -166,7 +169,7 @@ DoStageData(const char *stageCommand) if (partitionMethod == DISTRIBUTE_BY_HASH) { psql_error("\\stage: staging data into hash partitioned tables is not " - "supported\n"); + "supported\n"); free_copy_options(stageOptions); FreeTableMetadata(tableMetadata); @@ -179,7 +182,7 @@ DoStageData(const char *stageCommand) bool tableOptionsOK = ColumnarTableOptionsOK(tableMetadata->logicalRelid); if (!tableOptionsOK) { - return false; /* error message already displayed */ + return false; /* error message already displayed */ } } @@ -225,7 +228,7 @@ DoStageData(const char *stageCommand) */ FreeCommonStageData(stageOptions, tableMetadata, shardMetadataList); - return false; /* abort immediately */ + return false; /* abort immediately */ } /* save allocated shard metadata */ @@ -245,7 +248,7 @@ DoStageData(const char *stageCommand) */ for (nodeIndex = 0; nodeIndex < shardMetadata->nodeCount; nodeIndex++) { - char *remoteNodeName = shardMetadata->nodeNameList[nodeIndex]; + char *remoteNodeName = shardMetadata->nodeNameList[nodeIndex]; uint32 remoteNodePort = shardMetadata->nodePortList[nodeIndex]; PGconn *remoteNode = NULL; @@ -341,7 +344,6 @@ DoStageData(const char *stageCommand) /* update current file offset */ currentFileOffset = nextFileOffset; - } /* while more file data left for sharding */ /* @@ -390,9 +392,9 @@ ConnectToWorkerNode(const char *nodeName, uint32 nodePort, const char *nodeDatab char nodePortString[MAXPGPATH]; char connInfoString[MAXPGPATH]; - /* transcribe port number and connection info to their string values */ + /* transcribe port number and connection info to their string values */ snprintf(nodePortString, MAXPGPATH, "%u", nodePort); - snprintf(connInfoString, MAXPGPATH, CONN_INFO_TEMPLATE, + snprintf(connInfoString, MAXPGPATH, CONN_INFO_TEMPLATE, nodeDatabase, CLIENT_CONNECT_TIMEOUT); workerNode = PQsetdb(nodeName, nodePortString, nodeOptions, nodeTty, connInfoString); @@ -421,16 +423,16 @@ ExecuteRemoteCommand(PGconn *remoteConnection, const char *remoteCommand, { PGresult *result = NULL; - const Oid *parameterType = NULL; /* let the backend deduce type */ + const Oid *parameterType = NULL; /* let the backend deduce type */ const int *parameterLength = NULL; /* text params do not need length */ const int *parameterFormat = NULL; /* text params have Null by default */ - const int resultFormat = 0; /* ask for results in text format */ + const int resultFormat = 0; /* ask for results in text format */ result = PQexecParams(remoteConnection, remoteCommand, parameterCount, parameterType, parameterValues, parameterLength, parameterFormat, resultFormat); - if (PQresultStatus(result) != PGRES_COMMAND_OK && + if (PQresultStatus(result) != PGRES_COMMAND_OK && PQresultStatus(result) != PGRES_TUPLES_OK) { psql_error("remote command \"%s\" failed with %s", @@ -488,7 +490,7 @@ FreeTableMetadata(TableMetadata *tableMetadata) for (eventIndex = 0; eventIndex < eventCount; eventIndex++) { char *ddlEvent = tableMetadata->ddlEventList[eventIndex]; - + free(ddlEvent); ddlEvent = NULL; } @@ -552,7 +554,7 @@ FreeShardMetadata(ShardMetadata *shardMetadata) for (nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { char *nodeName = shardMetadata->nodeNameList[nodeIndex]; - + free(nodeName); nodeName = NULL; } @@ -655,7 +657,7 @@ ExtendTablename(const char *baseTablename, uint64 shardId) { char *extendedTablename = (char *) pg_malloc0(NAMEDATALEN); - snprintf(extendedTablename, NAMEDATALEN, "%s%c" UINT64_FORMAT, + snprintf(extendedTablename, NAMEDATALEN, "%s%c" UINT64_FORMAT, baseTablename, SHARD_NAME_SEPARATOR, shardId); return extendedTablename; @@ -678,7 +680,7 @@ GetValueUint64(const PGresult *result, int rowNumber, int columnNumber) errno = 0; value = strtoull(valueString, &valueStringEnd, 0); - + if (errno != 0 || (*valueStringEnd) != '\0') { return INVALID_UINT64; @@ -716,7 +718,7 @@ MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata) char *tableStorageType = NULL; char *partitionMethod = NULL; char *partitionKey = NULL; - int partitionKeyLength = 0; + int partitionKeyLength = 0; uint64 logicalRelid = 0; uint64 shardReplicaCount = 0; uint64 shardMaxSize = 0; @@ -727,7 +729,7 @@ MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata) parameterValue, parameterCount); if (result == NULL) { - return false; /* error message already displayed */ + return false; /* error message already displayed */ } /* find column numbers associated with column names */ @@ -798,13 +800,13 @@ MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata) int ddlEventIndex = 0; /* fetch DDL events needed for table creation */ - result = ExecuteRemoteCommand(masterNode, remoteCommand, + result = ExecuteRemoteCommand(masterNode, remoteCommand, parameterValue, parameterCount); if (result == NULL) { return false; } - + /* check that we have at least one DDL event */ ddlEventCount = PQntuples(result); if (ddlEventCount <= 0) @@ -825,7 +827,7 @@ MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata) { char *ddlEvent = NULL; char *ddlEventValue = PQgetvalue(result, ddlEventIndex, 0); - int ddlEventLength = PQgetlength(result, ddlEventIndex, 0); + int ddlEventLength = PQgetlength(result, ddlEventIndex, 0); if (ddlEventLength <= 0) { @@ -866,7 +868,7 @@ MasterGetNewShardId(ShardMetadata *shardMetadata) uint64 shardId = 0; /* fetch unique shardId for shard to be created */ - result = ExecuteRemoteCommand(masterNode, remoteCommand, + result = ExecuteRemoteCommand(masterNode, remoteCommand, parameterValue, parameterCount); if (result == NULL) { @@ -877,7 +879,7 @@ MasterGetNewShardId(ShardMetadata *shardMetadata) shardId = GetValueUint64(result, 0, 0); if (shardId == INVALID_UINT64) { - psql_error("remote command \"%s\" failed with invalid shardId\n", + psql_error("remote command \"%s\" failed with invalid shardId\n", remoteCommand); PQclear(result); @@ -996,11 +998,11 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) /* walk over fetched node name/port list, and assign them to metadata */ for (nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { - char *nodeName = NULL; + char *nodeName = NULL; uint64 nodePort = 0; - char *nodeNameValue = PQgetvalue(result, nodeIndex, nodeNameIndex); - int nodeNameLength = PQgetlength(result, nodeIndex, nodeNameIndex); + char *nodeNameValue = PQgetvalue(result, nodeIndex, nodeNameIndex); + int nodeNameLength = PQgetlength(result, nodeIndex, nodeNameIndex); if (nodeNameLength <= 0) { @@ -1014,7 +1016,7 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) /* deep copy node name and assign to metadata */ nodeName = (char *) pg_malloc0(nodeNameLength + 1); strncpy(nodeName, nodeNameValue, nodeNameLength + 1); - + shardMetadata->nodeNameList[nodeIndex] = nodeName; /* convert port value string to 64-bit integer, and assign to metadata */ @@ -1024,7 +1026,7 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) psql_error("remote command \"%s\" failed to fetch valid port number\n", remoteCommand); PQclear(result); - + return false; } @@ -1107,12 +1109,12 @@ MasterInsertPlacementRows(const ShardMetadata *shardMetadata) bool staged = shardMetadata->nodeStageList[nodeIndex]; if (staged) { - char *nodeName = shardMetadata->nodeNameList[nodeIndex]; + char *nodeName = shardMetadata->nodeNameList[nodeIndex]; uint32 nodePort = shardMetadata->nodePortList[nodeIndex]; /* convert parameter to its string representation */ snprintf(nodePortString, NAMEDATALEN, "%u", nodePort); - + parameterValue[3] = nodeName; parameterValue[4] = nodePortString; @@ -1136,7 +1138,7 @@ MasterInsertPlacementRows(const ShardMetadata *shardMetadata) * staged to worker nodes. The function executes shard metadata insert commands * within a single transaction so that either all or none of the metadata are * finalized. On success, the function commits the transaction and returns true. - * On failure, the function rolls back the transaction and returns false. + * On failure, the function rolls back the transaction and returns false. */ static bool MasterInsertShardMetadata(uint32 logicalRelid, char storageType, @@ -1199,7 +1201,7 @@ IssueTransactionCommand(PGconn *connection, const char *command) return false; } - + PQclear(result); return true; } @@ -1729,7 +1731,7 @@ ShardColumnarTableSize(PGconn *workerNode, const char *tablename, uint64 shardId * failure, the function returns false. */ static bool -ShardMinMaxValues(PGconn *workerNode, const char *tablename, +ShardMinMaxValues(PGconn *workerNode, const char *tablename, const char *partitionKey, ShardMetadata *shardMetadata) { const int MinValueIndex = 0; @@ -1744,7 +1746,7 @@ ShardMinMaxValues(PGconn *workerNode, const char *tablename, int maxValueLength = 0; extendedTablename = ExtendTablename(tablename, shardMetadata->shardId); - snprintf(remoteCommand, MAXPGPATH, SHARD_MIN_MAX_COMMAND, + snprintf(remoteCommand, MAXPGPATH, SHARD_MIN_MAX_COMMAND, partitionKey, partitionKey, extendedTablename); result = PQexec(workerNode, remoteCommand); diff --git a/src/bin/csql/stage.h b/src/bin/csql/stage.h index 01575f886..1f8a9ccff 100644 --- a/src/bin/csql/stage.h +++ b/src/bin/csql/stage.h @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * stage.h * Declarations for the csql meta-command \stage. These declarations define a * protocol for the client to communicate to the master and worker nodes. @@ -30,42 +30,44 @@ #define ROLLBACK_COMMAND "ROLLBACK" /* Names of remote function calls to execute on the master. */ -#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)" +#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)" #define MASTER_GET_TABLE_DDL_EVENTS "SELECT * FROM master_get_table_ddl_events($1::text)" -#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()" -#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES "SELECT * FROM \ - master_get_local_first_candidate_nodes()" -#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES "SELECT * FROM \ - master_get_round_robin_candidate_nodes($1::int8)" +#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()" +#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES \ + "SELECT * FROM master_get_local_first_candidate_nodes()" +#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES \ + "SELECT * FROM master_get_round_robin_candidate_nodes($1::int8)" -#define MASTER_INSERT_SHARD_ROW "INSERT INTO pg_dist_shard \ - (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES \ - ($1::oid, $2::int8, $3::char, $4::text, $5::text)" -#define MASTER_INSERT_PLACEMENT_ROW "INSERT INTO pg_dist_shard_placement \ - (shardid, shardstate, shardlength, nodename, nodeport) VALUES \ - ($1::int8, $2::int4, $3::int8, $4::text, $5::int4)" +#define MASTER_INSERT_SHARD_ROW \ + "INSERT INTO pg_dist_shard " \ + "(logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES " \ + "($1::oid, $2::int8, $3::char, $4::text, $5::text)" +#define MASTER_INSERT_PLACEMENT_ROW \ + "INSERT INTO pg_dist_shard_placement " \ + "(shardid, shardstate, shardlength, nodename, nodeport) VALUES " \ + "($1::int8, $2::int4, $3::int8, $4::text, $5::int4)" /* Column names used to identify response fields as returned from the master. */ -#define LOGICAL_RELID_FIELD "logical_relid" -#define PART_STORAGE_TYPE_FIELD "part_storage_type" -#define PART_METHOD_FIELD "part_method" -#define PART_KEY_FIELD "part_key" -#define PART_REPLICA_COUNT_FIELD "part_replica_count" -#define PART_MAX_SIZE_FIELD "part_max_size" -#define PART_PLACEMENT_POLICY_FIELD "part_placement_policy" -#define NODE_NAME_FIELD "node_name" -#define NODE_PORT_FIELD "node_port" +#define LOGICAL_RELID_FIELD "logical_relid" +#define PART_STORAGE_TYPE_FIELD "part_storage_type" +#define PART_METHOD_FIELD "part_method" +#define PART_KEY_FIELD "part_key" +#define PART_REPLICA_COUNT_FIELD "part_replica_count" +#define PART_MAX_SIZE_FIELD "part_max_size" +#define PART_PLACEMENT_POLICY_FIELD "part_placement_policy" +#define NODE_NAME_FIELD "node_name" +#define NODE_PORT_FIELD "node_port" /* the tablename in the overloaded COPY statement is the to-be-transferred file */ #define TRANSMIT_REGULAR_COMMAND "COPY \"%s\" FROM STDIN WITH (format 'transmit')" -#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s" +#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s" #define SHARD_TABLE_SIZE_COMMAND "SELECT pg_table_size('%s')" #define SET_FOREIGN_TABLE_FILENAME "ALTER FOREIGN TABLE %s OPTIONS (SET filename '%s')" -#define GET_COLUMNAR_TABLE_FILENAME_OPTION "SELECT * FROM \ - (SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table \ - WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';" -#define APPLY_SHARD_DDL_COMMAND "SELECT * FROM worker_apply_shard_ddl_command \ - ($1::int8, $2::text)" +#define GET_COLUMNAR_TABLE_FILENAME_OPTION \ + "SELECT * FROM (SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table " \ + "WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';" +#define APPLY_SHARD_DDL_COMMAND \ + "SELECT * FROM worker_apply_shard_ddl_command ($1::int8, $2::text)" #define REMOTE_FILE_SIZE_COMMAND "SELECT size FROM pg_stat_file('%s')" #define SHARD_COLUMNAR_TABLE_SIZE_COMMAND "SELECT cstore_table_size('%s')" @@ -90,17 +92,16 @@ */ typedef struct TableMetadata { - uint32 logicalRelid; /* table's relationId on the master */ - char tableStorageType; /* relay file, foreign table, or table */ - char partitionMethod; /* table's partition method */ - char *partitionKey; /* partition key expression */ - uint32 shardReplicaCount; /* shard replication factor */ - uint64 shardMaxSize; /* create new shard when shard reaches max size */ + uint32 logicalRelid; /* table's relationId on the master */ + char tableStorageType; /* relay file, foreign table, or table */ + char partitionMethod; /* table's partition method */ + char *partitionKey; /* partition key expression */ + uint32 shardReplicaCount; /* shard replication factor */ + uint64 shardMaxSize; /* create new shard when shard reaches max size */ uint32 shardPlacementPolicy; /* policy to use when choosing nodes to place shards */ char **ddlEventList; /* DDL statements used for creating new shard */ uint32 ddlEventCount; /* DDL statement count; statement list size */ - } TableMetadata; @@ -112,17 +113,16 @@ typedef struct TableMetadata */ typedef struct ShardMetadata { - uint64 shardId; /* global shardId; created on the master node */ + uint64 shardId; /* global shardId; created on the master node */ - char **nodeNameList; /* candidate node name list for shard uploading */ + char **nodeNameList; /* candidate node name list for shard uploading */ uint32 *nodePortList; /* candidate node port list for shard uploading */ - uint32 nodeCount; /* candidate node count; node list size */ - bool *nodeStageList; /* shard uploaded to corresponding candidate node? */ + uint32 nodeCount; /* candidate node count; node list size */ + bool *nodeStageList; /* shard uploaded to corresponding candidate node? */ char *shardMinValue; /* partition key's minimum value in shard */ char *shardMaxValue; /* partition key's maximum value in shard */ - uint64 shardSize; /* shard size; updated during staging */ - + uint64 shardSize; /* shard size; updated during staging */ } ShardMetadata; diff --git a/src/include/.gitignore b/src/include/.gitignore index 30f297ab7..5207873bf 100644 --- a/src/include/.gitignore +++ b/src/include/.gitignore @@ -1,4 +1,4 @@ /stamp-h /stamp-ext-h -/citusdb_config.h -/citusdb_config.h.in~ +/citus_config.h +/citus_config.h.in~ diff --git a/src/include/citusdb_config.h.in b/src/include/citus_config.h.in similarity index 71% rename from src/include/citusdb_config.h.in rename to src/include/citus_config.h.in index 61ced741c..33da446e7 100644 --- a/src/include/citusdb_config.h.in +++ b/src/include/citus_config.h.in @@ -1,9 +1,9 @@ -/* src/include/citusdb_config.h.in. Generated from configure.in by autoheader. */ +/* src/include/citus_config.h.in. Generated from configure.in by autoheader. */ /* - * citusdb_config.h.in is generated by autoconf/autoheader and - * converted into citusdb_config.h by configure. Include when code needs to + * citus_config.h.in is generated by autoconf/autoheader and + * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! diff --git a/src/include/distributed/citus_nodefuncs.h b/src/include/distributed/citus_nodefuncs.h index cd273d587..e0c8373e5 100644 --- a/src/include/distributed/citus_nodefuncs.h +++ b/src/include/distributed/citus_nodefuncs.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_nodefuncs.h - * Node (de-)serialization support for CitusDB. + * Node (de-)serialization support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * diff --git a/src/include/distributed/citus_nodes.h b/src/include/distributed/citus_nodes.h index 05a536cf0..3bab073b1 100644 --- a/src/include/distributed/citus_nodes.h +++ b/src/include/distributed/citus_nodes.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_nodes.h - * Additional node types, and related infrastructure, for CitusDB. + * Additional node types, and related infrastructure, for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * @@ -52,7 +52,7 @@ typedef enum CitusNodeTag #define CitusIsA(nodeptr,_type_) (CitusNodeTag(nodeptr) == T_##_type_) -/* CitusDB variant of newNode(), don't use directly. */ +/* Citus variant of newNode(), don't use directly. */ #define CitusNewNode(size, tag) \ ({ Node *_result; \ AssertMacro((size) >= sizeof(Node)); /* need the tag, at least */ \ @@ -63,7 +63,7 @@ typedef enum CitusNodeTag /* - * CitusMakeNode is CitusDB variant of makeNode(). Use it to create nodes of + * CitusMakeNode is Citus variant of makeNode(). Use it to create nodes of * the types listed in the CitusNodeTag enum and plain NodeTag. Initializes * memory, besides the node tag, to 0. */ diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 0f2402748..e53b1c34d 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -1,8 +1,8 @@ /*------------------------------------------------------------------------- * * citus_ruleutils.h - * CitusDB ruleutils wrapper functions and exported PostgreSQL ruleutils - * functions. + * Citus ruleutils wrapper functions and exported PostgreSQL ruleutils + * functions. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -15,17 +15,18 @@ #include "nodes/parsenodes.h" -/* Function declarations for version independent CitusDB ruleutils wrapper functions */ -extern char *pg_get_extensiondef_string(Oid tableRelationId); -extern char *pg_get_serverdef_string(Oid tableRelationId); -extern char *pg_get_tableschemadef_string(Oid tableRelationId); -extern char *pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); -extern char *pg_get_indexclusterdef_string(Oid indexRelationId); +/* Function declarations for version independent Citus ruleutils wrapper functions */ +extern char * pg_get_extensiondef_string(Oid tableRelationId); +extern char * pg_get_serverdef_string(Oid tableRelationId); +extern char * pg_get_tableschemadef_string(Oid tableRelationId); +extern char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); +extern char * pg_get_indexclusterdef_string(Oid indexRelationId); /* Function declarations for version dependent PostgreSQL ruleutils functions */ -extern void pg_get_query_def(Query *query, StringInfo buffer); -extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer); -extern char *generate_relation_name(Oid relid, List *namespaces); +extern void pg_get_query_def(Query *query, StringInfo buffer); +extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo + buffer); +extern char * generate_relation_name(Oid relid, List *namespaces); #endif /* CITUS_RULEUTILS_H */ diff --git a/src/include/distributed/connection_cache.h b/src/include/distributed/connection_cache.h index 2a1f997d6..45f61742b 100644 --- a/src/include/distributed/connection_cache.h +++ b/src/include/distributed/connection_cache.h @@ -51,7 +51,7 @@ typedef struct NodeConnectionEntry /* function declarations for obtaining and using a connection */ -extern PGconn * GetConnection(char *nodeName, int32 nodePort); +extern PGconn * GetOrEstablishConnection(char *nodeName, int32 nodePort); extern void PurgeConnection(PGconn *connection); extern void ReportRemoteError(PGconn *connection, PGresult *result); diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index 3e9cd3917..62b4ec6f1 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -9,8 +9,8 @@ *------------------------------------------------------------------------- */ -#ifndef CITUSDB_LISTUTILS_H -#define CITUSDB_LISTUTILS_H +#ifndef CITUS_LISTUTILS_H +#define CITUS_LISTUTILS_H #include "postgres.h" #include "c.h" @@ -23,4 +23,4 @@ extern List * SortList(List *pointerList, int (*ComparisonFunction)(const void *, const void *)); -#endif /* CITUSDB_LISTUTILS_H */ +#endif /* CITUS_LISTUTILS_H */ diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index b3cfb6548..4fb377571 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -30,15 +30,14 @@ typedef struct ShardInterval CitusNodeTag type; Oid relationId; char storageType; - Oid valueTypeId; /* min/max value datum's typeId */ - int valueTypeLen; /* min/max value datum's typelen */ - bool valueByVal; /* min/max value datum's byval */ + Oid valueTypeId; /* min/max value datum's typeId */ + int valueTypeLen; /* min/max value datum's typelen */ + bool valueByVal; /* min/max value datum's byval */ bool minValueExists; bool maxValueExists; - Datum minValue; /* a shard's typed min value datum */ - Datum maxValue; /* a shard's typed max value datum */ + Datum minValue; /* a shard's typed min value datum */ + Datum maxValue; /* a shard's typed max value datum */ uint64 shardId; - } ShardInterval; @@ -46,13 +45,12 @@ typedef struct ShardInterval typedef struct ShardPlacement { CitusNodeTag type; - Oid tupleOid; /* unique oid that implies this row's insertion order */ + Oid tupleOid; /* unique oid that implies this row's insertion order */ uint64 shardId; uint64 shardLength; RelayFileState shardState; char *nodeName; uint32 nodePort; - } ShardPlacement; diff --git a/src/include/distributed/master_protocol.h b/src/include/distributed/master_protocol.h index ea40538d3..28964427c 100644 --- a/src/include/distributed/master_protocol.h +++ b/src/include/distributed/master_protocol.h @@ -49,13 +49,14 @@ #define SHARDID_SEQUENCE_NAME "pg_dist_shardid_seq" /* Remote call definitions to help with data staging and deletion */ -#define WORKER_APPLY_SHARD_DDL_COMMAND "SELECT worker_apply_shard_ddl_command \ - ("UINT64_FORMAT", %s)" -#define WORKER_APPEND_TABLE_TO_SHARD "SELECT worker_append_table_to_shard \ - (%s, %s, %s, %u)" +#define WORKER_APPLY_SHARD_DDL_COMMAND \ + "SELECT worker_apply_shard_ddl_command (" UINT64_FORMAT ", %s)" +#define WORKER_APPEND_TABLE_TO_SHARD \ + "SELECT worker_append_table_to_shard (%s, %s, %s, %u)" #define SHARD_MIN_VALUE_QUERY "SELECT min(%s) FROM %s" #define SHARD_MAX_VALUE_QUERY "SELECT max(%s) FROM %s" #define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size('%s')" +#define SHARD_CSTORE_TABLE_SIZE_QUERY "SELECT cstore_table_size('%s')" #define DROP_REGULAR_TABLE_COMMAND "DROP TABLE IF EXISTS %s" #define DROP_FOREIGN_TABLE_COMMAND "DROP FOREIGN TABLE IF EXISTS %s" #define CREATE_SCHEMA_COMMAND "CREATE SCHEMA IF NOT EXISTS %s" @@ -67,7 +68,6 @@ typedef enum SHARD_PLACEMENT_INVALID_FIRST = 0, SHARD_PLACEMENT_LOCAL_NODE_FIRST = 1, SHARD_PLACEMENT_ROUND_ROBIN = 2 - } ShardPlacementPolicyType; @@ -83,8 +83,8 @@ extern Oid ResolveRelationId(text *relationName); extern List * GetTableDDLEvents(Oid relationId); extern void CheckDistributedTable(Oid relationId); extern void CreateShardPlacements(int64 shardId, List *ddlEventList, - List *workerNodeList, int workerStartIndex, - int replicationFactor); + List *workerNodeList, int workerStartIndex, + int replicationFactor); /* Function declarations for generating metadata for shard creation */ extern Datum master_get_table_metadata(PG_FUNCTION_ARGS); diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index 7c088832b..66f0c33ca 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * metadata_cache.h - * Executor support for CitusDB. + * Executor support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * @@ -47,7 +47,7 @@ extern ShardInterval * LoadShardInterval(uint64 shardId); extern DistTableCacheEntry * DistributedTableCacheEntry(Oid distributedRelationId); extern void CitusInvalidateRelcacheByRelid(Oid relationId); -extern bool CitusDBHasBeenLoaded(void); +extern bool CitusHasBeenLoaded(void); /* relation oids */ extern Oid DistPartitionRelationId(void); diff --git a/src/include/distributed/modify_planner.h b/src/include/distributed/modify_planner.h index 3e52cfa5a..bd6df4755 100644 --- a/src/include/distributed/modify_planner.h +++ b/src/include/distributed/modify_planner.h @@ -24,6 +24,7 @@ #define INVALID_TASK_ID 0 #if (PG_VERSION_NUM >= 90500) + /* reserved alias name for UPSERTs */ #define UPSERT_ALIAS "citus_table_alias" #endif diff --git a/src/include/distributed/multi_client_executor.h b/src/include/distributed/multi_client_executor.h index 470b53673..0f1203031 100644 --- a/src/include/distributed/multi_client_executor.h +++ b/src/include/distributed/multi_client_executor.h @@ -15,21 +15,20 @@ #define MULTI_CLIENT_EXECUTOR_H -#define INVALID_CONNECTION_ID -1 /* identifies an invalid connection */ -#define CLIENT_CONNECT_TIMEOUT 5 /* connection timeout in seconds */ +#define INVALID_CONNECTION_ID -1 /* identifies an invalid connection */ +#define CLIENT_CONNECT_TIMEOUT 5 /* connection timeout in seconds */ #define MAX_CONNECTION_COUNT 2048 /* simultaneous client connection count */ -#define STRING_BUFFER_SIZE 1024 /* buffer size for character arrays */ +#define STRING_BUFFER_SIZE 1024 /* buffer size for character arrays */ #define CONN_INFO_TEMPLATE "host=%s port=%u dbname=%s connect_timeout=%u" /* Enumeration to track one client connection's status */ typedef enum { - CLIENT_INVALID_CONNECT = 0, - CLIENT_CONNECTION_BAD = 1, - CLIENT_CONNECTION_BUSY = 2, + CLIENT_INVALID_CONNECT = 0, + CLIENT_CONNECTION_BAD = 1, + CLIENT_CONNECTION_BUSY = 2, CLIENT_CONNECTION_READY = 3 - } ConnectStatus; @@ -38,9 +37,8 @@ typedef enum { CLIENT_INVALID_RESULT_STATUS = 0, CLIENT_RESULT_UNAVAILABLE = 1, - CLIENT_RESULT_BUSY = 2, + CLIENT_RESULT_BUSY = 2, CLIENT_RESULT_READY = 3 - } ResultStatus; @@ -48,10 +46,9 @@ typedef enum typedef enum { CLIENT_INVALID_QUERY = 0, - CLIENT_QUERY_FAILED = 1, + CLIENT_QUERY_FAILED = 1, CLIENT_QUERY_DONE = 2, CLIENT_QUERY_COPY = 3 - } QueryStatus; @@ -59,21 +56,19 @@ typedef enum typedef enum { CLIENT_INVALID_COPY = 0, - CLIENT_COPY_MORE = 1, - CLIENT_COPY_FAILED = 2, - CLIENT_COPY_DONE = 3 - + CLIENT_COPY_MORE = 1, + CLIENT_COPY_FAILED = 2, + CLIENT_COPY_DONE = 3 } CopyStatus; /* Enumeration to track the status of a query in a batch on the client */ typedef enum { - CLIENT_INVALID_BATCH_QUERY = 0, - CLIENT_BATCH_QUERY_FAILED = 1, + CLIENT_INVALID_BATCH_QUERY = 0, + CLIENT_BATCH_QUERY_FAILED = 1, CLIENT_BATCH_QUERY_CONTINUE = 2, - CLIENT_BATCH_QUERY_DONE = 3 - + CLIENT_BATCH_QUERY_DONE = 3 } BatchQueryStatus; diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h index bcf22fe2e..f09c44402 100644 --- a/src/include/distributed/multi_executor.h +++ b/src/include/distributed/multi_executor.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_executor.h - * Executor support for CitusDB. + * Executor support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -14,12 +14,12 @@ #include "nodes/parsenodes.h" /* signal currently executed statement is a master select statement or router execution */ -#define EXEC_FLAG_CITUS_MASTER_SELECT 0x100 -#define EXEC_FLAG_CITUS_ROUTER_EXECUTOR 0x200 +#define EXEC_FLAG_CITUS_MASTER_SELECT 0x100 +#define EXEC_FLAG_CITUS_ROUTER_EXECUTOR 0x200 extern void multi_ExecutorStart(QueryDesc *queryDesc, int eflags); extern void multi_ExecutorRun(QueryDesc *queryDesc, - ScanDirection direction, long count); + ScanDirection direction, long count); extern void multi_ExecutorFinish(QueryDesc *queryDesc); extern void multi_ExecutorEnd(QueryDesc *queryDesc); diff --git a/src/include/distributed/multi_explain.h b/src/include/distributed/multi_explain.h index 71ed52948..95912a38c 100644 --- a/src/include/distributed/multi_explain.h +++ b/src/include/distributed/multi_explain.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_explain.h - * Explain support for CitusDB. + * Explain support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_join_order.h b/src/include/distributed/multi_join_order.h index cccde6917..56ede010b 100644 --- a/src/include/distributed/multi_join_order.h +++ b/src/include/distributed/multi_join_order.h @@ -29,7 +29,7 @@ typedef enum JoinRuleType { JOIN_RULE_INVALID_FIRST = 0, BROADCAST_JOIN = 1, - LOCAL_PARTITION_JOIN = 2, + LOCAL_PARTITION_JOIN = 2, SINGLE_PARTITION_JOIN = 3, DUAL_PARTITION_JOIN = 4, CARTESIAN_PRODUCT = 5, @@ -40,7 +40,6 @@ typedef enum JoinRuleType * RuleNameArray. */ JOIN_RULE_LAST - } JoinRuleType; @@ -53,7 +52,6 @@ typedef struct TableEntry { Oid relationId; uint32 rangeTableId; - } TableEntry; @@ -65,14 +63,13 @@ typedef struct TableEntry */ typedef struct JoinOrderNode { - TableEntry *tableEntry; /* this node's relation and range table id */ - JoinRuleType joinRuleType; /* not relevant for the first table */ - JoinType joinType; /* not relevant for the first table */ - Var *partitionColumn; /* not relevant for the first table */ + TableEntry *tableEntry; /* this node's relation and range table id */ + JoinRuleType joinRuleType; /* not relevant for the first table */ + JoinType joinType; /* not relevant for the first table */ + Var *partitionColumn; /* not relevant for the first table */ char partitionMethod; - List *joinClauseList; /* not relevant for the first table */ + List *joinClauseList; /* not relevant for the first table */ List *shardIntervalList; - } JoinOrderNode; diff --git a/src/include/distributed/multi_logical_optimizer.h b/src/include/distributed/multi_logical_optimizer.h index f8c145751..4efc1bfed 100644 --- a/src/include/distributed/multi_logical_optimizer.h +++ b/src/include/distributed/multi_logical_optimizer.h @@ -44,7 +44,7 @@ * * Please note that the order of values in this enumeration is tied to the order * of elements in the following AggregateNames array. This order needs to be - * preserved. + * preserved. */ typedef enum { @@ -55,7 +55,6 @@ typedef enum AGGREGATE_SUM = 4, AGGREGATE_COUNT = 5, AGGREGATE_ARRAY_AGG = 6 - } AggregateType; @@ -69,7 +68,6 @@ typedef enum PUSH_DOWN_VALID = 1, PUSH_DOWN_NOT_VALID = 2, PUSH_DOWN_SPECIAL_CONDITIONS = 3 - } PushDownStatus; @@ -82,7 +80,6 @@ typedef enum PULL_UP_INVALID_FIRST = 0, PULL_UP_VALID = 1, PULL_UP_NOT_VALID = 2 - } PullUpStatus; @@ -97,8 +94,10 @@ typedef enum * Please note that the order of elements in this array is tied to the order of * values in the preceding AggregateType enum. This order needs to be preserved. */ -static const char * const AggregateNames[] = { "invalid", "avg", "min", "max", - "sum", "count", "array_agg" }; +static const char *const AggregateNames[] = { + "invalid", "avg", "min", "max", "sum", + "count", "array_agg" +}; /* Config variable managed via guc.c */ diff --git a/src/include/distributed/multi_logical_planner.h b/src/include/distributed/multi_logical_planner.h index 15b0cba07..40ee39c70 100644 --- a/src/include/distributed/multi_logical_planner.h +++ b/src/include/distributed/multi_logical_planner.h @@ -40,8 +40,8 @@ typedef struct MultiNode CitusNodeTag type; struct MultiNode *parentNode; - /* child node(s) are defined in unary and binary nodes */ + /* child node(s) are defined in unary and binary nodes */ } MultiNode; @@ -51,7 +51,6 @@ typedef struct MultiUnaryNode MultiNode node; struct MultiNode *childNode; - } MultiUnaryNode; @@ -62,7 +61,6 @@ typedef struct MultiBinaryNode struct MultiNode *leftChildNode; struct MultiNode *rightChildNode; - } MultiBinaryNode; @@ -73,7 +71,6 @@ typedef struct MultiBinaryNode typedef struct MultiTreeRoot { MultiUnaryNode unaryNode; - } MultiTreeRoot; @@ -91,7 +88,6 @@ typedef struct MultiTable Alias *alias; Alias *referenceNames; Query *subquery; /* this field is only valid for non-relation subquery types */ - } MultiTable; @@ -100,7 +96,6 @@ typedef struct MultiProject { MultiUnaryNode unaryNode; List *columnList; - } MultiProject; @@ -112,7 +107,6 @@ typedef struct MultiProject typedef struct MultiCollect { MultiUnaryNode unaryNode; - } MultiCollect; @@ -125,7 +119,6 @@ typedef struct MultiSelect { MultiUnaryNode unaryNode; List *selectClauseList; - } MultiSelect; @@ -140,7 +133,6 @@ typedef struct MultiJoin List *joinClauseList; JoinRuleType joinRuleType; JoinType joinType; - } MultiJoin; @@ -150,7 +142,6 @@ typedef struct MultiPartition MultiUnaryNode unaryNode; Var *partitionColumn; uint32 splitPointTableId; - } MultiPartition; @@ -158,7 +149,6 @@ typedef struct MultiPartition typedef struct MultiCartesianProduct { MultiBinaryNode binaryNode; - } MultiCartesianProduct; @@ -183,7 +173,6 @@ typedef struct MultiExtendedOp List *sortClauseList; Node *limitCount; Node *limitOffset; - } MultiExtendedOp; diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 09d28e952..3ad053b22 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -2,7 +2,7 @@ * * multi_physical_planner.h * Type and function declarations used in creating the distributed execution - * plan. + * plan. * * Copyright (c) 2012, Citus Data, Inc. * @@ -40,17 +40,18 @@ (" UINT64_FORMAT ", %d, %s, '%s', %d, %d)" #define MERGE_FILES_INTO_TABLE_COMMAND "SELECT worker_merge_files_into_table \ (" UINT64_FORMAT ", %d, '%s', '%s')" -#define MERGE_FILES_AND_RUN_QUERY_COMMAND "SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')" +#define MERGE_FILES_AND_RUN_QUERY_COMMAND \ + "SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')" typedef enum CitusRTEKind { - CITUS_RTE_RELATION = RTE_RELATION, /* ordinary relation reference */ - CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ - CITUS_RTE_JOIN = RTE_JOIN, /* join */ - CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ - CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ - CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ + CITUS_RTE_RELATION = RTE_RELATION, /* ordinary relation reference */ + CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ + CITUS_RTE_JOIN = RTE_JOIN, /* join */ + CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ + CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ + CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ CITUS_RTE_SHARD, CITUS_RTE_REMOTE_QUERY } CitusRTEKind; @@ -61,8 +62,7 @@ typedef enum { PARTITION_INVALID_FIRST = 0, RANGE_PARTITION_TYPE = 1, - HASH_PARTITION_TYPE = 2 - + HASH_PARTITION_TYPE = 2 } PartitionType; @@ -77,7 +77,6 @@ typedef enum MAP_OUTPUT_FETCH_TASK = 5, MERGE_FETCH_TASK = 6, MODIFY_TASK = 7 - } TaskType; @@ -88,7 +87,6 @@ typedef enum TASK_ASSIGNMENT_GREEDY = 1, TASK_ASSIGNMENT_ROUND_ROBIN = 2, TASK_ASSIGNMENT_FIRST_REPLICA = 3 - } TaskAssignmentPolicyType; @@ -99,7 +97,6 @@ typedef enum JOIN_MAP_MERGE_JOB = 1, SUBQUERY_MAP_MERGE_JOB = 2, TOP_LEVEL_WORKER_JOB = 3 - } BoundaryNodeJobType; @@ -133,7 +130,6 @@ typedef struct MapMergeJob ShardInterval **sortedShardIntervalArray; /* only applies to range partitioning */ List *mapTaskList; List *mergeTaskList; - } MapMergeJob; @@ -153,18 +149,17 @@ typedef struct Task uint64 jobId; uint32 taskId; char *queryString; - uint64 anchorShardId; /* only applies to compute tasks */ - List *taskPlacementList; /* only applies to compute tasks */ - List *dependedTaskList; /* only applies to compute tasks */ + uint64 anchorShardId; /* only applies to compute tasks */ + List *taskPlacementList; /* only applies to compute tasks */ + List *dependedTaskList; /* only applies to compute tasks */ uint32 partitionId; - uint32 upstreamTaskId; /* only applies to data fetch tasks */ + uint32 upstreamTaskId; /* only applies to data fetch tasks */ ShardInterval *shardInterval; /* only applies to merge tasks */ bool assignmentConstrained; /* only applies to merge tasks */ - uint64 shardId; /* only applies to shard fetch tasks */ + uint64 shardId; /* only applies to shard fetch tasks */ TaskExecution *taskExecution; /* used by task tracker executor */ - bool upsertQuery; /* only applies to modify tasks */ - + bool upsertQuery; /* only applies to modify tasks */ } Task; @@ -177,7 +172,6 @@ typedef struct RangeTableFragment CitusRTEKind fragmentType; void *fragmentReference; uint32 rangeTableId; - } RangeTableFragment; @@ -190,7 +184,6 @@ typedef struct JoinSequenceNode { uint32 rangeTableId; int32 joiningRangeTableId; - } JoinSequenceNode; @@ -203,7 +196,6 @@ typedef struct MultiPlan Job *workerJob; Query *masterQuery; char *masterTableName; - } MultiPlan; diff --git a/src/include/distributed/multi_planner.h b/src/include/distributed/multi_planner.h index a14b2b65e..d98743c9f 100644 --- a/src/include/distributed/multi_planner.h +++ b/src/include/distributed/multi_planner.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_planner.h - * General CitusDB planner code. + * General Citus planner code. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -13,8 +13,8 @@ #include "nodes/plannodes.h" #include "nodes/relation.h" -extern PlannedStmt *multi_planner(Query *parse, int cursorOptions, - ParamListInfo boundParams); +extern PlannedStmt * multi_planner(Query *parse, int cursorOptions, + ParamListInfo boundParams); extern bool HasCitusToplevelNode(PlannedStmt *planStatement); struct MultiPlan; diff --git a/src/include/distributed/multi_resowner.h b/src/include/distributed/multi_resowner.h index 195e2d9f0..3ffc04016 100644 --- a/src/include/distributed/multi_resowner.h +++ b/src/include/distributed/multi_resowner.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_resowner.h - * CitusDB resource owner integration. + * Citus resource owner integration. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_server_executor.h b/src/include/distributed/multi_server_executor.h index 83105cc54..e6e17f566 100644 --- a/src/include/distributed/multi_server_executor.h +++ b/src/include/distributed/multi_server_executor.h @@ -20,9 +20,9 @@ #define MAX_TASK_EXECUTION_FAILURES 3 /* allowed failure count for one task */ -#define MAX_TRACKER_FAILURE_COUNT 3 /* allowed failure count for one tracker */ +#define MAX_TRACKER_FAILURE_COUNT 3 /* allowed failure count for one tracker */ #define REMOTE_NODE_CONNECT_TIMEOUT 4000 /* async connect timeout in ms */ -#define RESERVED_FD_COUNT 64 /* file descriptors unavailable to executor */ +#define RESERVED_FD_COUNT 64 /* file descriptors unavailable to executor */ /* copy out query results */ #define COPY_QUERY_TO_STDOUT_TEXT "COPY (%s) TO STDOUT" @@ -32,9 +32,9 @@ /* Task tracker executor related defines */ #define TASK_ASSIGNMENT_QUERY "SELECT task_tracker_assign_task \ - ("UINT64_FORMAT", %u, %s)" -#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT", %u)" -#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT")" + ("UINT64_FORMAT ", %u, %s)" +#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT ", %u)" +#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT ")" #define JOB_CLEANUP_TASK_ID INT_MAX @@ -43,9 +43,9 @@ typedef enum { EXEC_TASK_INVALID_FIRST = 0, EXEC_TASK_CONNECT_START = 1, - EXEC_TASK_CONNECT_POLL = 2, + EXEC_TASK_CONNECT_POLL = 2, EXEC_TASK_FAILED = 3, - EXEC_FETCH_TASK_LOOP = 4, + EXEC_FETCH_TASK_LOOP = 4, EXEC_FETCH_TASK_START = 5, EXEC_FETCH_TASK_RUNNING = 6, EXEC_COMPUTE_TASK_START = 7, @@ -60,7 +60,6 @@ typedef enum EXEC_TASK_TRACKER_FAILED = 14, EXEC_SOURCE_TASK_TRACKER_RETRY = 15, EXEC_SOURCE_TASK_TRACKER_FAILED = 16 - } TaskExecStatus; @@ -74,7 +73,6 @@ typedef enum EXEC_TRANSMIT_TRACKER_RETRY = 4, EXEC_TRANSMIT_TRACKER_FAILED = 5, EXEC_TRANSMIT_DONE = 6 - } TransmitExecStatus; @@ -86,7 +84,6 @@ typedef enum TRACKER_CONNECT_POLL = 2, TRACKER_CONNECTED = 3, TRACKER_CONNECTION_FAILED = 4 - } TrackerStatus; @@ -97,7 +94,6 @@ typedef enum MULTI_EXECUTOR_REAL_TIME = 1, MULTI_EXECUTOR_TASK_TRACKER = 2, MULTI_EXECUTOR_ROUTER = 3 - } MultiExecutorType; @@ -107,7 +103,6 @@ typedef enum CONNECT_ACTION_NONE = 0, CONNECT_ACTION_OPENED = 1, CONNECT_ACTION_CLOSED = 2 - } ConnectAction; @@ -132,7 +127,6 @@ struct TaskExecution uint32 querySourceNodeIndex; /* only applies to map fetch tasks */ int32 dataFetchTaskIndex; uint32 failureCount; - }; @@ -147,7 +141,6 @@ typedef struct TrackerTaskState uint32 taskId; TaskStatus status; StringInfo taskAssignmentQuery; - } TrackerTaskState; @@ -158,7 +151,7 @@ typedef struct TrackerTaskState */ typedef struct TaskTracker { - uint32 workerPort; /* node's port; part of hash table key */ + uint32 workerPort; /* node's port; part of hash table key */ char workerName[WORKER_LENGTH]; /* node's name; part of hash table key */ TrackerStatus trackerStatus; int32 connectionId; @@ -171,7 +164,6 @@ typedef struct TaskTracker int32 currentTaskIndex; bool connectionBusy; TrackerTaskState *connectionBusyOnTask; - } TaskTracker; @@ -184,7 +176,6 @@ typedef struct WorkerNodeState uint32 workerPort; char workerName[WORKER_LENGTH]; uint32 openConnectionCount; - } WorkerNodeState; diff --git a/src/include/distributed/multi_utility.h b/src/include/distributed/multi_utility.h index 0a50b5b08..d37c2f3c8 100644 --- a/src/include/distributed/multi_utility.h +++ b/src/include/distributed/multi_utility.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_utility.h - * CitusDB utility hook and related functionality. + * Citus utility hook and related functionality. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/pg_dist_partition.h b/src/include/distributed/pg_dist_partition.h index d3db82638..d277bc8ce 100644 --- a/src/include/distributed/pg_dist_partition.h +++ b/src/include/distributed/pg_dist_partition.h @@ -21,9 +21,9 @@ */ typedef struct FormData_pg_dist_partition { - Oid logicalrelid; /* logical relation id; references pg_class oid */ - char partmethod; /* partition method; see codes below */ - text partkey; /* partition key expression */ + Oid logicalrelid; /* logical relation id; references pg_class oid */ + char partmethod; /* partition method; see codes below */ + text partkey; /* partition key expression */ } FormData_pg_dist_partition; /* ---------------- @@ -37,16 +37,16 @@ typedef FormData_pg_dist_partition *Form_pg_dist_partition; * compiler constants for pg_dist_partitions * ---------------- */ -#define Natts_pg_dist_partition 3 -#define Anum_pg_dist_partition_logicalrelid 1 -#define Anum_pg_dist_partition_partmethod 2 -#define Anum_pg_dist_partition_partkey 3 +#define Natts_pg_dist_partition 3 +#define Anum_pg_dist_partition_logicalrelid 1 +#define Anum_pg_dist_partition_partmethod 2 +#define Anum_pg_dist_partition_partkey 3 /* valid values for partmethod include append, hash, and range */ -#define DISTRIBUTE_BY_APPEND 'a' -#define DISTRIBUTE_BY_HASH 'h' -#define DISTRIBUTE_BY_RANGE 'r' -#define REDISTRIBUTE_BY_HASH 'x' +#define DISTRIBUTE_BY_APPEND 'a' +#define DISTRIBUTE_BY_HASH 'h' +#define DISTRIBUTE_BY_RANGE 'r' +#define REDISTRIBUTE_BY_HASH 'x' #endif /* PG_DIST_PARTITION_H */ diff --git a/src/include/distributed/pg_dist_shard.h b/src/include/distributed/pg_dist_shard.h index dfe1c86c7..b093bb59d 100644 --- a/src/include/distributed/pg_dist_shard.h +++ b/src/include/distributed/pg_dist_shard.h @@ -22,13 +22,13 @@ */ typedef struct FormData_pg_dist_shard { - Oid logicalrelid; /* logical relation id; references pg_class oid */ - int64 shardid; /* global shardId representing remote partition */ - char shardstorage; /* shard storage type; see codes below */ -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text shardalias; /* user specified table name for shard, if any */ - text shardminvalue; /* partition key's minimum value in shard */ - text shardmaxvalue; /* partition key's maximum value in shard */ + Oid logicalrelid; /* logical relation id; references pg_class oid */ + int64 shardid; /* global shardId representing remote partition */ + char shardstorage; /* shard storage type; see codes below */ +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text shardalias; /* user specified table name for shard, if any */ + text shardminvalue; /* partition key's minimum value in shard */ + text shardmaxvalue; /* partition key's maximum value in shard */ #endif } FormData_pg_dist_shard; @@ -43,22 +43,22 @@ typedef FormData_pg_dist_shard *Form_pg_dist_shard; * compiler constants for pg_dist_shards * ---------------- */ -#define Natts_pg_dist_shard 6 -#define Anum_pg_dist_shard_logicalrelid 1 -#define Anum_pg_dist_shard_shardid 2 -#define Anum_pg_dist_shard_shardstorage 3 -#define Anum_pg_dist_shard_shardalias 4 -#define Anum_pg_dist_shard_shardminvalue 5 -#define Anum_pg_dist_shard_shardmaxvalue 6 +#define Natts_pg_dist_shard 6 +#define Anum_pg_dist_shard_logicalrelid 1 +#define Anum_pg_dist_shard_shardid 2 +#define Anum_pg_dist_shard_shardstorage 3 +#define Anum_pg_dist_shard_shardalias 4 +#define Anum_pg_dist_shard_shardminvalue 5 +#define Anum_pg_dist_shard_shardmaxvalue 6 /* * Valid values for shard storage types include relay file, foreign table, * (standard) table and columnar table. Relay file types are currently unused. */ -#define SHARD_STORAGE_RELAY 'r' -#define SHARD_STORAGE_FOREIGN 'f' -#define SHARD_STORAGE_TABLE 't' -#define SHARD_STORAGE_COLUMNAR 'c' +#define SHARD_STORAGE_RELAY 'r' +#define SHARD_STORAGE_FOREIGN 'f' +#define SHARD_STORAGE_TABLE 't' +#define SHARD_STORAGE_COLUMNAR 'c' #endif /* PG_DIST_SHARD_H */ diff --git a/src/include/distributed/pg_dist_shard_placement.h b/src/include/distributed/pg_dist_shard_placement.h index 505daffa2..955e4efa6 100644 --- a/src/include/distributed/pg_dist_shard_placement.h +++ b/src/include/distributed/pg_dist_shard_placement.h @@ -23,12 +23,12 @@ */ typedef struct FormData_pg_dist_shard_placement { - int64 shardid; /* global shardId on remote node */ - int32 shardstate; /* shard state on remote node; see RelayFileState */ - int64 shardlength; /* shard length on remote node; stored as bigint */ -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text nodename; /* remote node's host name */ - int32 nodeport; /* remote node's port number */ + int64 shardid; /* global shardId on remote node */ + int32 shardstate; /* shard state on remote node; see RelayFileState */ + int64 shardlength; /* shard length on remote node; stored as bigint */ +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text nodename; /* remote node's host name */ + int32 nodeport; /* remote node's port number */ #endif } FormData_pg_dist_shard_placement; @@ -43,12 +43,12 @@ typedef FormData_pg_dist_shard_placement *Form_pg_dist_shard_placement; * compiler constants for pg_dist_shard_placement * ---------------- */ -#define Natts_pg_dist_shard_placement 5 -#define Anum_pg_dist_shard_placement_shardid 1 -#define Anum_pg_dist_shard_placement_shardstate 2 -#define Anum_pg_dist_shard_placement_shardlength 3 -#define Anum_pg_dist_shard_placement_nodename 4 -#define Anum_pg_dist_shard_placement_nodeport 5 +#define Natts_pg_dist_shard_placement 5 +#define Anum_pg_dist_shard_placement_shardid 1 +#define Anum_pg_dist_shard_placement_shardstate 2 +#define Anum_pg_dist_shard_placement_shardlength 3 +#define Anum_pg_dist_shard_placement_nodename 4 +#define Anum_pg_dist_shard_placement_nodeport 5 #endif /* PG_DIST_SHARD_PLACEMENT_H */ diff --git a/src/include/distributed/relay_utility.h b/src/include/distributed/relay_utility.h index 3685b3559..ba032d91f 100644 --- a/src/include/distributed/relay_utility.h +++ b/src/include/distributed/relay_utility.h @@ -3,7 +3,7 @@ * relay_utility.h * * Header and type declarations that extend relation, index and constraint names - * with the appropriate shard identifiers. + * with the appropriate shard identifiers. * * Copyright (c) 2012, Citus Data, Inc. * @@ -36,7 +36,6 @@ typedef enum FILE_CACHED = 2, FILE_INACTIVE = 3, FILE_TO_DELETE = 4 - } RelayFileState; diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h index 6c1c8ffcf..730bfdd4d 100644 --- a/src/include/distributed/resource_lock.h +++ b/src/include/distributed/resource_lock.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * resource_lock.h - * Locking Infrastructure for CitusDB. + * Locking Infrastructure for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -21,7 +21,7 @@ * advisory locks. Only 1 and 2 are used allowing us to define non-conflicting * lock methods. * - * In case postgres starts to use additional values, CitusDB's values + * In case postgres starts to use additional values, Citus's values * will have to be changed. That just requires re-compiling and a restart. */ typedef enum AdvisoryLocktagClass @@ -29,7 +29,8 @@ typedef enum AdvisoryLocktagClass /* values defined in postgres' lockfuncs.c */ ADV_LOCKTAG_CLASS_INT64 = 1, ADV_LOCKTAG_CLASS_INT32 = 2, - /* CitusDB lock types */ + + /* Citus lock types */ ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4, ADV_LOCKTAG_CLASS_CITUS_SHARD = 5, ADV_LOCKTAG_CLASS_CITUS_JOB = 6 diff --git a/src/include/distributed/task_tracker.h b/src/include/distributed/task_tracker.h index ad41b8589..2fc657d49 100644 --- a/src/include/distributed/task_tracker.h +++ b/src/include/distributed/task_tracker.h @@ -19,10 +19,10 @@ #include "utils/hsearch.h" -#define HIGH_PRIORITY_TASK_TIME 1 /* assignment time for high priority tasks */ -#define RESERVED_JOB_ID 1 /* reserved for cleanup and shutdown tasks */ +#define HIGH_PRIORITY_TASK_TIME 1 /* assignment time for high priority tasks */ +#define RESERVED_JOB_ID 1 /* reserved for cleanup and shutdown tasks */ #define SHUTDOWN_MARKER_TASK_ID UINT_MAX /* used to identify task tracker shutdown */ -#define MAX_TASK_FAILURE_COUNT 2 /* allowed failure count for one task */ +#define MAX_TASK_FAILURE_COUNT 2 /* allowed failure count for one task */ #define LOCAL_HOST_NAME "localhost" /* connect to local backends using this name */ #define TASK_CALL_STRING_SIZE 12288 /* max length of task call string */ #define TEMPLATE0_NAME "template0" /* skip job schema cleanup for template0 */ @@ -37,13 +37,13 @@ typedef enum { TASK_STATUS_INVALID_FIRST = 0, - TASK_ASSIGNED = 1, /* master node and task tracker */ + TASK_ASSIGNED = 1, /* master node and task tracker */ TASK_SCHEDULED = 2, TASK_RUNNING = 3, - TASK_FAILED = 4, + TASK_FAILED = 4, TASK_PERMANENTLY_FAILED = 5, TASK_SUCCEEDED = 6, - TASK_CANCEL_REQUESTED = 7, /* master node only */ + TASK_CANCEL_REQUESTED = 7, /* master node only */ TASK_CANCELED = 8, TASK_TO_REMOVE = 9, @@ -63,7 +63,6 @@ typedef enum * TASK_STATUS_LAST, should never have their numbers changed. */ TASK_STATUS_LAST - } TaskStatus; @@ -76,16 +75,15 @@ typedef enum */ typedef struct WorkerTask { - uint64 jobId; /* job id (upper 32-bits reserved); part of hash table key */ - uint32 taskId; /* task id; part of hash table key */ + uint64 jobId; /* job id (upper 32-bits reserved); part of hash table key */ + uint32 taskId; /* task id; part of hash table key */ uint32 assignedAt; /* task assignment time in epoch seconds */ char taskCallString[TASK_CALL_STRING_SIZE]; /* query or function call string */ - TaskStatus taskStatus; /* task's current execution status */ - char databaseName[NAMEDATALEN]; /* name to use for local backend connection */ - int32 connectionId; /* connection id to local backend */ - uint32 failureCount; /* number of task failures */ - + TaskStatus taskStatus; /* task's current execution status */ + char databaseName[NAMEDATALEN]; /* name to use for local backend connection */ + int32 connectionId; /* connection id to local backend */ + uint32 failureCount; /* number of task failures */ } WorkerTask; @@ -97,6 +95,7 @@ typedef struct WorkerTasksSharedStateData { /* Hash table shared by the task tracker and task tracker protocol functions */ HTAB *taskHash; + /* Lock protecting workerNodesHash */ LWLock *taskHashLock; } WorkerTasksSharedStateData; diff --git a/src/include/distributed/test_helper_functions.h b/src/include/distributed/test_helper_functions.h index 1d304bd4d..410239099 100644 --- a/src/include/distributed/test_helper_functions.h +++ b/src/include/distributed/test_helper_functions.h @@ -10,8 +10,8 @@ *------------------------------------------------------------------------- */ -#ifndef CITUSDB_TEST_HELPER_FUNCTIONS_H -#define CITUSDB_TEST_HELPER_FUNCTIONS_H +#ifndef CITUS_TEST_HELPER_FUNCTIONS_H +#define CITUS_TEST_HELPER_FUNCTIONS_H #include "postgres.h" #include "c.h" @@ -70,4 +70,4 @@ extern Datum prune_using_both_values(PG_FUNCTION_ARGS); extern Datum debug_equality_expression(PG_FUNCTION_ARGS); -#endif /* CITUSDB_TEST_HELPER_FUNCTIONS_H */ +#endif /* CITUS_TEST_HELPER_FUNCTIONS_H */ diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h index 57a38194a..96d84fe40 100644 --- a/src/include/distributed/worker_manager.h +++ b/src/include/distributed/worker_manager.h @@ -23,7 +23,7 @@ /* Maximum length of worker port number (represented as string) */ #define MAX_PORT_LENGTH 10 -/* default filename for citusdb.worker_list_file */ +/* default filename for citus.worker_list_file */ #define WORKER_LIST_FILENAME "pg_worker_list.conf" /* Implementation specific definitions used in finding worker nodes */ @@ -43,12 +43,11 @@ */ typedef struct WorkerNode { - uint32 workerPort; /* node's port; part of hash table key */ + uint32 workerPort; /* node's port; part of hash table key */ char workerName[WORKER_LENGTH]; /* node's name; part of hash table key */ char workerRack[WORKER_LENGTH]; /* node's network location */ - bool inWorkerFile; /* is node in current membership file? */ - + bool inWorkerFile; /* is node in current membership file? */ } WorkerNode; diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index e797b7396..ac2985f6a 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -64,8 +64,7 @@ typedef struct RangePartitionContext { FmgrInfo *comparisonFunction; Datum *splitPointArray; - int32 splitPointCount; - + int32 splitPointCount; } RangePartitionContext; @@ -77,7 +76,6 @@ typedef struct HashPartitionContext { FmgrInfo *hashFunction; uint32 partitionCount; - } HashPartitionContext; @@ -88,16 +86,16 @@ typedef struct HashPartitionContext */ typedef struct PartialCopyStateData { - StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for - * dest == COPY_NEW_FE in COPY FROM */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool binary; /* binary format? */ - char *null_print; /* NULL marker string (server encoding!) */ - char *null_print_client; /* same converted to file encoding */ - char *delim; /* column delimiter (must be 1 byte) */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for + * dest == COPY_NEW_FE in COPY FROM */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool binary; /* binary format? */ + char *null_print; /* NULL marker string (server encoding!) */ + char *null_print_client; /* same converted to file encoding */ + char *delim; /* column delimiter (must be 1 byte) */ - MemoryContext rowcontext; /* per-row evaluation context */ + MemoryContext rowcontext; /* per-row evaluation context */ } PartialCopyStateData; typedef struct PartialCopyStateData *PartialCopyState; @@ -114,7 +112,6 @@ typedef struct FileOutputStream File fileDescriptor; StringInfo fileBuffer; StringInfo filePath; - } FileOutputStream; diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 2b9ef7078..398ed9cac 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -1,9 +1,9 @@ -# Makefile for tests of the CitusDB extension +# Makefile for tests of the Citus extension -citusdb_subdir = src/test/regress -citusdb_top_builddir = ../../.. +citus_subdir = src/test/regress +citus_top_builddir = ../../.. -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global # ensure MAJORVERSION is defined (missing in older versions) ifndef MAJORVERSION @@ -11,11 +11,11 @@ MAJORVERSION := $(basename $(VERSION)) endif ## -## CitusDB regression support +## Citus regression support ## MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install -pg_regress_multi_check = $(PERL) $(citusdb_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" -MULTI_REGRESS_OPTS = --inputdir=$(citusdb_abs_srcdir) $(pg_regress_locale_flags) +pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" +MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) # XXX: Can't actually do useful testruns against install - $libdir # etc will point to the directory configured during postgres' @@ -26,12 +26,12 @@ cleandir-main: ### tempinstall-main: cleandir-main #### mkdir -p $(MULTI_INSTALLDIR) -### $(MAKE) DESTDIR=$(MULTI_INSTALLDIR) -C $(citusdb_top_builddir) install > tmp_check/install.log 2>&1 +### $(MAKE) DESTDIR=$(MULTI_INSTALLDIR) -C $(citus_top_builddir) install > tmp_check/install.log 2>&1 # Test input and expected files. These are created by pg_regress itself, so we # don't have a rule to create them. We do need rules to clean them however. -input_files := $(patsubst $(citusdb_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citusdb_abs_srcdir)/input/*.source)) -output_files := $(patsubst $(citusdb_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citusdb_abs_srcdir)/output/*.source)) +input_files := $(patsubst $(citus_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citus_abs_srcdir)/input/*.source)) +output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citus_abs_srcdir)/output/*.source)) # have make check actually run all tests, but keep check-full as an # intermediate, for muscle memory backward compatibility. @@ -42,32 +42,32 @@ check-full: check-multi check-multi-task-tracker check-multi-binary check-worker # using pg_regress_multi_check unnecessarily starts up multiple nodes, which isn't needed # for check-worker. But that's harmless besides a few cycles. check-worker: all - $(pg_regress_multi_check) --load-extension=citusdb \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/worker_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/worker_schedule $(EXTRA_TESTS) check-multi: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-fdw: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb --load-extension=file_fdw -- \ - $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_fdw_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus --load-extension=file_fdw -- \ + $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_fdw_schedule $(EXTRA_TESTS) check-multi-hll: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb --load-extension=hll -- \ + $(pg_regress_multi_check) --load-extension=citus --load-extension=hll -- \ $(MULTI_REGRESS_OPTS) $(EXTRA_TESTS) multi_create_table multi_master_protocol multi_stage_data multi_agg_approximate_distinct check-multi-task-tracker: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - --server-option=citusdb.task_executor_type=task-tracker \ - --server-option=citusdb.task_tracker_delay=50ms \ - --server-option=citusdb.large_table_shard_count=1 \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.task_executor_type=task-tracker \ + --server-option=citus.task_tracker_delay=50ms \ + --server-option=citus.large_table_shard_count=1 \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-binary: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - --server-option=citusdb.binary_worker_copy_format=on \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.binary_worker_copy_format=on \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) clean distclean maintainer-clean: rm -f $(output_files) $(input_files) diff --git a/src/test/regress/expected/multi_agg_approximate_distinct.out b/src/test/regress/expected/multi_agg_approximate_distinct.out index 13f4fcdf5..852142d0f 100644 --- a/src/test/regress/expected/multi_agg_approximate_distinct.out +++ b/src/test/regress/expected/multi_agg_approximate_distinct.out @@ -7,14 +7,14 @@ ERROR: cannot compute aggregate (distinct) DETAIL: table partitioning is unsuitable for aggregate (distinct) HINT: You can load the hll extension from contrib packages and enable distinct approximations. -- Check approximate count(distinct) at different precisions / error rates -SET citusdb.count_distinct_error_rate = 0.1; +SET citus.count_distinct_error_rate = 0.1; SELECT count(distinct l_orderkey) FROM lineitem; count ------- 2612 (1 row) -SET citusdb.count_distinct_error_rate = 0.01; +SET citus.count_distinct_error_rate = 0.01; SELECT count(distinct l_orderkey) FROM lineitem; count ------- @@ -102,7 +102,7 @@ SELECT count(DISTINCT l_orderkey) as distinct_order_count, l_quantity FROM linei -- If we have an order by on count(distinct) that we intend to push down to -- worker nodes, we need to error out. Otherwise, we are fine. -SET citusdb.limit_clause_row_fetch_count = 1000; +SET citus.limit_clause_row_fetch_count = 1000; SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as total FROM lineitem GROUP BY l_returnflag @@ -123,7 +123,7 @@ SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as t (3 rows) -- Check that we can revert config and disable count(distinct) approximations -SET citusdb.count_distinct_error_rate = 0.0; +SET citus.count_distinct_error_rate = 0.0; SELECT count(distinct l_orderkey) FROM lineitem; ERROR: cannot compute aggregate (distinct) DETAIL: table partitioning is unsuitable for aggregate (distinct) diff --git a/src/test/regress/expected/multi_array_agg.out b/src/test/regress/expected/multi_array_agg.out index 9a1d0b00c..b3fe1b307 100644 --- a/src/test/regress/expected/multi_array_agg.out +++ b/src/test/regress/expected/multi_array_agg.out @@ -96,10 +96,10 @@ SELECT l_quantity, count(*), avg(l_extendedprice), array_agg(l_orderkey) FROM li GROUP BY l_quantity ORDER BY l_quantity; l_quantity | count | avg | array_agg ------------+-------+-----------------------+-------------------------------------------------------------------------------------------------- - 1.00 | 17 | 1477.1258823529411765 | {5543,5633,5634,5698,5766,5856,5857,5986,8997,9026,9158,9184,9220,9222,9348,9383,9476} - 2.00 | 19 | 3078.4242105263157895 | {5506,5540,5573,5669,5703,5730,5798,5831,5893,5920,5923,9030,9058,9123,9124,9188,9344,9441,9476} - 3.00 | 14 | 4714.0392857142857143 | {5509,5543,5605,5606,5827,9124,9157,9184,9223,9254,9349,9414,9475,9477} - 4.00 | 19 | 5929.7136842105263158 | {5504,5507,5508,5511,5538,5764,5766,5826,5829,5862,5959,5985,9091,9120,9281,9347,9382,9440,9473} + 1.00 | 17 | 1477.1258823529411765 | {8997,9026,9158,9184,9220,9222,9348,9383,9476,5543,5633,5634,5698,5766,5856,5857,5986} + 2.00 | 19 | 3078.4242105263157895 | {9030,9058,9123,9124,9188,9344,9441,9476,5506,5540,5573,5669,5703,5730,5798,5831,5893,5920,5923} + 3.00 | 14 | 4714.0392857142857143 | {9124,9157,9184,9223,9254,9349,9414,9475,9477,5509,5543,5605,5606,5827} + 4.00 | 19 | 5929.7136842105263158 | {9091,9120,9281,9347,9382,9440,9473,5504,5507,5508,5511,5538,5764,5766,5826,5829,5862,5959,5985} (4 rows) SELECT l_quantity, array_agg(extract (month FROM o_orderdate)) AS my_month @@ -107,10 +107,10 @@ SELECT l_quantity, array_agg(extract (month FROM o_orderdate)) AS my_month AND l_orderkey > 5500 AND l_orderkey < 9500 GROUP BY l_quantity ORDER BY l_quantity; l_quantity | my_month ------------+------------------------------------------------ - 1.00 | {9,5,7,5,9,11,11,4,7,7,4,7,4,2,6,3,5} - 2.00 | {11,10,8,5,5,12,3,11,7,11,5,7,6,6,10,1,12,6,5} - 3.00 | {4,9,8,11,7,10,6,7,8,5,8,9,11,3} - 4.00 | {1,5,6,11,12,10,9,6,1,2,5,1,11,6,2,8,2,6,10} + 1.00 | {7,7,4,7,4,2,6,3,5,9,5,7,5,9,11,11,4} + 2.00 | {7,6,6,10,1,12,6,5,11,10,8,5,5,12,3,11,7,11,5} + 3.00 | {10,6,7,8,5,8,9,11,3,4,9,8,11,7} + 4.00 | {11,6,2,8,2,6,10,1,5,6,11,12,10,9,6,1,2,5,1} (4 rows) SELECT l_quantity, array_agg(l_orderkey * 2 + 1) FROM lineitem WHERE l_quantity < 5 @@ -118,10 +118,10 @@ SELECT l_quantity, array_agg(l_orderkey * 2 + 1) FROM lineitem WHERE l_quantity AND l_orderkey > 5500 AND l_orderkey < 9500 GROUP BY l_quantity ORDER BY l_quantity; l_quantity | array_agg ------------+--------------------------------------------- - 1.00 | {11269,11397,11713,11715,11973,18317,18445} - 2.00 | {11847,18061,18247,18953} + 1.00 | {18317,18445,11269,11397,11713,11715,11973} + 2.00 | {18061,18247,18953,11847} 3.00 | {18249,18315,18699,18951,18955} - 4.00 | {11653,11659,18241,18765} + 4.00 | {18241,18765,11653,11659} (4 rows) -- Check that we can execute array_agg() with an expression containing NULL values diff --git a/src/test/regress/expected/multi_binary_master_copy_format.out b/src/test/regress/expected/multi_binary_master_copy_format.out index eef2f7f05..505d18a34 100644 --- a/src/test/regress/expected/multi_binary_master_copy_format.out +++ b/src/test/regress/expected/multi_binary_master_copy_format.out @@ -2,8 +2,8 @@ -- MULTI_BINARY_MASTER_COPY -- -- Try binary master copy for different executors -SET citusdb.binary_master_copy_format TO 'on'; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.binary_master_copy_format TO 'on'; +SET citus.task_executor_type TO 'task-tracker'; SELECT count(*) FROM lineitem; count ------- @@ -17,7 +17,7 @@ SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; MAIL (2 rows) -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; SELECT count(*) FROM lineitem; count ------- diff --git a/src/test/regress/expected/multi_connection_cache.out b/src/test/regress/expected/multi_connection_cache.out index acd921a1c..ac808a0af 100644 --- a/src/test/regress/expected/multi_connection_cache.out +++ b/src/test/regress/expected/multi_connection_cache.out @@ -3,19 +3,19 @@ -- =================================================================== CREATE FUNCTION initialize_remote_temp_table(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION count_remote_temp_table_rows(cstring, integer) RETURNS integer - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION get_and_purge_connection(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION set_connection_status_bad(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test connection hash functionality diff --git a/src/test/regress/expected/multi_create_fdw.out b/src/test/regress/expected/multi_create_fdw.out index abd6a5616..006ad6d1a 100644 --- a/src/test/regress/expected/multi_create_fdw.out +++ b/src/test/regress/expected/multi_create_fdw.out @@ -4,7 +4,7 @@ -- create fake fdw for use in tests CREATE FUNCTION fake_fdw_handler() RETURNS fdw_handler -AS 'citusdb' +AS 'citus' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER fake_fdw HANDLER fake_fdw_handler; CREATE SERVER fake_fdw_server FOREIGN DATA WRAPPER fake_fdw; diff --git a/src/test/regress/expected/multi_create_insert_proxy.out b/src/test/regress/expected/multi_create_insert_proxy.out index 5737338ef..d90262c4a 100644 --- a/src/test/regress/expected/multi_create_insert_proxy.out +++ b/src/test/regress/expected/multi_create_insert_proxy.out @@ -9,8 +9,8 @@ CREATE SCHEMA "A$AP Mob" id bigint PRIMARY KEY, data text NOT NULL DEFAULT 'lorem ipsum' ); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet \set insert_target '"A$AP Mob"."Dr. Bronner''s ""Magic"" Soaps"' -- create proxy and save proxy table name SELECT create_insert_proxy_for_table(:'insert_target') AS proxy_tablename diff --git a/src/test/regress/expected/multi_create_shards.out b/src/test/regress/expected/multi_create_shards.out index 01e44580d..39cb45eef 100644 --- a/src/test/regress/expected/multi_create_shards.out +++ b/src/test/regress/expected/multi_create_shards.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION sort_names(cstring, cstring, cstring) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- create a custom type... CREATE TYPE dummy_type AS ( @@ -47,7 +47,7 @@ SELECT master_create_distributed_table('table_to_distribute', 'bad_column', 'has ERROR: column "bad_column" of relation "table_to_distribute" does not exist -- use unrecognized partition type SELECT master_create_distributed_table('table_to_distribute', 'name', 'unrecognized'); -ERROR: invalid input value for enum citusdb.distribution_type: "unrecognized" +ERROR: invalid input value for enum citus.distribution_type: "unrecognized" LINE 1: ..._distributed_table('table_to_distribute', 'name', 'unrecogni... ^ -- use a partition column of a type lacking any default operator class @@ -209,7 +209,7 @@ SELECT master_create_worker_shards('weird_shard_count', 7, 1); (1 row) --- CitusDB ensures all shards are roughly the same size +-- Citus ensures all shards are roughly the same size SELECT shardmaxvalue::integer - shardminvalue::integer AS shard_size FROM pg_dist_shard WHERE logicalrelid = 'weird_shard_count'::regclass diff --git a/src/test/regress/expected/multi_create_table.out b/src/test/regress/expected/multi_create_table.out index b079d7873..dd3f1c167 100644 --- a/src/test/regress/expected/multi_create_table.out +++ b/src/test/regress/expected/multi_create_table.out @@ -109,7 +109,7 @@ SELECT master_create_distributed_table('supplier', 's_suppkey', 'append'); (1 row) --- now test that CitusDB cannot distribute unique constraints that do not include +-- now test that Citus cannot distribute unique constraints that do not include -- the partition column CREATE TABLE primary_key_on_non_part_col ( @@ -127,7 +127,7 @@ CREATE TABLE unique_const_on_non_part_col SELECT master_create_distributed_table('primary_key_on_non_part_col', 'partition_col', 'hash'); ERROR: cannot distribute relation: "primary_key_on_non_part_col" DETAIL: Distributed relations cannot have UNIQUE constraints or PRIMARY KEYs that do not include the partition column. --- now show that CitusDB can distribute unique constrints that include +-- now show that Citus can distribute unique constrints that include -- the partition column CREATE TABLE primary_key_on_part_col ( diff --git a/src/test/regress/expected/multi_data_types.out b/src/test/regress/expected/multi_data_types.out index 8dc78f841..18b7a54dc 100644 --- a/src/test/regress/expected/multi_data_types.out +++ b/src/test/regress/expected/multi_data_types.out @@ -121,7 +121,7 @@ CREATE TABLE varchar_hash_partitioned_table id int, name varchar ); -SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'id', 'hash'); +SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'name', 'hash'); master_create_distributed_table --------------------------------- @@ -139,16 +139,16 @@ INSERT INTO varchar_hash_partitioned_table VALUES (2, 'Ozgun'); INSERT INTO varchar_hash_partitioned_table VALUES (3, 'Onder'); INSERT INTO varchar_hash_partitioned_table VALUES (4, 'Sumedh'); INSERT INTO varchar_hash_partitioned_table VALUES (5, 'Marco'); -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Onder'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 1; id | name ----+------- - 3 | Onder + 1 | Jason (1 row) -UPDATE varchar_hash_partitioned_table SET name = 'Samay' WHERE id = 5; -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Samay'; +UPDATE varchar_hash_partitioned_table SET id = 6 WHERE name = 'Jason'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 6; id | name ----+------- - 5 | Samay + 6 | Jason (1 row) diff --git a/src/test/regress/expected/multi_distribution_metadata.out b/src/test/regress/expected/multi_distribution_metadata.out index 6c69be7b8..d626b179e 100644 --- a/src/test/regress/expected/multi_distribution_metadata.out +++ b/src/test/regress/expected/multi_distribution_metadata.out @@ -3,55 +3,55 @@ -- =================================================================== CREATE FUNCTION load_shard_id_array(regclass) RETURNS bigint[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_interval_array(bigint, anyelement) RETURNS anyarray - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_placement_array(bigint, bool) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_column_id(regclass) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_type(regclass) RETURNS "char" - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION is_distributed_table(regclass) RETURNS boolean - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column_id(regclass, cstring) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_monolithic_shard_row(regclass) RETURNS bigint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_healthy_local_shard_placement_row(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION delete_shard_placement_row(bigint, text, bigint) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION update_shard_placement_row_state(bigint, text, bigint, int) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION acquire_shared_shard_lock(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column(regclass, text) RETURNS text - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test distribution metadata functionality diff --git a/src/test/regress/expected/multi_generate_ddl_commands.out b/src/test/regress/expected/multi_generate_ddl_commands.out index aa4d58822..ef63548ab 100644 --- a/src/test/regress/expected/multi_generate_ddl_commands.out +++ b/src/test/regress/expected/multi_generate_ddl_commands.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test ddl command generation functionality @@ -33,8 +33,8 @@ SELECT table_ddl_command_array('not_null_table'); -- ensure tables not in search path are schema-prefixed CREATE SCHEMA not_in_path CREATE TABLE simple_table (id bigint); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT table_ddl_command_array('not_in_path.simple_table'); table_ddl_command_array ------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_generate_ddl_commands_0.out b/src/test/regress/expected/multi_generate_ddl_commands_0.out index 1d042bc26..ff856b423 100644 --- a/src/test/regress/expected/multi_generate_ddl_commands_0.out +++ b/src/test/regress/expected/multi_generate_ddl_commands_0.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test ddl command generation functionality @@ -33,8 +33,8 @@ SELECT table_ddl_command_array('not_null_table'); -- ensure tables not in search path are schema-prefixed CREATE SCHEMA not_in_path CREATE TABLE simple_table (id bigint); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT table_ddl_command_array('not_in_path.simple_table'); table_ddl_command_array ------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_index_statements.out b/src/test/regress/expected/multi_index_statements.out index 245e2870f..7461e6025 100644 --- a/src/test/regress/expected/multi_index_statements.out +++ b/src/test/regress/expected/multi_index_statements.out @@ -10,59 +10,59 @@ SET client_min_messages TO DEBUG2; -- -- Verify that we can create different types of indexes CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_index" on table "lineitem" CREATE INDEX lineitem_partkey_desc_index ON lineitem (l_partkey DESC); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partkey_desc_index" on table "lineitem" CREATE INDEX lineitem_partial_index ON lineitem (l_shipdate) WHERE l_shipdate < '1995-01-01'; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partial_index" on table "lineitem" CREATE INDEX lineitem_orderkey_hash_index ON lineitem USING hash (l_partkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_hash_index" on table "lineitem" @@ -88,15 +88,15 @@ ERROR: specifying tablespaces with CREATE INDEX statements is currently unsuppo -- Verify that we error out in case of postgres errors on supported statement -- types. CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: relation "lineitem_orderkey_index_102014" already exists ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem USING gist (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: data type bigint has no default operator class for access method "gist" ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem (non_existent_column); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: column "non_existent_column" does not exist ERROR: could not execute DDL command on worker node shards -- Verify that none of failed indexes got created on the master node @@ -123,42 +123,42 @@ DROP INDEX CONCURRENTLY lineitem_orderkey_index; ERROR: dropping indexes concurrently on distributed tables is currently unsupported -- Verify that we can succesfully drop indexes DROP INDEX lineitem_orderkey_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_partkey_desc_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_partial_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 -- Verify that we handle if exists statements correctly @@ -167,16 +167,16 @@ ERROR: index "non_existent_index" does not exist DROP INDEX IF EXISTS non_existent_index; NOTICE: index "non_existent_index" does not exist, skipping DROP INDEX IF EXISTS lineitem_orderkey_hash_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_orderkey_hash_index; diff --git a/src/test/regress/expected/multi_index_statements_0.out b/src/test/regress/expected/multi_index_statements_0.out index 36cf47981..57c5b244a 100644 --- a/src/test/regress/expected/multi_index_statements_0.out +++ b/src/test/regress/expected/multi_index_statements_0.out @@ -10,59 +10,59 @@ SET client_min_messages TO DEBUG2; -- -- Verify that we can create different types of indexes CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_index" on table "lineitem" CREATE INDEX lineitem_partkey_desc_index ON lineitem (l_partkey DESC); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partkey_desc_index" on table "lineitem" CREATE INDEX lineitem_partial_index ON lineitem (l_shipdate) WHERE l_shipdate < '1995-01-01'; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partial_index" on table "lineitem" CREATE INDEX lineitem_orderkey_hash_index ON lineitem USING hash (l_partkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 WARNING: hash indexes are not WAL-logged and their use is discouraged @@ -89,15 +89,15 @@ ERROR: specifying tablespaces with CREATE INDEX statements is currently unsuppo -- Verify that we error out in case of postgres errors on supported statement -- types. CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: relation "lineitem_orderkey_index_102014" already exists ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem USING gist (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: data type bigint has no default operator class for access method "gist" ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem (non_existent_column); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: column "non_existent_column" does not exist ERROR: could not execute DDL command on worker node shards -- Verify that none of failed indexes got created on the master node @@ -124,44 +124,44 @@ DROP INDEX CONCURRENTLY lineitem_orderkey_index; ERROR: dropping indexes concurrently on distributed tables is currently unsupported -- Verify that we can succesfully drop indexes DROP INDEX lineitem_orderkey_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16541 DROP INDEX lineitem_partkey_desc_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16541 DROP INDEX lineitem_partial_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16541 @@ -171,16 +171,16 @@ ERROR: index "non_existent_index" does not exist DROP INDEX IF EXISTS non_existent_index; NOTICE: index "non_existent_index" does not exist, skipping DROP INDEX IF EXISTS lineitem_orderkey_hash_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16541 diff --git a/src/test/regress/expected/multi_join_order_additional.out b/src/test/regress/expected/multi_join_order_additional.out index c641140a7..9fd9cac07 100644 --- a/src/test/regress/expected/multi_join_order_additional.out +++ b/src/test/regress/expected/multi_join_order_additional.out @@ -2,7 +2,7 @@ -- MULTI_JOIN_ORDER_ADDITIONAL -- -- Set configuration to print table join order and pruned shards -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO DEBUG2; -- The following query checks that we can correctly handle self-joins EXPLAIN SELECT l1.l_quantity FROM lineitem l1, lineitem l2 @@ -42,7 +42,7 @@ DEBUG: join prunable for intervals [1,2496] and [2497,4964] (1 row) -- Update configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO LOG; -- The following queries check that we correctly handle joins and OR clauses. In -- particular, these queries check that we factorize out OR clauses if possible, @@ -106,7 +106,7 @@ LOG: join order: [ "customer" ][ broadcast join "nation" ] UPDATE pg_dist_partition SET partmethod = 'a' WHERE logicalrelid = (SELECT relfilenode FROM pg_class WHERE relname = 'customer'); -- Update the large table shard count for all the following tests. -SET citusdb.large_table_shard_count TO 1; +SET citus.large_table_shard_count TO 1; -- Validate that we don't use a single-partition join method for a hash -- re-partitioned table, thus preventing a partition of just the customer table. EXPLAIN SELECT count(*) FROM orders, lineitem, customer diff --git a/src/test/regress/expected/multi_join_order_tpch_large.out b/src/test/regress/expected/multi_join_order_tpch_large.out index 14bc84cff..deb1f20da 100644 --- a/src/test/regress/expected/multi_join_order_tpch_large.out +++ b/src/test/regress/expected/multi_join_order_tpch_large.out @@ -2,13 +2,13 @@ -- MULTI_JOIN_ORDER_TPCH_LARGE -- -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem, orders, customer, and part tables as -- large. The following queries are basically the same as the ones in tpch_small -- except that more data has been staged to customer and part tables. Therefore, -- we will apply different distributed join strategies for these queries. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark EXPLAIN SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_join_order_tpch_small.out b/src/test/regress/expected/multi_join_order_tpch_small.out index 696736a00..6601fb53c 100644 --- a/src/test/regress/expected/multi_join_order_tpch_small.out +++ b/src/test/regress/expected/multi_join_order_tpch_small.out @@ -2,10 +2,10 @@ -- MULTI_JOIN_ORDER_TPCH_SMALL -- -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark EXPLAIN SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_join_pruning.out b/src/test/regress/expected/multi_join_pruning.out index e690636ee..578964ac5 100644 --- a/src/test/regress/expected/multi_join_pruning.out +++ b/src/test/regress/expected/multi_join_pruning.out @@ -6,7 +6,7 @@ -- future we want to check for pruning between re-partitioned relations as well. SET client_min_messages TO DEBUG2; -- Change configuration to treat all tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders WHERE l_orderkey = o_orderkey; DEBUG: join prunable for intervals [13921,14947] and [1,5986] diff --git a/src/test/regress/expected/multi_large_table_join_planning.out b/src/test/regress/expected/multi_large_table_join_planning.out index 499a1f5be..842f164cc 100644 --- a/src/test/regress/expected/multi_large_table_join_planning.out +++ b/src/test/regress/expected/multi_large_table_join_planning.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG4; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -63,12 +63,12 @@ DEBUG: generated sql query for job 1250 and task 15 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102010 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > '10'::numeric)" DEBUG: generated sql query for job 1250 and task 18 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102009 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > '10'::numeric)" -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: generated sql query for job 1251 and task 3 @@ -99,9 +99,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: completed cleanup query for job 1252 on node "localhost:57638" DEBUG: completed cleanup query for job 1252 on node "localhost:57637" DEBUG: completed cleanup query for job 1251 on node "localhost:57638" @@ -167,18 +167,18 @@ DEBUG: generated sql query for job 1253 and task 10 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102010 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for job 1253 and task 12 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102009 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: generated sql query for job 1254 and task 2 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102016 orders WHERE (o_totalprice <> 4.0)" DEBUG: generated sql query for job 1254 and task 4 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102015 orders WHERE (o_totalprice <> 4.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/expected/multi_large_table_join_planning_0.out b/src/test/regress/expected/multi_large_table_join_planning_0.out index 48c7f8a77..33700d111 100644 --- a/src/test/regress/expected/multi_large_table_join_planning_0.out +++ b/src/test/regress/expected/multi_large_table_join_planning_0.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG4; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -63,12 +63,12 @@ DEBUG: generated sql query for job 1250 and task 15 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102010 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > 10::numeric)" DEBUG: generated sql query for job 1250 and task 18 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102009 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > 10::numeric)" -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: generated sql query for job 1251 and task 3 @@ -99,9 +99,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: completed cleanup query for job 1252 on node "localhost:57638" DEBUG: completed cleanup query for job 1252 on node "localhost:57637" DEBUG: completed cleanup query for job 1251 on node "localhost:57638" @@ -167,18 +167,18 @@ DEBUG: generated sql query for job 1253 and task 10 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102010 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for job 1253 and task 12 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102009 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: generated sql query for job 1254 and task 2 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102016 orders WHERE (o_totalprice <> 4.0)" DEBUG: generated sql query for job 1254 and task 4 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102015 orders WHERE (o_totalprice <> 4.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/expected/multi_large_table_pruning.out b/src/test/regress/expected/multi_large_table_pruning.out index d957e7669..20b62fa4d 100644 --- a/src/test/regress/expected/multi_large_table_pruning.out +++ b/src/test/regress/expected/multi_large_table_pruning.out @@ -4,9 +4,9 @@ -- Tests covering partition and join-pruning for large table joins. Note that we -- set executor type to task tracker executor here, as we cannot run repartition -- jobs with real time executor. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO DEBUG2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Single range-repartition join to test join-pruning behaviour. SELECT count(*) diff --git a/src/test/regress/expected/multi_large_table_task_assignment.out b/src/test/regress/expected/multi_large_table_task_assignment.out index 9bf12ff00..7ba87fa94 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment.out +++ b/src/test/regress/expected/multi_large_table_task_assignment.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG3; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -25,8 +25,8 @@ FROM WHERE o_custkey = c_custkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [6001,7000] @@ -39,9 +39,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 11 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: CommitTransactionCommand count ------- @@ -52,7 +52,7 @@ DEBUG: CommitTransactionCommand -- more than one shard. This situation results in multiple sql tasks depending on -- the same merge task, and tests our constraint group creation and assignment -- propagation. Here 'orders' is considered the small table. -SET citusdb.large_table_shard_count TO 3; +SET citus.large_table_shard_count TO 3; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -64,12 +64,12 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 15 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 18 to node localhost:57638 DEBUG: assigned task 9 to node localhost:57637 DEBUG: assigned task 12 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,2496] and [13921,14947] DEBUG: join prunable for intervals [1,2496] and [11554,13920] DEBUG: join prunable for intervals [1,2496] and [8997,11554] @@ -114,12 +114,12 @@ DEBUG: pruning merge fetch taskId 19 DETAIL: Creating dependency on merge taskId 47 DEBUG: pruning merge fetch taskId 22 DETAIL: Creating dependency on merge taskId 54 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 9 to node localhost:57637 -DEBUG: assigned task 12 to node localhost:57638 -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 24 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 24 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 DEBUG: propagating assignment from merge task 40 to constrained sql task 15 DEBUG: propagating assignment from merge task 47 to constrained sql task 21 DEBUG: CommitTransactionCommand @@ -128,7 +128,7 @@ DEBUG: CommitTransactionCommand 11998 (1 row) -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -154,15 +154,15 @@ FROM WHERE l_partkey = c_nationkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: assigned task 4 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/expected/multi_limit_clause_approximate.out b/src/test/regress/expected/multi_limit_clause_approximate.out index d65c6ae6f..d1bad570e 100644 --- a/src/test/regress/expected/multi_limit_clause_approximate.out +++ b/src/test/regress/expected/multi_limit_clause_approximate.out @@ -24,7 +24,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem (10 rows) -- Enable limit optimization to fetch one third of each shard's data -SET citusdb.limit_clause_row_fetch_count TO 600; +SET citus.limit_clause_row_fetch_count TO 600; SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem GROUP BY l_partkey ORDER BY aggregate DESC LIMIT 10; @@ -45,7 +45,7 @@ DEBUG: push down of limit count: 600 -- Disable limit optimization for our second test. This time, we have a query -- that joins several tables, and that groups and orders the results. -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem WHERE c_custkey = o_custkey AND l_orderkey = o_orderkey @@ -68,8 +68,8 @@ SELECT c_custkey, c_name, count(*) as lineitem_count -- Now, enable limit optimization to fetch half of each task's results. For this -- test, we also change a config setting to ensure that we don't repartition any -- of the tables during the query. -SET citusdb.limit_clause_row_fetch_count TO 150; -SET citusdb.large_table_shard_count TO 2; +SET citus.limit_clause_row_fetch_count TO 150; +SET citus.large_table_shard_count TO 2; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem WHERE c_custkey = o_custkey AND l_orderkey = o_orderkey @@ -90,7 +90,7 @@ DEBUG: push down of limit count: 150 145 | Customer#000000145 | 30 (10 rows) -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- We now test scenarios where applying the limit optimization wouldn't produce -- meaningful results. First, we check that we don't push down the limit clause -- for non-commutative aggregates. @@ -147,5 +147,5 @@ SELECT count(*) count_quantity, l_quantity FROM lineitem WHERE l_quantity < 10.0 258 | 9.00 (9 rows) -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; RESET client_min_messages; diff --git a/src/test/regress/expected/multi_modifications.out b/src/test/regress/expected/multi_modifications.out index 5da80026b..3b62f7c38 100644 --- a/src/test/regress/expected/multi_modifications.out +++ b/src/test/regress/expected/multi_modifications.out @@ -61,10 +61,14 @@ SELECT master_create_empty_shard('range_partitioned') AS new_shard_id \gset UPDATE pg_dist_shard SET shardminvalue = 50000, shardmaxvalue = 99999 WHERE shardid = :new_shard_id; --- create append-partitioned shard +-- create append-partitioned shards SELECT master_create_empty_shard('append_partitioned') AS new_shard_id \gset -UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 100000 +UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 500000 +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('append_partitioned') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 500000, shardmaxvalue = 1000000 WHERE shardid = :new_shard_id; -- basic single-row INSERT INSERT INTO limit_orders VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', @@ -82,9 +86,12 @@ ERROR: distributed modifications must target exactly one shard -- try an insert to a range-partitioned table INSERT INTO range_partitioned VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- ensure the value is where we put it and query to find it is properly pruned +-- also insert to an append-partitioned table +INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', + 20.69); +-- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; -SET citusdb.task_executor_type TO 'router'; +SET citus.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; DEBUG: predicate pruning for shardId 103070 id | symbol | bidder_id | placed_at | kind | limit_price @@ -92,17 +99,23 @@ DEBUG: predicate pruning for shardId 103070 32743 | AAPL | 9580 | Tue Oct 19 10:23:54 2004 | buy | 20.69 (1 row) +SELECT * FROM append_partitioned WHERE id = 414123; +DEBUG: predicate pruning for shardId 103072 + id | symbol | bidder_id | placed_at | kind | limit_price +--------+--------+-----------+--------------------------+------+------------- + 414123 | AAPL | 9580 | Tue Oct 19 10:23:54 2004 | buy | 20.69 +(1 row) + SET client_min_messages TO DEFAULT; -SET citusdb.task_executor_type TO DEFAULT; --- also try inserting without a range-partitioned shard to receive the value +SET citus.task_executor_type TO DEFAULT; +-- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); ERROR: distributed modifications must target exactly one shard --- try an insert to an append-partitioned table -INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', +-- and insert into an append-partitioned table with a value that spans shards: +INSERT INTO append_partitioned VALUES (500000, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); -ERROR: cannot perform distributed planning for the given modification -DETAIL: Only hash- or range-partitioned tables may be the target of distributed modifications +ERROR: distributed modifications must target exactly one shard -- INSERT with DEFAULT in the target list INSERT INTO limit_orders VALUES (12756, 'MSFT', 10959, '2013-05-08 07:29:23', 'sell', DEFAULT); diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning.out b/src/test/regress/expected/multi_null_minmax_value_pruning.out index 53e5a10a5..486db51fb 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning.out @@ -5,7 +5,7 @@ -- and that we don't partition or join prune shards that have null values. SET client_min_messages TO DEBUG2; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102009; shardminvalue | shardmaxvalue ---------------+--------------- diff --git a/src/test/regress/expected/multi_prepare_plsql.out b/src/test/regress/expected/multi_prepare_plsql.out index c2848b7cb..e76482076 100644 --- a/src/test/regress/expected/multi_prepare_plsql.out +++ b/src/test/regress/expected/multi_prepare_plsql.out @@ -173,7 +173,7 @@ BEGIN l_year; END; $$ LANGUAGE plpgsql; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- now, run plain SQL functions SELECT sql_test_no_1(); @@ -313,7 +313,7 @@ SELECT plpgsql_test_2(); -- run the tests which do not require re-partition -- with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- now, run plain SQL functions SELECT sql_test_no_1(); sql_test_no_1 diff --git a/src/test/regress/expected/multi_prepare_sql.out b/src/test/regress/expected/multi_prepare_sql.out index 1aff5bccf..b0d097a3f 100644 --- a/src/test/regress/expected/multi_prepare_sql.out +++ b/src/test/regress/expected/multi_prepare_sql.out @@ -95,7 +95,7 @@ ORDER BY supp_nation, cust_nation, l_year; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- execute prepared statements EXECUTE prepared_test_1; @@ -217,7 +217,7 @@ SELECT * from prepared_sql_test_7; (1 row) -- now, run some of the tests with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- execute prepared statements EXECUTE prepared_test_1; count diff --git a/src/test/regress/expected/multi_prune_shard_list.out b/src/test/regress/expected/multi_prune_shard_list.out index 029228841..2f74b1991 100644 --- a/src/test/regress/expected/multi_prune_shard_list.out +++ b/src/test/regress/expected/multi_prune_shard_list.out @@ -3,23 +3,23 @@ -- =================================================================== CREATE FUNCTION prune_using_no_values(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_single_value(regclass, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C; CREATE FUNCTION prune_using_either_value(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_both_values(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION debug_equality_expression(regclass) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test shard pruning functionality diff --git a/src/test/regress/expected/multi_simple_queries.out b/src/test/regress/expected/multi_simple_queries.out index 4407ec697..764e3c705 100644 --- a/src/test/regress/expected/multi_simple_queries.out +++ b/src/test/regress/expected/multi_simple_queries.out @@ -169,7 +169,7 @@ SELECT title, author_id FROM articles (10 rows) -- add in some grouping expressions, still on same shard --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles WHERE author_id = 1 OR author_id = 7 OR author_id = 8 OR author_id = 10 GROUP BY author_id @@ -190,7 +190,7 @@ ERROR: cannot plan queries that include both regular and partitioned relations SELECT * FROM articles, position('om' in 'Thomas'); ERROR: cannot perform distributed planning on this query DETAIL: Complex table expressions are currently unsupported --- subqueries are not supported in WHERE clause in CitusDB +-- subqueries are not supported in WHERE clause in Citus SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); ERROR: cannot plan queries that include both regular and partitioned relations -- subqueries are supported in FROM clause @@ -285,7 +285,7 @@ SELECT COUNT(*) FROM articles; 50 (1 row) --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles GROUP BY author_id HAVING sum(word_count) > 25000 @@ -293,23 +293,23 @@ SELECT author_id, sum(word_count) AS corpus_size FROM articles LIMIT 5; ERROR: cannot perform distributed planning on this query DETAIL: Having qual is currently unsupported --- more proof CitusDB doesn't support having clauses +-- more proof Citus doesn't support having clauses SELECT author_id FROM articles GROUP BY author_id HAVING sum(word_count) > 50000 ORDER BY author_id; ERROR: cannot perform distributed planning on this query DETAIL: Having qual is currently unsupported --- now, test the cases where CitusDB do or do not need to create +-- now, test the cases where Citus do or do not need to create -- the master queries -SET citusdb.task_executor_type TO 'router'; -SET citusdb.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'router'; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO 'DEBUG2'; -- start with the simple lookup query SELECT * FROM articles WHERE author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+--------------+------------ 1 | 1 | arsenous | 9572 @@ -323,7 +323,7 @@ DEBUG: predicate pruning for shardId 103093 SELECT * FROM articles WHERE author_id = 1 OR author_id = 17; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+--------------+------------ 1 | 1 | arsenous | 9572 @@ -338,12 +338,12 @@ SELECT * FROM articles WHERE author_id = 1 OR author_id = 18; ERROR: cannot use router executor with queries that hit multiple shards -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- rename the output columns on a no master query case SELECT id as article_id, word_count * id as random_value FROM articles WHERE author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 article_id | random_value ------------+-------------- 1 | 9572 @@ -360,7 +360,7 @@ SELECT a.author_id as first_author, b.word_count as second_word_count WHERE a.author_id = 10 and a.author_id = b.author_id LIMIT 3; DEBUG: push down of limit count: 3 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 DEBUG: join prunable for intervals [-2147483648,-1] and [0,2147483647] first_author | second_word_count --------------+------------------- @@ -375,16 +375,16 @@ SELECT a.author_id as first_author, b.word_count as second_word_count WHERE a.author_id = 10 and a.author_id = b.author_id LIMIT 3; DEBUG: push down of limit count: 3 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with JOINs -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- do not create the master query for LIMIT on a single shard SELECT SELECT * FROM articles WHERE author_id = 1 LIMIT 2; DEBUG: push down of limit count: 2 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+----------+------------ 1 | 1 | arsenous | 9572 @@ -398,7 +398,7 @@ SELECT id FROM articles WHERE author_id = 1 GROUP BY id; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id ---- 41 @@ -415,26 +415,26 @@ COPY articles_single_shard TO stdout; SELECT avg(word_count) FROM articles WHERE author_id = 2; -DEBUG: predicate pruning for shardId 103092 +DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- max, min, sum, count is somehow implemented -- differently in distributed planning but, still error out SELECT max(word_count) as max, min(word_count) as min, sum(word_count) as sum, count(word_count) as cnt FROM articles WHERE author_id = 2; -DEBUG: predicate pruning for shardId 103092 +DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY SELECT * FROM articles WHERE author_id = 1 ORDER BY word_count; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY and LIMIT SELECT * FROM articles @@ -442,22 +442,22 @@ SELECT * ORDER BY word_count LIMIT 2; DEBUG: push down of limit count: 2 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with aggregates and GROUP BY SELECT max(word_count) FROM articles WHERE author_id = 1 GROUP BY author_id; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with repartition jobs SELECT * FROM articles a, articles b WHERE a.id = b.id AND a.author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 @@ -487,12 +487,12 @@ DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 11 DETAIL: Creating dependency on merge taskId 14 ERROR: cannot use router executor with repartition jobs -HINT: Set citusdb.task_executor_type to "task-tracker". +HINT: Set citus.task_executor_type to "task-tracker". -- error out for queries which hit more than 1 shards SELECT * FROM articles WHERE author_id >= 1 AND author_id <= 3; ERROR: cannot use router executor with queries that hit multiple shards -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". SET client_min_messages to 'NOTICE'; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; diff --git a/src/test/regress/expected/multi_single_relation_subquery.out b/src/test/regress/expected/multi_single_relation_subquery.out index 1573e3a0a..b80833a2e 100644 --- a/src/test/regress/expected/multi_single_relation_subquery.out +++ b/src/test/regress/expected/multi_single_relation_subquery.out @@ -2,7 +2,7 @@ -- MULTI_SINGLE_RELATION_SUBQUERY -- -- This test checks that we are able to run selected set of distributed SQL subqueries. -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; select number_sum, count(*) as total, @@ -171,6 +171,18 @@ from l_tax) as distributed_table; ERROR: cannot perform distributed planning on this query DETAIL: Subqueries without aggregates are not supported yet +-- Check that we don't support subqueries with count(distinct). +select + different_shipment_days +from + (select + count(distinct l_shipdate) as different_shipment_days + from + lineitem + group by + l_partkey) as distributed_table; +ERROR: cannot compute count (distinct) +DETAIL: Subqueries with aggregate (distinct) are not supported yet -- Check that if subquery is pulled, we don't error and run query properly. SELECT max(l_suppkey) FROM ( diff --git a/src/test/regress/expected/multi_table_ddl.out b/src/test/regress/expected/multi_table_ddl.out index ebf07f786..0b8f0f789 100644 --- a/src/test/regress/expected/multi_table_ddl.out +++ b/src/test/regress/expected/multi_table_ddl.out @@ -9,12 +9,12 @@ SELECT master_create_distributed_table('testtableddl', 'distributecol', 'append' (1 row) --- verify that the citusdb extension can't be dropped while distributed tables exist -DROP EXTENSION citusdb; +-- verify that the citus extension can't be dropped while distributed tables exist +DROP EXTENSION citus; WARNING: could not clean the metadata cache on DROP EXTENSION command HINT: Reconnect to the server again. -ERROR: cannot drop extension citusdb because other objects depend on it -DETAIL: table testtableddl depends on extension citusdb +ERROR: cannot drop extension citus because other objects depend on it +DETAIL: table testtableddl depends on extension citus HINT: Use DROP ... CASCADE to drop the dependent objects too. -- verify that the distribution column can't have its type changed ALTER TABLE testtableddl ALTER COLUMN distributecol TYPE text; @@ -65,8 +65,8 @@ SELECT * FROM pg_dist_shard_placement; -- check that the extension now can be dropped (and recreated). We reconnect -- before creating the extension to expire extension specific variables which -- are cached for performance. -DROP EXTENSION citusdb; +DROP EXTENSION citus; WARNING: could not clean the metadata cache on DROP EXTENSION command HINT: Reconnect to the server again. \c -CREATE EXTENSION citusdb; +CREATE EXTENSION citus; diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index 11f4b5115..381d57b50 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -46,7 +46,7 @@ BEGIN; SET client_min_messages TO DEBUG3; DEBUG: CommitTransactionCommand -- First test the default greedy task assignment policy -SET citusdb.task_assignment_policy TO 'greedy'; +SET citus.task_assignment_policy TO 'greedy'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -75,7 +75,7 @@ DEBUG: CommitTransactionCommand (1 row) -- Next test the first-replica task assignment policy -SET citusdb.task_assignment_policy TO 'first-replica'; +SET citus.task_assignment_policy TO 'first-replica'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -118,7 +118,7 @@ DEBUG: CommitTransactionCommand (1 row) -- Finally test the round-robin task assignment policy -SET citusdb.task_assignment_policy TO 'round-robin'; +SET citus.task_assignment_policy TO 'round-robin'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -158,7 +158,7 @@ DEBUG: CommitTransactionCommand explain statements for distributed queries are currently unsupported (1 row) -RESET citusdb.task_assignment_policy; +RESET citus.task_assignment_policy; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand diff --git a/src/test/regress/expected/multi_tpch_query1.out b/src/test/regress/expected/multi_tpch_query1.out index 39e75276a..23a507e06 100644 --- a/src/test/regress/expected/multi_tpch_query1.out +++ b/src/test/regress/expected/multi_tpch_query1.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY1 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #1 from the TPC-H decision support benchmark SELECT l_returnflag, diff --git a/src/test/regress/expected/multi_tpch_query10.out b/src/test/regress/expected/multi_tpch_query10.out index ea6f8406e..82eefa6df 100644 --- a/src/test/regress/expected/multi_tpch_query10.out +++ b/src/test/regress/expected/multi_tpch_query10.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY10 -- -- Query #10 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT c_custkey, diff --git a/src/test/regress/expected/multi_tpch_query12.out b/src/test/regress/expected/multi_tpch_query12.out index 8d692752d..e54d3e0a4 100644 --- a/src/test/regress/expected/multi_tpch_query12.out +++ b/src/test/regress/expected/multi_tpch_query12.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY12 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #12 from the TPC-H decision support benchmark SELECT l_shipmode, diff --git a/src/test/regress/expected/multi_tpch_query14.out b/src/test/regress/expected/multi_tpch_query14.out index 1f96ff96c..2eded6dda 100644 --- a/src/test/regress/expected/multi_tpch_query14.out +++ b/src/test/regress/expected/multi_tpch_query14.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY14 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #14 from the TPC-H decision support benchmark SELECT 100.00 * sum(case diff --git a/src/test/regress/expected/multi_tpch_query19.out b/src/test/regress/expected/multi_tpch_query19.out index 784c2f5ce..d6ee657c1 100644 --- a/src/test/regress/expected/multi_tpch_query19.out +++ b/src/test/regress/expected/multi_tpch_query19.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY19 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #19 from the TPC-H decision support benchmark. Note that we modified -- the query from its original to make it work on smaller data sets. SELECT diff --git a/src/test/regress/expected/multi_tpch_query3.out b/src/test/regress/expected/multi_tpch_query3.out index c16171339..da87dc8f5 100644 --- a/src/test/regress/expected/multi_tpch_query3.out +++ b/src/test/regress/expected/multi_tpch_query3.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY3 -- -- Query #3 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT l_orderkey, diff --git a/src/test/regress/expected/multi_tpch_query6.out b/src/test/regress/expected/multi_tpch_query6.out index 977009bbb..9badebd9e 100644 --- a/src/test/regress/expected/multi_tpch_query6.out +++ b/src/test/regress/expected/multi_tpch_query6.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY6 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_tpch_query7.out b/src/test/regress/expected/multi_tpch_query7.out index 253da5c71..55ab9ae63 100644 --- a/src/test/regress/expected/multi_tpch_query7.out +++ b/src/test/regress/expected/multi_tpch_query7.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY7 -- -- Change configuration to treat lineitem AND orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H decision support benchmark SELECT supp_nation, diff --git a/src/test/regress/expected/multi_tpch_query7_nested.out b/src/test/regress/expected/multi_tpch_query7_nested.out index 493b3bf1e..d6d6acf71 100644 --- a/src/test/regress/expected/multi_tpch_query7_nested.out +++ b/src/test/regress/expected/multi_tpch_query7_nested.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY7_NESTED -- -- Change configuration to treat lineitem AND orders tables AS large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H benchmark; modified to include sub-selects SELECT supp_nation, diff --git a/src/test/regress/expected/multi_upsert.out b/src/test/regress/expected/multi_upsert.out index 6222c87e8..cc16748d0 100644 --- a/src/test/regress/expected/multi_upsert.out +++ b/src/test/regress/expected/multi_upsert.out @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. CREATE TABLE upsert_test @@ -124,7 +124,7 @@ SELECT master_create_worker_shards('upsert_test_2', '4', '2'); (1 row) --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; -- this errors out since there is no unique constraint on partition key @@ -214,7 +214,7 @@ INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFL ALTER TABLE dropcol_distributed DROP COLUMN drop1; INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFLICT(key) DO UPDATE SET keep1 = dropcol.keep1; --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/expected/multi_upsert_0.out b/src/test/regress/expected/multi_upsert_0.out index 461eb804c..c8c1e705e 100644 --- a/src/test/regress/expected/multi_upsert_0.out +++ b/src/test/regress/expected/multi_upsert_0.out @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. CREATE TABLE upsert_test @@ -157,7 +157,7 @@ SELECT master_create_worker_shards('upsert_test_2', '4', '2'); (1 row) --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; ERROR: syntax error at or near "ON" @@ -284,7 +284,7 @@ INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFL ERROR: syntax error at or near "AS" LINE 1: INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALU... ^ --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/expected/multi_utility_statements.out b/src/test/regress/expected/multi_utility_statements.out index 04317ce81..ca5d7b4ad 100644 --- a/src/test/regress/expected/multi_utility_statements.out +++ b/src/test/regress/expected/multi_utility_statements.out @@ -38,7 +38,7 @@ SELECT * FROM lineitem_pricing_summary ORDER BY l_returnflag, l_linestatus; (4 rows) -- Test we can handle joins -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; CREATE TABLE shipping_priority AS ( SELECT diff --git a/src/test/regress/expected/multi_utility_warnings.out b/src/test/regress/expected/multi_utility_warnings.out index 6b564d473..b10cc7598 100644 --- a/src/test/regress/expected/multi_utility_warnings.out +++ b/src/test/regress/expected/multi_utility_warnings.out @@ -4,15 +4,15 @@ -- Tests to check if we inform the user about potential caveats of creating new -- databases, schemas, and roles. CREATE DATABASE new_database; -NOTICE: CitusDB partially supports CREATE DATABASE for distributed databases -DETAIL: CitusDB does not propagate CREATE DATABASE command to workers +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers HINT: You can manually create a database and its extensions on workers. CREATE SCHEMA new_schema; -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet CREATE ROLE new_role; -NOTICE: CitusDB does not support CREATE ROLE/USER for distributed databases +NOTICE: Citus does not support CREATE ROLE/USER for distributed databases DETAIL: Multiple roles are currently supported only for local tables CREATE USER new_user; -NOTICE: CitusDB does not support CREATE ROLE/USER for distributed databases +NOTICE: Citus does not support CREATE ROLE/USER for distributed databases DETAIL: Multiple roles are currently supported only for local tables diff --git a/src/test/regress/input/multi_agg_distinct.source b/src/test/regress/input/multi_agg_distinct.source index 5d7fa7e53..69e431f5a 100644 --- a/src/test/regress/input/multi_agg_distinct.source +++ b/src/test/regress/input/multi_agg_distinct.source @@ -35,14 +35,14 @@ SELECT avg(distinct l_orderkey) FROM lineitem_range; -- sharded table. For this test, we also change a config setting to ensure that -- we don't repartition any of the tables during the query. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part WHERE l_partkey = p_partkey GROUP BY p_partkey ORDER BY p_partkey LIMIT 10; -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- Check that we don't support count(distinct) on non-partition column, and -- complex expressions. diff --git a/src/test/regress/input/multi_append_table_to_shard.source b/src/test/regress/input/multi_append_table_to_shard.source index 3c33fe34c..04ff276d4 100644 --- a/src/test/regress/input/multi_append_table_to_shard.source +++ b/src/test/regress/input/multi_append_table_to_shard.source @@ -17,16 +17,16 @@ CREATE TABLE multi_append_table_to_shard_left SELECT master_create_distributed_table('multi_append_table_to_shard_left', 'left_number', 'append'); -- Replicate 'left' table on both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' -- Place 'right' table only on the primary worker -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); \STAGE multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' -- Reset shard replication factor to ensure tasks will be assigned to both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); -- All 8 rows in left table match a row in right table SELECT COUNT(*) diff --git a/src/test/regress/input/multi_fdw_stage_data.source b/src/test/regress/input/multi_fdw_stage_data.source index 67e45d351..8597407ca 100644 --- a/src/test/regress/input/multi_fdw_stage_data.source +++ b/src/test/regress/input/multi_fdw_stage_data.source @@ -10,7 +10,7 @@ \STAGE orders FROM '@abs_srcdir@/data/orders.1.data' \STAGE orders FROM '@abs_srcdir@/data/orders.2.data' -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); \STAGE customer FROM '@abs_srcdir@/data/customer.1.data' \STAGE nation FROM '@abs_srcdir@/data/nation.data' diff --git a/src/test/regress/input/multi_outer_join.source b/src/test/regress/input/multi_outer_join.source index 7b46790ce..b0d0fce73 100644 --- a/src/test/regress/input/multi_outer_join.source +++ b/src/test/regress/input/multi_outer_join.source @@ -1,5 +1,5 @@ -SET citusdb.large_table_shard_count TO 2; -SET citusdb.log_multi_join_order to true; +SET citus.large_table_shard_count TO 2; +SET citus.log_multi_join_order to true; SET client_min_messages TO LOG; CREATE TABLE multi_outer_join_left diff --git a/src/test/regress/input/multi_stage_data.source b/src/test/regress/input/multi_stage_data.source index 3e093f9ed..9effe58d5 100644 --- a/src/test/regress/input/multi_stage_data.source +++ b/src/test/regress/input/multi_stage_data.source @@ -3,12 +3,9 @@ -- -- Tests for staging data in a distributed cluster. Please note that the number --- of shards uploaded depends on two config values: citusdb.shard_replication_factor and --- citusdb.shard_max_size. These values are manually set in pg_regress.c. We also set --- the shard placement policy to the local-node-first policy as other regression --- tests expect the placements to be in that order. - -SET citusdb.shard_placement_policy TO 'local-node-first'; +-- of shards uploaded depends on two config values: citus.shard_replication_factor and +-- citus.shard_max_size. These values are set in pg_regress_multi.pl. Shard placement +-- policy is left to the default value (round-robin) to test the common install case. \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' diff --git a/src/test/regress/input/multi_stage_large_records.source b/src/test/regress/input/multi_stage_large_records.source index 5e92bb960..8e015d537 100644 --- a/src/test/regress/input/multi_stage_large_records.source +++ b/src/test/regress/input/multi_stage_large_records.source @@ -6,7 +6,7 @@ -- size, which is 32kB) in a distributed cluster. These tests make sure that we -- are creating shards of correct size even when records are large. -SET citusdb.shard_max_size TO "256kB"; +SET citus.shard_max_size TO "256kB"; CREATE TABLE large_records_table (data_id integer, data text); SELECT master_create_distributed_table('large_records_table', 'data_id', 'append'); @@ -17,4 +17,4 @@ SELECT shardminvalue, shardmaxvalue FROM pg_dist_shard, pg_class WHERE pg_class.oid=logicalrelid AND relname='large_records_table' ORDER BY shardid; -RESET citusdb.shard_max_size; +RESET citus.shard_max_size; diff --git a/src/test/regress/input/multi_stage_more_data.source b/src/test/regress/input/multi_stage_more_data.source index 4b685988a..3ec97813f 100644 --- a/src/test/regress/input/multi_stage_more_data.source +++ b/src/test/regress/input/multi_stage_more_data.source @@ -5,10 +5,6 @@ -- We stage more data to customer and part tables to test distributed joins. The -- staging causes the planner to consider customer and part tables as large, and -- evaluate plans where some of the underlying tables need to be repartitioned. --- We also set the shard placement policy to the local-node-first policy as other --- regression tests expect the placements to be in that order. - -SET citusdb.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' diff --git a/src/test/regress/input/multi_subquery.source b/src/test/regress/input/multi_subquery.source index c4d8bb86f..0accf9135 100644 --- a/src/test/regress/input/multi_subquery.source +++ b/src/test/regress/input/multi_subquery.source @@ -37,7 +37,7 @@ CREATE TABLE orders_subquery ( PRIMARY KEY(o_orderkey) ); SELECT master_create_distributed_table('orders_subquery', 'o_orderkey', 'range'); -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Check that we don't allow subquery pushdown in default settings. @@ -55,7 +55,7 @@ FROM GROUP BY l_orderkey) AS unit_prices; -SET citusdb.subquery_pushdown to TRUE; +SET citus.subquery_pushdown to TRUE; -- Check that we don't crash if there are not any shards. @@ -75,7 +75,7 @@ FROM -- Stage data to tables. -SET citusdb.shard_max_size TO "1MB"; +SET citus.shard_max_size TO "1MB"; \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' diff --git a/src/test/regress/output/multi_agg_distinct.source b/src/test/regress/output/multi_agg_distinct.source index 19510dbd0..060bb39d2 100644 --- a/src/test/regress/output/multi_agg_distinct.source +++ b/src/test/regress/output/multi_agg_distinct.source @@ -43,7 +43,7 @@ SELECT avg(distinct l_orderkey) FROM lineitem_range; -- Run count(distinct) on join between a range partitioned table and a single -- sharded table. For this test, we also change a config setting to ensure that -- we don't repartition any of the tables during the query. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part WHERE l_partkey = p_partkey GROUP BY p_partkey @@ -62,7 +62,7 @@ SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part 222 | 1 (10 rows) -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- Check that we don't support count(distinct) on non-partition column, and -- complex expressions. SELECT count(distinct l_partkey) FROM lineitem_range; diff --git a/src/test/regress/output/multi_append_table_to_shard.source b/src/test/regress/output/multi_append_table_to_shard.source index 62580647c..682671c74 100644 --- a/src/test/regress/output/multi_append_table_to_shard.source +++ b/src/test/regress/output/multi_append_table_to_shard.source @@ -25,7 +25,7 @@ SELECT master_create_distributed_table('multi_append_table_to_shard_left', 'left (1 row) -- Replicate 'left' table on both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); set_config ------------ 2 @@ -34,7 +34,7 @@ SELECT set_config('citusdb.shard_replication_factor', '2', false); \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' -- Place 'right' table only on the primary worker -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); set_config ------------ 1 @@ -42,7 +42,7 @@ SELECT set_config('citusdb.shard_replication_factor', '1', false); \STAGE multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' -- Reset shard replication factor to ensure tasks will be assigned to both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); set_config ------------ 2 diff --git a/src/test/regress/output/multi_create_schema.source b/src/test/regress/output/multi_create_schema.source index 26512abd3..690a309d4 100644 --- a/src/test/regress/output/multi_create_schema.source +++ b/src/test/regress/output/multi_create_schema.source @@ -4,8 +4,8 @@ CREATE TABLE nation ( n_name char(25) not null, n_regionkey integer not null, n_comment varchar(152)); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT master_create_distributed_table('tpch.nation', 'n_nationkey', 'append'); master_create_distributed_table --------------------------------- @@ -13,10 +13,10 @@ SELECT master_create_distributed_table('tpch.nation', 'n_nationkey', 'append'); (1 row) \STAGE tpch.nation FROM '@abs_srcdir@/data/nation.data' with delimiter '|' -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT count(*) from tpch.nation; count ------- diff --git a/src/test/regress/output/multi_fdw_stage_data.source b/src/test/regress/output/multi_fdw_stage_data.source index b32c1321b..3cb781624 100644 --- a/src/test/regress/output/multi_fdw_stage_data.source +++ b/src/test/regress/output/multi_fdw_stage_data.source @@ -14,7 +14,7 @@ NOTICE: extension "file_fdw" already exists, skipping \STAGE orders FROM '@abs_srcdir@/data/orders.2.data' NOTICE: extension "file_fdw" already exists, skipping NOTICE: extension "file_fdw" already exists, skipping -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); set_config ------------ 1 diff --git a/src/test/regress/output/multi_outer_join.source b/src/test/regress/output/multi_outer_join.source index 02e0ce21f..025f258fe 100644 --- a/src/test/regress/output/multi_outer_join.source +++ b/src/test/regress/output/multi_outer_join.source @@ -1,5 +1,5 @@ -SET citusdb.large_table_shard_count TO 2; -SET citusdb.log_multi_join_order to true; +SET citus.large_table_shard_count TO 2; +SET citus.log_multi_join_order to true; SET client_min_messages TO LOG; CREATE TABLE multi_outer_join_left ( diff --git a/src/test/regress/output/multi_stage_data.source b/src/test/regress/output/multi_stage_data.source index 58a8b15a0..0eed8965f 100644 --- a/src/test/regress/output/multi_stage_data.source +++ b/src/test/regress/output/multi_stage_data.source @@ -2,11 +2,9 @@ -- MULTI_STAGE_DATA -- -- Tests for staging data in a distributed cluster. Please note that the number --- of shards uploaded depends on two config values: citusdb.shard_replication_factor and --- citusdb.shard_max_size. These values are manually set in pg_regress.c. We also set --- the shard placement policy to the local-node-first policy as other regression --- tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; +-- of shards uploaded depends on two config values: citus.shard_replication_factor and +-- citus.shard_max_size. These values are set in pg_regress_multi.pl. Shard placement +-- policy is left to the default value (round-robin) to test the common install case. \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' \STAGE orders FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' diff --git a/src/test/regress/output/multi_stage_large_records.source b/src/test/regress/output/multi_stage_large_records.source index a8e7549f3..24eaa14e6 100644 --- a/src/test/regress/output/multi_stage_large_records.source +++ b/src/test/regress/output/multi_stage_large_records.source @@ -4,7 +4,7 @@ -- Tests for staging data with large records (i.e. greater than the read buffer -- size, which is 32kB) in a distributed cluster. These tests make sure that we -- are creating shards of correct size even when records are large. -SET citusdb.shard_max_size TO "256kB"; +SET citus.shard_max_size TO "256kB"; CREATE TABLE large_records_table (data_id integer, data text); SELECT master_create_distributed_table('large_records_table', 'data_id', 'append'); master_create_distributed_table @@ -22,4 +22,4 @@ SELECT shardminvalue, shardmaxvalue FROM pg_dist_shard, pg_class 2 | 2 (2 rows) -RESET citusdb.shard_max_size; +RESET citus.shard_max_size; diff --git a/src/test/regress/output/multi_stage_more_data.source b/src/test/regress/output/multi_stage_more_data.source index d4b95c01d..2e0f38f1d 100644 --- a/src/test/regress/output/multi_stage_more_data.source +++ b/src/test/regress/output/multi_stage_more_data.source @@ -4,9 +4,6 @@ -- We stage more data to customer and part tables to test distributed joins. The -- staging causes the planner to consider customer and part tables as large, and -- evaluate plans where some of the underlying tables need to be repartitioned. --- We also set the shard placement policy to the local-node-first policy as other --- regression tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' \STAGE part FROM '@abs_srcdir@/data/part.more.data' with delimiter '|' diff --git a/src/test/regress/output/multi_subquery.source b/src/test/regress/output/multi_subquery.source index 84e38aa5c..f0e6f0250 100644 --- a/src/test/regress/output/multi_subquery.source +++ b/src/test/regress/output/multi_subquery.source @@ -43,7 +43,7 @@ SELECT master_create_distributed_table('orders_subquery', 'o_orderkey', 'range') (1 row) -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Check that we don't allow subquery pushdown in default settings. SELECT avg(unit_price) @@ -60,7 +60,7 @@ FROM l_orderkey) AS unit_prices; ERROR: cannot perform distributed planning on this query DETAIL: Join in subqueries is not supported yet -SET citusdb.subquery_pushdown to TRUE; +SET citus.subquery_pushdown to TRUE; -- Check that we don't crash if there are not any shards. SELECT avg(unit_price) @@ -81,7 +81,7 @@ FROM (1 row) -- Stage data to tables. -SET citusdb.shard_max_size TO "1MB"; +SET citus.shard_max_size TO "1MB"; \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' \STAGE orders_subquery FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index e1f106f90..aa863021e 100644 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -w #---------------------------------------------------------------------- # -# pg_regress_multi.pl - Test runner for CitusDB +# pg_regress_multi.pl - Test runner for Citus # # Portions Copyright (c) 2012-2015, Citus Data, Inc. # Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group @@ -19,7 +19,7 @@ use Getopt::Long; sub Usage() { - print "pg_regress_multi - CitusDB test runner\n"; + print "pg_regress_multi - Citus test runner\n"; print "\n"; print "Usage:\n"; print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n"; @@ -56,7 +56,7 @@ GetOptions( # Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc - # pointing to the libdir - that's required so the right version of -# libpq, citusdb et al is being picked up. +# libpq, citus et al is being picked up. # # XXX: There's some issues with el capitan's SIP here, causing # DYLD_LIBRARY_PATH not being inherited if SIP is enabled. That's a @@ -87,12 +87,12 @@ push(@pgOptions, '-c', "listen_addresses='${host}'"); # not required, and we don't necessarily have access to the default directory push(@pgOptions, '-c', "unix_socket_directories="); push(@pgOptions, '-c', "fsync=off"); -push(@pgOptions, '-c', "shared_preload_libraries=citusdb"); +push(@pgOptions, '-c', "shared_preload_libraries=citus"); -# CitusDB options set for the tests -push(@pgOptions, '-c', "citusdb.shard_max_size=300kB"); -push(@pgOptions, '-c', "citusdb.max_running_tasks_per_node=4"); -push(@pgOptions, '-c', "citusdb.expire_cached_shards=on"); +# Citus options set for the tests +push(@pgOptions, '-c', "citus.shard_max_size=300kB"); +push(@pgOptions, '-c', "citus.max_running_tasks_per_node=4"); +push(@pgOptions, '-c', "citus.expire_cached_shards=on"); # Add externally added options last, so they overwrite the default ones above for my $option (@userPgOptions) @@ -107,7 +107,7 @@ for my $option (@userPgOptions) 'bug_status', ' ENUM (\'new\', \'open\', \'closed\')'); # define functions as signature->definition -%functions = ('fake_fdw_handler()', 'fdw_handler AS \'citusdb\' LANGUAGE C STRICT;'); +%functions = ('fake_fdw_handler()', 'fdw_handler AS \'citus\' LANGUAGE C STRICT;'); #define fdws as name->handler name %fdws = ('fake_fdw', 'fake_fdw_handler'); @@ -274,7 +274,7 @@ elsif ($majorversion eq '9.4') } else { - die "CitusDB is not compatible with the detected PostgreSQL version $majorversion"; + die "Citus is not compatible with the detected PostgreSQL version $majorversion"; } # Add load extension parameters to the argument list diff --git a/src/test/regress/sql/multi_agg_approximate_distinct.sql b/src/test/regress/sql/multi_agg_approximate_distinct.sql index 386ca3abb..5ed4d527c 100644 --- a/src/test/regress/sql/multi_agg_approximate_distinct.sql +++ b/src/test/regress/sql/multi_agg_approximate_distinct.sql @@ -8,10 +8,10 @@ SELECT count(distinct l_orderkey) FROM lineitem; -- Check approximate count(distinct) at different precisions / error rates -SET citusdb.count_distinct_error_rate = 0.1; +SET citus.count_distinct_error_rate = 0.1; SELECT count(distinct l_orderkey) FROM lineitem; -SET citusdb.count_distinct_error_rate = 0.01; +SET citus.count_distinct_error_rate = 0.01; SELECT count(distinct l_orderkey) FROM lineitem; -- Check approximate count(distinct) for different data types @@ -50,7 +50,7 @@ SELECT count(DISTINCT l_orderkey) as distinct_order_count, l_quantity FROM linei -- If we have an order by on count(distinct) that we intend to push down to -- worker nodes, we need to error out. Otherwise, we are fine. -SET citusdb.limit_clause_row_fetch_count = 1000; +SET citus.limit_clause_row_fetch_count = 1000; SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as total FROM lineitem GROUP BY l_returnflag @@ -65,5 +65,5 @@ SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as t -- Check that we can revert config and disable count(distinct) approximations -SET citusdb.count_distinct_error_rate = 0.0; +SET citus.count_distinct_error_rate = 0.0; SELECT count(distinct l_orderkey) FROM lineitem; diff --git a/src/test/regress/sql/multi_binary_master_copy_format.sql b/src/test/regress/sql/multi_binary_master_copy_format.sql index 37623ce36..4c67eb2d2 100644 --- a/src/test/regress/sql/multi_binary_master_copy_format.sql +++ b/src/test/regress/sql/multi_binary_master_copy_format.sql @@ -4,13 +4,13 @@ -- Try binary master copy for different executors -SET citusdb.binary_master_copy_format TO 'on'; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.binary_master_copy_format TO 'on'; +SET citus.task_executor_type TO 'task-tracker'; SELECT count(*) FROM lineitem; SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; SELECT count(*) FROM lineitem; SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; diff --git a/src/test/regress/sql/multi_connection_cache.sql b/src/test/regress/sql/multi_connection_cache.sql index 33bad147e..ea1bdff7d 100644 --- a/src/test/regress/sql/multi_connection_cache.sql +++ b/src/test/regress/sql/multi_connection_cache.sql @@ -4,22 +4,22 @@ CREATE FUNCTION initialize_remote_temp_table(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION count_remote_temp_table_rows(cstring, integer) RETURNS integer - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION get_and_purge_connection(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION set_connection_status_bad(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_create_fdw.sql b/src/test/regress/sql/multi_create_fdw.sql index 0f237be11..dd828a66b 100644 --- a/src/test/regress/sql/multi_create_fdw.sql +++ b/src/test/regress/sql/multi_create_fdw.sql @@ -5,7 +5,7 @@ -- create fake fdw for use in tests CREATE FUNCTION fake_fdw_handler() RETURNS fdw_handler -AS 'citusdb' +AS 'citus' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER fake_fdw HANDLER fake_fdw_handler; diff --git a/src/test/regress/sql/multi_create_shards.sql b/src/test/regress/sql/multi_create_shards.sql index 5b51dccd2..1180338ac 100644 --- a/src/test/regress/sql/multi_create_shards.sql +++ b/src/test/regress/sql/multi_create_shards.sql @@ -4,7 +4,7 @@ CREATE FUNCTION sort_names(cstring, cstring, cstring) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- create a custom type... @@ -125,7 +125,7 @@ CREATE TABLE weird_shard_count SELECT master_create_distributed_table('weird_shard_count', 'id', 'hash'); SELECT master_create_worker_shards('weird_shard_count', 7, 1); --- CitusDB ensures all shards are roughly the same size +-- Citus ensures all shards are roughly the same size SELECT shardmaxvalue::integer - shardminvalue::integer AS shard_size FROM pg_dist_shard WHERE logicalrelid = 'weird_shard_count'::regclass diff --git a/src/test/regress/sql/multi_create_table.sql b/src/test/regress/sql/multi_create_table.sql index 987988771..3509436f6 100644 --- a/src/test/regress/sql/multi_create_table.sql +++ b/src/test/regress/sql/multi_create_table.sql @@ -83,7 +83,7 @@ CREATE TABLE supplier SELECT master_create_distributed_table('supplier', 's_suppkey', 'append'); --- now test that CitusDB cannot distribute unique constraints that do not include +-- now test that Citus cannot distribute unique constraints that do not include -- the partition column CREATE TABLE primary_key_on_non_part_col ( @@ -99,7 +99,7 @@ CREATE TABLE unique_const_on_non_part_col ); SELECT master_create_distributed_table('primary_key_on_non_part_col', 'partition_col', 'hash'); --- now show that CitusDB can distribute unique constrints that include +-- now show that Citus can distribute unique constrints that include -- the partition column CREATE TABLE primary_key_on_part_col ( diff --git a/src/test/regress/sql/multi_data_types.sql b/src/test/regress/sql/multi_data_types.sql index 99c603e63..68b84044f 100644 --- a/src/test/regress/sql/multi_data_types.sql +++ b/src/test/regress/sql/multi_data_types.sql @@ -104,7 +104,7 @@ CREATE TABLE varchar_hash_partitioned_table name varchar ); -SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'id', 'hash'); +SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'name', 'hash'); SELECT master_create_worker_shards('varchar_hash_partitioned_table', 4, 1); -- execute INSERT, SELECT and UPDATE queries on composite_type_partitioned_table @@ -114,8 +114,8 @@ INSERT INTO varchar_hash_partitioned_table VALUES (3, 'Onder'); INSERT INTO varchar_hash_partitioned_table VALUES (4, 'Sumedh'); INSERT INTO varchar_hash_partitioned_table VALUES (5, 'Marco'); -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Onder'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 1; -UPDATE varchar_hash_partitioned_table SET name = 'Samay' WHERE id = 5; +UPDATE varchar_hash_partitioned_table SET id = 6 WHERE name = 'Jason'; -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Samay'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 6; diff --git a/src/test/regress/sql/multi_distribution_metadata.sql b/src/test/regress/sql/multi_distribution_metadata.sql index 51b3fe2ee..d53d88c14 100644 --- a/src/test/regress/sql/multi_distribution_metadata.sql +++ b/src/test/regress/sql/multi_distribution_metadata.sql @@ -4,67 +4,67 @@ CREATE FUNCTION load_shard_id_array(regclass) RETURNS bigint[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_interval_array(bigint, anyelement) RETURNS anyarray - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_placement_array(bigint, bool) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_column_id(regclass) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_type(regclass) RETURNS "char" - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION is_distributed_table(regclass) RETURNS boolean - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column_id(regclass, cstring) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_monolithic_shard_row(regclass) RETURNS bigint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_healthy_local_shard_placement_row(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION delete_shard_placement_row(bigint, text, bigint) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION update_shard_placement_row_state(bigint, text, bigint, int) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION acquire_shared_shard_lock(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column(regclass, text) RETURNS text - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_generate_ddl_commands.sql b/src/test/regress/sql/multi_generate_ddl_commands.sql index 5e7c4be99..0bd2c28fa 100644 --- a/src/test/regress/sql/multi_generate_ddl_commands.sql +++ b/src/test/regress/sql/multi_generate_ddl_commands.sql @@ -4,7 +4,7 @@ CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_join_order_additional.sql b/src/test/regress/sql/multi_join_order_additional.sql index 678d9077c..c1668f061 100644 --- a/src/test/regress/sql/multi_join_order_additional.sql +++ b/src/test/regress/sql/multi_join_order_additional.sql @@ -4,7 +4,7 @@ -- Set configuration to print table join order and pruned shards -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO DEBUG2; -- The following query checks that we can correctly handle self-joins @@ -14,7 +14,7 @@ EXPLAIN SELECT l1.l_quantity FROM lineitem l1, lineitem l2 -- Update configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO LOG; -- The following queries check that we correctly handle joins and OR clauses. In @@ -66,7 +66,7 @@ UPDATE pg_dist_partition SET partmethod = 'a' WHERE logicalrelid = (SELECT relfilenode FROM pg_class WHERE relname = 'customer'); -- Update the large table shard count for all the following tests. -SET citusdb.large_table_shard_count TO 1; +SET citus.large_table_shard_count TO 1; -- Validate that we don't use a single-partition join method for a hash -- re-partitioned table, thus preventing a partition of just the customer table. diff --git a/src/test/regress/sql/multi_join_order_tpch_large.sql b/src/test/regress/sql/multi_join_order_tpch_large.sql index 7b0431fd8..a0d1e5a9f 100644 --- a/src/test/regress/sql/multi_join_order_tpch_large.sql +++ b/src/test/regress/sql/multi_join_order_tpch_large.sql @@ -4,7 +4,7 @@ -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem, orders, customer, and part tables as @@ -12,7 +12,7 @@ SET client_min_messages TO LOG; -- except that more data has been staged to customer and part tables. Therefore, -- we will apply different distributed join strategies for these queries. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_join_order_tpch_small.sql b/src/test/regress/sql/multi_join_order_tpch_small.sql index 545bab32c..b4663cbe9 100644 --- a/src/test/regress/sql/multi_join_order_tpch_small.sql +++ b/src/test/regress/sql/multi_join_order_tpch_small.sql @@ -4,12 +4,12 @@ -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_join_pruning.sql b/src/test/regress/sql/multi_join_pruning.sql index 29df48095..293b05f4c 100644 --- a/src/test/regress/sql/multi_join_pruning.sql +++ b/src/test/regress/sql/multi_join_pruning.sql @@ -10,7 +10,7 @@ SET client_min_messages TO DEBUG2; -- Change configuration to treat all tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders WHERE l_orderkey = o_orderkey; diff --git a/src/test/regress/sql/multi_large_table_join_planning.sql b/src/test/regress/sql/multi_large_table_join_planning.sql index 06a914361..8ad2a3fe6 100644 --- a/src/test/regress/sql/multi_large_table_join_planning.sql +++ b/src/test/regress/sql/multi_large_table_join_planning.sql @@ -9,8 +9,8 @@ BEGIN; SET client_min_messages TO DEBUG4; -SET citusdb.large_table_shard_count TO 2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'task-tracker'; -- Debug4 log messages display jobIds within them. We explicitly set the jobId -- sequence here so that the regression output becomes independent of the number diff --git a/src/test/regress/sql/multi_large_table_pruning.sql b/src/test/regress/sql/multi_large_table_pruning.sql index 897aa73a7..ec46eff0c 100644 --- a/src/test/regress/sql/multi_large_table_pruning.sql +++ b/src/test/regress/sql/multi_large_table_pruning.sql @@ -6,9 +6,9 @@ -- set executor type to task tracker executor here, as we cannot run repartition -- jobs with real time executor. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO DEBUG2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Single range-repartition join to test join-pruning behaviour. diff --git a/src/test/regress/sql/multi_large_table_task_assignment.sql b/src/test/regress/sql/multi_large_table_task_assignment.sql index 8fd902950..0793e4492 100644 --- a/src/test/regress/sql/multi_large_table_task_assignment.sql +++ b/src/test/regress/sql/multi_large_table_task_assignment.sql @@ -9,8 +9,8 @@ BEGIN; SET client_min_messages TO DEBUG3; -SET citusdb.large_table_shard_count TO 2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'task-tracker'; -- Single range repartition join to test anchor-shard based task assignment and -- assignment propagation to merge and data-fetch tasks. @@ -27,7 +27,7 @@ WHERE -- the same merge task, and tests our constraint group creation and assignment -- propagation. Here 'orders' is considered the small table. -SET citusdb.large_table_shard_count TO 3; +SET citus.large_table_shard_count TO 3; SELECT count(*) @@ -37,7 +37,7 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- The next test, dual hash repartition join, uses the current jobId to assign -- tasks in a round-robin fashion. We therefore need to ensure that jobIds start diff --git a/src/test/regress/sql/multi_limit_clause_approximate.sql b/src/test/regress/sql/multi_limit_clause_approximate.sql index 8daca0930..659e91ab6 100644 --- a/src/test/regress/sql/multi_limit_clause_approximate.sql +++ b/src/test/regress/sql/multi_limit_clause_approximate.sql @@ -16,7 +16,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem -- Enable limit optimization to fetch one third of each shard's data -SET citusdb.limit_clause_row_fetch_count TO 600; +SET citus.limit_clause_row_fetch_count TO 600; SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem GROUP BY l_partkey @@ -25,7 +25,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem -- Disable limit optimization for our second test. This time, we have a query -- that joins several tables, and that groups and orders the results. -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem @@ -37,8 +37,8 @@ SELECT c_custkey, c_name, count(*) as lineitem_count -- test, we also change a config setting to ensure that we don't repartition any -- of the tables during the query. -SET citusdb.limit_clause_row_fetch_count TO 150; -SET citusdb.large_table_shard_count TO 2; +SET citus.limit_clause_row_fetch_count TO 150; +SET citus.large_table_shard_count TO 2; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem @@ -46,7 +46,7 @@ SELECT c_custkey, c_name, count(*) as lineitem_count GROUP BY c_custkey, c_name ORDER BY lineitem_count DESC, c_custkey LIMIT 10; -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- We now test scenarios where applying the limit optimization wouldn't produce -- meaningful results. First, we check that we don't push down the limit clause @@ -69,5 +69,5 @@ SELECT count(*) count_quantity, l_quantity FROM lineitem WHERE l_quantity < 10.0 GROUP BY l_quantity ORDER BY count_quantity ASC, l_quantity ASC; -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; RESET client_min_messages; diff --git a/src/test/regress/sql/multi_modifications.sql b/src/test/regress/sql/multi_modifications.sql index 040411079..605f8905f 100644 --- a/src/test/regress/sql/multi_modifications.sql +++ b/src/test/regress/sql/multi_modifications.sql @@ -40,10 +40,15 @@ SELECT master_create_empty_shard('range_partitioned') AS new_shard_id UPDATE pg_dist_shard SET shardminvalue = 50000, shardmaxvalue = 99999 WHERE shardid = :new_shard_id; --- create append-partitioned shard +-- create append-partitioned shards SELECT master_create_empty_shard('append_partitioned') AS new_shard_id \gset -UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 100000 +UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 500000 +WHERE shardid = :new_shard_id; + +SELECT master_create_empty_shard('append_partitioned') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 500000, shardmaxvalue = 1000000 WHERE shardid = :new_shard_id; -- basic single-row INSERT @@ -59,19 +64,23 @@ INSERT INTO insufficient_shards VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:5 INSERT INTO range_partitioned VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- ensure the value is where we put it and query to find it is properly pruned +-- also insert to an append-partitioned table +INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', + 20.69); +-- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; -SET citusdb.task_executor_type TO 'router'; +SET citus.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; +SELECT * FROM append_partitioned WHERE id = 414123; SET client_min_messages TO DEFAULT; -SET citusdb.task_executor_type TO DEFAULT; +SET citus.task_executor_type TO DEFAULT; --- also try inserting without a range-partitioned shard to receive the value +-- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- try an insert to an append-partitioned table -INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', +-- and insert into an append-partitioned table with a value that spans shards: +INSERT INTO append_partitioned VALUES (500000, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); -- INSERT with DEFAULT in the target list diff --git a/src/test/regress/sql/multi_null_minmax_value_pruning.sql b/src/test/regress/sql/multi_null_minmax_value_pruning.sql index 91828359d..9604f9ab4 100644 --- a/src/test/regress/sql/multi_null_minmax_value_pruning.sql +++ b/src/test/regress/sql/multi_null_minmax_value_pruning.sql @@ -9,7 +9,7 @@ SET client_min_messages TO DEBUG2; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102009; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102010; diff --git a/src/test/regress/sql/multi_prepare_plsql.sql b/src/test/regress/sql/multi_prepare_plsql.sql index 2a91d5483..f62e256df 100644 --- a/src/test/regress/sql/multi_prepare_plsql.sql +++ b/src/test/regress/sql/multi_prepare_plsql.sql @@ -187,7 +187,7 @@ BEGIN END; $$ LANGUAGE plpgsql; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- now, run plain SQL functions @@ -222,7 +222,7 @@ SELECT plpgsql_test_2(); -- run the tests which do not require re-partition -- with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- now, run plain SQL functions SELECT sql_test_no_1(); diff --git a/src/test/regress/sql/multi_prepare_sql.sql b/src/test/regress/sql/multi_prepare_sql.sql index 64a1c0b10..346b8de6a 100644 --- a/src/test/regress/sql/multi_prepare_sql.sql +++ b/src/test/regress/sql/multi_prepare_sql.sql @@ -104,7 +104,7 @@ ORDER BY cust_nation, l_year; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- execute prepared statements @@ -136,7 +136,7 @@ CREATE TEMP TABLE prepared_sql_test_7 AS EXECUTE prepared_test_7('UNITED KINGDOM SELECT * from prepared_sql_test_7; -- now, run some of the tests with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- execute prepared statements EXECUTE prepared_test_1; diff --git a/src/test/regress/sql/multi_prune_shard_list.sql b/src/test/regress/sql/multi_prune_shard_list.sql index f684bfa83..2a215c5a5 100644 --- a/src/test/regress/sql/multi_prune_shard_list.sql +++ b/src/test/regress/sql/multi_prune_shard_list.sql @@ -4,27 +4,27 @@ CREATE FUNCTION prune_using_no_values(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_single_value(regclass, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C; CREATE FUNCTION prune_using_either_value(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_both_values(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION debug_equality_expression(regclass) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_simple_queries.sql b/src/test/regress/sql/multi_simple_queries.sql index 1e774ebb6..45fc1ebf2 100644 --- a/src/test/regress/sql/multi_simple_queries.sql +++ b/src/test/regress/sql/multi_simple_queries.sql @@ -112,7 +112,7 @@ SELECT title, author_id FROM articles ORDER BY author_id ASC, id; -- add in some grouping expressions, still on same shard --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles WHERE author_id = 1 OR author_id = 7 OR author_id = 8 OR author_id = 10 GROUP BY author_id @@ -130,7 +130,7 @@ SELECT title FROM articles; -- queries which involve functions in FROM clause are unsupported. SELECT * FROM articles, position('om' in 'Thomas'); --- subqueries are not supported in WHERE clause in CitusDB +-- subqueries are not supported in WHERE clause in Citus SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); -- subqueries are supported in FROM clause @@ -166,23 +166,23 @@ $sharded_sql$; -- test cross-shard queries SELECT COUNT(*) FROM articles; --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles GROUP BY author_id HAVING sum(word_count) > 25000 ORDER BY sum(word_count) DESC LIMIT 5; --- more proof CitusDB doesn't support having clauses +-- more proof Citus doesn't support having clauses SELECT author_id FROM articles GROUP BY author_id HAVING sum(word_count) > 50000 ORDER BY author_id; --- now, test the cases where CitusDB do or do not need to create +-- now, test the cases where Citus do or do not need to create -- the master queries -SET citusdb.task_executor_type TO 'router'; -SET citusdb.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'router'; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO 'DEBUG2'; -- start with the simple lookup query @@ -277,4 +277,4 @@ SELECT * WHERE author_id >= 1 AND author_id <= 3; SET client_min_messages to 'NOTICE'; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; diff --git a/src/test/regress/sql/multi_single_relation_subquery.sql b/src/test/regress/sql/multi_single_relation_subquery.sql index 19e56f60e..a3fb1787e 100644 --- a/src/test/regress/sql/multi_single_relation_subquery.sql +++ b/src/test/regress/sql/multi_single_relation_subquery.sql @@ -4,7 +4,7 @@ -- This test checks that we are able to run selected set of distributed SQL subqueries. -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; select number_sum, @@ -125,6 +125,18 @@ from group by l_tax) as distributed_table; +-- Check that we don't support subqueries with count(distinct). + +select + different_shipment_days +from + (select + count(distinct l_shipdate) as different_shipment_days + from + lineitem + group by + l_partkey) as distributed_table; + -- Check that if subquery is pulled, we don't error and run query properly. SELECT max(l_suppkey) FROM diff --git a/src/test/regress/sql/multi_table_ddl.sql b/src/test/regress/sql/multi_table_ddl.sql index f820ce70d..b5eb44bcf 100644 --- a/src/test/regress/sql/multi_table_ddl.sql +++ b/src/test/regress/sql/multi_table_ddl.sql @@ -6,8 +6,8 @@ CREATE TABLE testtableddl(somecol int, distributecol text NOT NULL); SELECT master_create_distributed_table('testtableddl', 'distributecol', 'append'); --- verify that the citusdb extension can't be dropped while distributed tables exist -DROP EXTENSION citusdb; +-- verify that the citus extension can't be dropped while distributed tables exist +DROP EXTENSION citus; -- verify that the distribution column can't have its type changed ALTER TABLE testtableddl ALTER COLUMN distributecol TYPE text; @@ -37,6 +37,6 @@ SELECT * FROM pg_dist_shard_placement; -- check that the extension now can be dropped (and recreated). We reconnect -- before creating the extension to expire extension specific variables which -- are cached for performance. -DROP EXTENSION citusdb; +DROP EXTENSION citus; \c -CREATE EXTENSION citusdb; +CREATE EXTENSION citus; diff --git a/src/test/regress/sql/multi_task_assignment_policy.sql b/src/test/regress/sql/multi_task_assignment_policy.sql index 935c13558..5ee4cddfc 100644 --- a/src/test/regress/sql/multi_task_assignment_policy.sql +++ b/src/test/regress/sql/multi_task_assignment_policy.sql @@ -55,7 +55,7 @@ SET client_min_messages TO DEBUG3; -- First test the default greedy task assignment policy -SET citusdb.task_assignment_policy TO 'greedy'; +SET citus.task_assignment_policy TO 'greedy'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -63,7 +63,7 @@ EXPLAIN SELECT count(*) FROM task_assignment_test_table; -- Next test the first-replica task assignment policy -SET citusdb.task_assignment_policy TO 'first-replica'; +SET citus.task_assignment_policy TO 'first-replica'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -80,7 +80,7 @@ SELECT case when (currval('pg_dist_jobid_seq') % 2) = 0 -- Finally test the round-robin task assignment policy -SET citusdb.task_assignment_policy TO 'round-robin'; +SET citus.task_assignment_policy TO 'round-robin'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -88,7 +88,7 @@ EXPLAIN SELECT count(*) FROM task_assignment_test_table; EXPLAIN SELECT count(*) FROM task_assignment_test_table; -RESET citusdb.task_assignment_policy; +RESET citus.task_assignment_policy; RESET client_min_messages; COMMIT; diff --git a/src/test/regress/sql/multi_tpch_query1.sql b/src/test/regress/sql/multi_tpch_query1.sql index f79aa6586..d6727d15a 100644 --- a/src/test/regress/sql/multi_tpch_query1.sql +++ b/src/test/regress/sql/multi_tpch_query1.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #1 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query10.sql b/src/test/regress/sql/multi_tpch_query10.sql index 581647403..75bb475a5 100644 --- a/src/test/regress/sql/multi_tpch_query10.sql +++ b/src/test/regress/sql/multi_tpch_query10.sql @@ -3,7 +3,7 @@ -- -- Query #10 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT diff --git a/src/test/regress/sql/multi_tpch_query12.sql b/src/test/regress/sql/multi_tpch_query12.sql index fd85c82c0..380165bb2 100644 --- a/src/test/regress/sql/multi_tpch_query12.sql +++ b/src/test/regress/sql/multi_tpch_query12.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #12 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query14.sql b/src/test/regress/sql/multi_tpch_query14.sql index 83cf9e954..82b696fec 100644 --- a/src/test/regress/sql/multi_tpch_query14.sql +++ b/src/test/regress/sql/multi_tpch_query14.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #14 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query19.sql b/src/test/regress/sql/multi_tpch_query19.sql index 32255e74c..dfff2ea5d 100644 --- a/src/test/regress/sql/multi_tpch_query19.sql +++ b/src/test/regress/sql/multi_tpch_query19.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #19 from the TPC-H decision support benchmark. Note that we modified -- the query from its original to make it work on smaller data sets. diff --git a/src/test/regress/sql/multi_tpch_query3.sql b/src/test/regress/sql/multi_tpch_query3.sql index e23ba3329..62232e5fa 100644 --- a/src/test/regress/sql/multi_tpch_query3.sql +++ b/src/test/regress/sql/multi_tpch_query3.sql @@ -3,7 +3,7 @@ -- -- Query #3 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT diff --git a/src/test/regress/sql/multi_tpch_query6.sql b/src/test/regress/sql/multi_tpch_query6.sql index 7a31c47fe..994b9467b 100644 --- a/src/test/regress/sql/multi_tpch_query6.sql +++ b/src/test/regress/sql/multi_tpch_query6.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query7.sql b/src/test/regress/sql/multi_tpch_query7.sql index 875c1e86a..b9b3c5cd3 100644 --- a/src/test/regress/sql/multi_tpch_query7.sql +++ b/src/test/regress/sql/multi_tpch_query7.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem AND orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query7_nested.sql b/src/test/regress/sql/multi_tpch_query7_nested.sql index fd93a17c9..ad34dd7b5 100644 --- a/src/test/regress/sql/multi_tpch_query7_nested.sql +++ b/src/test/regress/sql/multi_tpch_query7_nested.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem AND orders tables AS large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H benchmark; modified to include sub-selects diff --git a/src/test/regress/sql/multi_upsert.sql b/src/test/regress/sql/multi_upsert.sql index 3e47862c6..b61833eaf 100644 --- a/src/test/regress/sql/multi_upsert.sql +++ b/src/test/regress/sql/multi_upsert.sql @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. @@ -93,7 +93,7 @@ CREATE TABLE upsert_test_2 SELECT master_create_distributed_table('upsert_test_2', 'part_key', 'hash'); SELECT master_create_worker_shards('upsert_test_2', '4', '2'); --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; @@ -165,7 +165,7 @@ ALTER TABLE dropcol_distributed DROP COLUMN drop1; INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFLICT(key) DO UPDATE SET keep1 = dropcol.keep1; --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/sql/multi_utility_statements.sql b/src/test/regress/sql/multi_utility_statements.sql index ddf0b37a5..ad67f2b36 100644 --- a/src/test/regress/sql/multi_utility_statements.sql +++ b/src/test/regress/sql/multi_utility_statements.sql @@ -35,7 +35,7 @@ SELECT * FROM lineitem_pricing_summary ORDER BY l_returnflag, l_linestatus; -- Test we can handle joins -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; CREATE TABLE shipping_priority AS (