external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into improve-mitmproxy-documentation

pull/2602/head
Hanefi Onaldi 2019-03-12 07:16:01 -07:00 committed by GitHub
parent c5c3d6d0a3 e813df4d7f
commit 419f52884f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
83 changed files with 5998 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
.circleci/config.yml Normal file
View File

@ -0,0 +1,96 @@
version: 2.1
orbs:
codecov: codecov/codecov@1.0.4
jobs:
build:
docker:
- {image: 'citusdata/extbuilder:latest'}
steps:
- checkout
- {run: {name: 'Configure, Build, and Install', command: build-ext}}
- {persist_to_workspace: {root: ., paths: [.]}}
check-style:
docker:
- {image: 'citusdata/stylechecker:latest'}
steps:
- checkout
- {run: {name: 'Check Style', command: citus_indent --check}}
test-10_check-multi:
docker:
- {image: 'citusdata/exttester-10:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}}
- {codecov/upload: {flags: 'test_10,multi'}}
test-10_check-tt-van-mx:
docker:
- {image: 'citusdata/exttester-10:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}}
- {codecov/upload: {flags: 'test_10,tracker,vanilla,mx'}}
test-10_check-iso-work-fol:
docker:
- {image: 'citusdata/exttester-10:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}}
- {codecov/upload: {flags: 'test_10,isolation,worker,follower'}}
test-10_check-failure:
docker:
- {image: 'citusdata/failtester-10:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}}
test-11_check-multi:
docker:
- {image: 'citusdata/exttester-11:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-multi)', command: 'install-and-test-ext check-multi'}}
- {codecov/upload: {flags: 'test_11,multi'}}
test-11_check-tt-van-mx:
docker:
- {image: 'citusdata/exttester-11:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-tt-van-mx)', command: 'install-and-test-ext check-multi-task-tracker-extra check-vanilla check-multi-mx'}}
- {codecov/upload: {flags: 'test_11,tracker,vanilla,mx'}}
test-11_check-iso-work-fol:
docker:
- {image: 'citusdata/exttester-11:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-iso-work-fol)', command: 'install-and-test-ext check-isolation check-worker check-follower-cluster'}}
- {codecov/upload: {flags: 'test_11,isolation,worker,follower'}}
test-11_check-failure:
docker:
- {image: 'citusdata/failtester-11:latest'}
working_directory: /home/circleci/project
steps:
- {attach_workspace: {at: .}}
- {run: {name: 'Install and Test (check-failure)', command: 'install-and-test-ext check-failure'}}
workflows:
version: 2
build_and_test:
jobs:
- build
- check-style
- {test-10_check-multi: {requires: [build]}}
- {test-10_check-tt-van-mx: {requires: [build]}}
- {test-10_check-iso-work-fol: {requires: [build]}}
- {test-10_check-failure: {requires: [build]}}
- {test-11_check-multi: {requires: [build]}}
- {test-11_check-tt-van-mx: {requires: [build]}}
- {test-11_check-iso-work-fol: {requires: [build]}}
- {test-11_check-failure: {requires: [build]}}

3
.travis.yml
View File

@ -49,8 +49,7 @@ install:
apt-get download "postgresql-${PGVERSION}-topn=2.2.0"
sudo dpkg --force-confold --force-confdef --force-all -i *topn*.deb
fi
before_script: citus_indent --quiet --check
before_script: citus_indent --quiet --check || echo 'Ignoring indent failures'
script: CFLAGS=-Werror pg_travis_multi_test check
after_success:
- sync_to_enterprise
- bash <(curl -s https://codecov.io/bash)

2
Makefile.global.in
View File

@ -69,7 +69,7 @@ endif
# Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/...
override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@
override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS)
override CPPFLAGS := @CPPFLAGS@ @CITUS_CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS)
override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@
# optional file with user defined, additional, rules

9
configure vendored
View File

@ -625,6 +625,7 @@ LIBOBJS
POSTGRES_BUILDDIR
POSTGRES_SRCDIR
CITUS_LDFLAGS
CITUS_CPPFLAGS
CITUS_CFLAGS
EGREP
GREP
@ -4052,7 +4053,9 @@ if test "${enable_coverage+set}" = set; then :
fi
if test "$enable_coverage" = yes; then
CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage"
CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage"
CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG"
CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
fi
#
@ -4183,7 +4186,9 @@ _ACEOF
CITUS_CFLAGS="$CITUS_CFLAGS"
CITUS_LDFLAGS="$LIBS"
CITUS_CPPFLAGS="$CITUS_CPPFLAGS"
CITUS_LDFLAGS="$LIBS $CITUS_LDFLAGS"
POSTGRES_SRCDIR="$POSTGRES_SRCDIR"

7
configure.in
View File

@ -170,7 +170,9 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=vla]) # visual studio does not support thes
#
AC_ARG_ENABLE([coverage], AS_HELP_STRING([--enable-coverage], [build with coverage testing instrumentation]))
if test "$enable_coverage" = yes; then
CITUS_CFLAGS="$CITUS_CFLAGS -fprofile-arcs -ftest-coverage"
CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage"
CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG"
CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage"
fi
#
@ -201,7 +203,8 @@ AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL",
[Base URL for statistics collection and update checks])
AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS")
AC_SUBST(CITUS_LDFLAGS, "$LIBS")
AC_SUBST(CITUS_CPPFLAGS, "$CITUS_CPPFLAGS")
AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS")
AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR")
AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR")

5
src/backend/distributed/.gitignore vendored
View File

@ -8,8 +8,3 @@
/regression.out
/results/
/tmp_check*
# ignore latest install file
citus--?.?.sql
citus--?.?-*.sql
!citus--?.?-*--?.?-*.sql

230
src/backend/distributed/Makefile
View File

@ -5,26 +5,9 @@ citus_top_builddir = ../../..
MODULE_big = citus
EXTENSION = citus
EXTVERSIONS = 5.0 5.0-1 5.0-2 \
5.1-1 5.1-2 5.1-3 5.1-4 5.1-5 5.1-6 5.1-7 5.1-8 \
5.2-1 5.2-2 5.2-3 5.2-4 \
6.0-1 6.0-2 6.0-3 6.0-4 6.0-5 6.0-6 6.0-7 6.0-8 6.0-9 6.0-10 6.0-11 6.0-12 6.0-13 6.0-14 6.0-15 6.0-16 6.0-17 6.0-18 \
6.1-1 6.1-2 6.1-3 6.1-4 6.1-5 6.1-6 6.1-7 6.1-8 6.1-9 6.1-10 6.1-11 6.1-12 6.1-13 6.1-14 6.1-15 6.1-16 6.1-17 \
6.2-1 6.2-2 6.2-3 6.2-4 \
7.0-1 7.0-2 7.0-3 7.0-4 7.0-5 7.0-6 7.0-7 7.0-8 7.0-9 7.0-10 7.0-11 7.0-12 7.0-13 7.0-14 7.0-15 \
7.1-1 7.1-2 7.1-3 7.1-4 \
7.2-1 7.2-2 7.2-3 \
7.3-1 7.3-2 7.3-3 \
7.4-1 7.4-2 7.4-3 \
7.5-1 7.5-2 7.5-3 7.5-4 7.5-5 7.5-6 7.5-7 \
8.0-1 8.0-2 8.0-3 8.0-4 8.0-5 8.0-6 8.0-7 8.0-8 8.0-9 8.0-10 8.0-11 8.0-12 8.0-13 \
8.1-1\
8.2-1
# All citus--*.sql files in the source directory
DATA = $(patsubst $(citus_abs_srcdir)/%.sql,%.sql,$(wildcard $(citus_abs_srcdir)/$(EXTENSION)--*--*.sql))
# Generated files for each version
DATA_built = $(foreach v,$(EXTVERSIONS),$(EXTENSION)--$(v).sql)
DATA = $(patsubst $(citus_abs_srcdir)/%.sql,%.sql,$(wildcard $(citus_abs_srcdir)/$(EXTENSION)--*.sql))
# directories with source files
SUBDIRS = . commands connection ddl executor master metadata planner progress relay test transaction utils worker
@ -37,217 +20,6 @@ OBJS += \
# be explicit about the default target
all:
# generate each version's file installation file by concatenating
# previous upgrade scripts
$(EXTENSION)--5.0.sql: $(EXTENSION).sql
cat $^ > $@
$(EXTENSION)--5.0-1.sql: $(EXTENSION)--5.0.sql $(EXTENSION)--5.0--5.0-1.sql
cat $^ > $@
$(EXTENSION)--5.0-2.sql: $(EXTENSION)--5.0-1.sql $(EXTENSION)--5.0-1--5.0-2.sql
cat $^ > $@
$(EXTENSION)--5.1-1.sql: $(EXTENSION)--5.0-2.sql $(EXTENSION)--5.0-2--5.1-1.sql
cat $^ > $@
$(EXTENSION)--5.1-2.sql: $(EXTENSION)--5.1-1.sql $(EXTENSION)--5.1-1--5.1-2.sql
cat $^ > $@
$(EXTENSION)--5.1-3.sql: $(EXTENSION)--5.1-2.sql $(EXTENSION)--5.1-2--5.1-3.sql
cat $^ > $@
$(EXTENSION)--5.1-4.sql: $(EXTENSION)--5.1-3.sql $(EXTENSION)--5.1-3--5.1-4.sql
cat $^ > $@
$(EXTENSION)--5.1-5.sql: $(EXTENSION)--5.1-4.sql $(EXTENSION)--5.1-4--5.1-5.sql
cat $^ > $@
$(EXTENSION)--5.1-6.sql: $(EXTENSION)--5.1-5.sql $(EXTENSION)--5.1-5--5.1-6.sql
cat $^ > $@
$(EXTENSION)--5.1-7.sql: $(EXTENSION)--5.1-6.sql $(EXTENSION)--5.1-6--5.1-7.sql
cat $^ > $@
$(EXTENSION)--5.1-8.sql: $(EXTENSION)--5.1-7.sql $(EXTENSION)--5.1-7--5.1-8.sql
cat $^ > $@
$(EXTENSION)--5.2-1.sql: $(EXTENSION)--5.1-8.sql $(EXTENSION)--5.1-8--5.2-1.sql
cat $^ > $@
$(EXTENSION)--5.2-2.sql: $(EXTENSION)--5.2-1.sql $(EXTENSION)--5.2-1--5.2-2.sql
cat $^ > $@
$(EXTENSION)--5.2-3.sql: $(EXTENSION)--5.2-2.sql $(EXTENSION)--5.2-2--5.2-3.sql
cat $^ > $@
$(EXTENSION)--5.2-4.sql: $(EXTENSION)--5.2-3.sql $(EXTENSION)--5.2-3--5.2-4.sql
cat $^ > $@
$(EXTENSION)--6.0-1.sql: $(EXTENSION)--5.2-4.sql $(EXTENSION)--5.2-4--6.0-1.sql
cat $^ > $@
$(EXTENSION)--6.0-2.sql: $(EXTENSION)--6.0-1.sql $(EXTENSION)--6.0-1--6.0-2.sql
cat $^ > $@
$(EXTENSION)--6.0-3.sql: $(EXTENSION)--6.0-2.sql $(EXTENSION)--6.0-2--6.0-3.sql
cat $^ > $@
$(EXTENSION)--6.0-4.sql: $(EXTENSION)--6.0-3.sql $(EXTENSION)--6.0-3--6.0-4.sql
cat $^ > $@
$(EXTENSION)--6.0-5.sql: $(EXTENSION)--6.0-4.sql $(EXTENSION)--6.0-4--6.0-5.sql
cat $^ > $@
$(EXTENSION)--6.0-6.sql: $(EXTENSION)--6.0-5.sql $(EXTENSION)--6.0-5--6.0-6.sql
cat $^ > $@
$(EXTENSION)--6.0-7.sql: $(EXTENSION)--6.0-6.sql $(EXTENSION)--6.0-6--6.0-7.sql
cat $^ > $@
$(EXTENSION)--6.0-8.sql: $(EXTENSION)--6.0-7.sql $(EXTENSION)--6.0-7--6.0-8.sql
cat $^ > $@
$(EXTENSION)--6.0-9.sql: $(EXTENSION)--6.0-8.sql $(EXTENSION)--6.0-8--6.0-9.sql
cat $^ > $@
$(EXTENSION)--6.0-10.sql: $(EXTENSION)--6.0-9.sql $(EXTENSION)--6.0-9--6.0-10.sql
cat $^ > $@
$(EXTENSION)--6.0-11.sql: $(EXTENSION)--6.0-10.sql $(EXTENSION)--6.0-10--6.0-11.sql
cat $^ > $@
$(EXTENSION)--6.0-12.sql: $(EXTENSION)--6.0-11.sql $(EXTENSION)--6.0-11--6.0-12.sql
cat $^ > $@
$(EXTENSION)--6.0-13.sql: $(EXTENSION)--6.0-12.sql $(EXTENSION)--6.0-12--6.0-13.sql
cat $^ > $@
$(EXTENSION)--6.0-14.sql: $(EXTENSION)--6.0-13.sql $(EXTENSION)--6.0-13--6.0-14.sql
cat $^ > $@
$(EXTENSION)--6.0-15.sql: $(EXTENSION)--6.0-14.sql $(EXTENSION)--6.0-14--6.0-15.sql
cat $^ > $@
$(EXTENSION)--6.0-16.sql: $(EXTENSION)--6.0-15.sql $(EXTENSION)--6.0-15--6.0-16.sql
cat $^ > $@
$(EXTENSION)--6.0-17.sql: $(EXTENSION)--6.0-16.sql $(EXTENSION)--6.0-16--6.0-17.sql
cat $^ > $@
$(EXTENSION)--6.0-18.sql: $(EXTENSION)--6.0-17.sql $(EXTENSION)--6.0-17--6.0-18.sql
cat $^ > $@
$(EXTENSION)--6.1-1.sql: $(EXTENSION)--6.0-18.sql $(EXTENSION)--6.0-18--6.1-1.sql
cat $^ > $@
$(EXTENSION)--6.1-2.sql: $(EXTENSION)--6.1-1.sql $(EXTENSION)--6.1-1--6.1-2.sql
cat $^ > $@
$(EXTENSION)--6.1-3.sql: $(EXTENSION)--6.1-2.sql $(EXTENSION)--6.1-2--6.1-3.sql
cat $^ > $@
$(EXTENSION)--6.1-4.sql: $(EXTENSION)--6.1-3.sql $(EXTENSION)--6.1-3--6.1-4.sql
cat $^ > $@
$(EXTENSION)--6.1-5.sql: $(EXTENSION)--6.1-4.sql $(EXTENSION)--6.1-4--6.1-5.sql
cat $^ > $@
$(EXTENSION)--6.1-6.sql: $(EXTENSION)--6.1-5.sql $(EXTENSION)--6.1-5--6.1-6.sql
cat $^ > $@
$(EXTENSION)--6.1-7.sql: $(EXTENSION)--6.1-6.sql $(EXTENSION)--6.1-6--6.1-7.sql
cat $^ > $@
$(EXTENSION)--6.1-8.sql: $(EXTENSION)--6.1-7.sql $(EXTENSION)--6.1-7--6.1-8.sql
cat $^ > $@
$(EXTENSION)--6.1-9.sql: $(EXTENSION)--6.1-8.sql $(EXTENSION)--6.1-8--6.1-9.sql
cat $^ > $@
$(EXTENSION)--6.1-10.sql: $(EXTENSION)--6.1-9.sql $(EXTENSION)--6.1-9--6.1-10.sql
cat $^ > $@
$(EXTENSION)--6.1-11.sql: $(EXTENSION)--6.1-10.sql $(EXTENSION)--6.1-10--6.1-11.sql
cat $^ > $@
$(EXTENSION)--6.1-12.sql: $(EXTENSION)--6.1-11.sql $(EXTENSION)--6.1-11--6.1-12.sql
cat $^ > $@
$(EXTENSION)--6.1-13.sql: $(EXTENSION)--6.1-12.sql $(EXTENSION)--6.1-12--6.1-13.sql
cat $^ > $@
$(EXTENSION)--6.1-14.sql: $(EXTENSION)--6.1-13.sql $(EXTENSION)--6.1-13--6.1-14.sql
cat $^ > $@
$(EXTENSION)--6.1-15.sql: $(EXTENSION)--6.1-14.sql $(EXTENSION)--6.1-14--6.1-15.sql
cat $^ > $@
$(EXTENSION)--6.1-16.sql: $(EXTENSION)--6.1-15.sql $(EXTENSION)--6.1-15--6.1-16.sql
cat $^ > $@
$(EXTENSION)--6.1-17.sql: $(EXTENSION)--6.1-16.sql $(EXTENSION)--6.1-16--6.1-17.sql
cat $^ > $@
$(EXTENSION)--6.2-1.sql: $(EXTENSION)--6.1-17.sql $(EXTENSION)--6.1-17--6.2-1.sql
cat $^ > $@
$(EXTENSION)--6.2-2.sql: $(EXTENSION)--6.2-1.sql $(EXTENSION)--6.2-1--6.2-2.sql
cat $^ > $@
$(EXTENSION)--6.2-3.sql: $(EXTENSION)--6.2-2.sql $(EXTENSION)--6.2-2--6.2-3.sql
cat $^ > $@
$(EXTENSION)--6.2-4.sql: $(EXTENSION)--6.2-3.sql $(EXTENSION)--6.2-3--6.2-4.sql
cat $^ > $@
$(EXTENSION)--7.0-1.sql: $(EXTENSION)--6.2-4.sql $(EXTENSION)--6.2-4--7.0-1.sql
cat $^ > $@
$(EXTENSION)--7.0-2.sql: $(EXTENSION)--7.0-1.sql $(EXTENSION)--7.0-1--7.0-2.sql
cat $^ > $@
$(EXTENSION)--7.0-3.sql: $(EXTENSION)--7.0-2.sql $(EXTENSION)--7.0-2--7.0-3.sql
cat $^ > $@
$(EXTENSION)--7.0-4.sql: $(EXTENSION)--7.0-3.sql $(EXTENSION)--7.0-3--7.0-4.sql
cat $^ > $@
$(EXTENSION)--7.0-5.sql: $(EXTENSION)--7.0-4.sql $(EXTENSION)--7.0-4--7.0-5.sql
cat $^ > $@
$(EXTENSION)--7.0-6.sql: $(EXTENSION)--7.0-5.sql $(EXTENSION)--7.0-5--7.0-6.sql
cat $^ > $@
$(EXTENSION)--7.0-7.sql: $(EXTENSION)--7.0-6.sql $(EXTENSION)--7.0-6--7.0-7.sql
cat $^ > $@
$(EXTENSION)--7.0-8.sql: $(EXTENSION)--7.0-7.sql $(EXTENSION)--7.0-7--7.0-8.sql
cat $^ > $@
$(EXTENSION)--7.0-9.sql: $(EXTENSION)--7.0-8.sql $(EXTENSION)--7.0-8--7.0-9.sql
cat $^ > $@
$(EXTENSION)--7.0-10.sql: $(EXTENSION)--7.0-9.sql $(EXTENSION)--7.0-9--7.0-10.sql
cat $^ > $@
$(EXTENSION)--7.0-11.sql: $(EXTENSION)--7.0-10.sql $(EXTENSION)--7.0-10--7.0-11.sql
cat $^ > $@
$(EXTENSION)--7.0-12.sql: $(EXTENSION)--7.0-11.sql $(EXTENSION)--7.0-11--7.0-12.sql
cat $^ > $@
$(EXTENSION)--7.0-13.sql: $(EXTENSION)--7.0-12.sql $(EXTENSION)--7.0-12--7.0-13.sql
cat $^ > $@
$(EXTENSION)--7.0-14.sql: $(EXTENSION)--7.0-13.sql $(EXTENSION)--7.0-13--7.0-14.sql
cat $^ > $@
$(EXTENSION)--7.0-15.sql: $(EXTENSION)--7.0-14.sql $(EXTENSION)--7.0-14--7.0-15.sql
cat $^ > $@
$(EXTENSION)--7.1-1.sql: $(EXTENSION)--7.0-15.sql $(EXTENSION)--7.0-15--7.1-1.sql
cat $^ > $@
$(EXTENSION)--7.1-2.sql: $(EXTENSION)--7.1-1.sql $(EXTENSION)--7.1-1--7.1-2.sql
cat $^ > $@
$(EXTENSION)--7.1-3.sql: $(EXTENSION)--7.1-2.sql $(EXTENSION)--7.1-2--7.1-3.sql
cat $^ > $@
$(EXTENSION)--7.1-4.sql: $(EXTENSION)--7.1-3.sql $(EXTENSION)--7.1-3--7.1-4.sql
cat $^ > $@
$(EXTENSION)--7.2-1.sql: $(EXTENSION)--7.1-4.sql $(EXTENSION)--7.1-4--7.2-1.sql
cat $^ > $@
$(EXTENSION)--7.2-2.sql: $(EXTENSION)--7.2-1.sql $(EXTENSION)--7.2-1--7.2-2.sql
cat $^ > $@
$(EXTENSION)--7.2-3.sql: $(EXTENSION)--7.2-2.sql $(EXTENSION)--7.2-2--7.2-3.sql
cat $^ > $@
$(EXTENSION)--7.3-1.sql: $(EXTENSION)--7.2-3.sql $(EXTENSION)--7.2-3--7.3-1.sql
cat $^ > $@
$(EXTENSION)--7.3-2.sql: $(EXTENSION)--7.3-1.sql $(EXTENSION)--7.3-1--7.3-2.sql
cat $^ > $@
$(EXTENSION)--7.3-3.sql: $(EXTENSION)--7.3-2.sql $(EXTENSION)--7.3-2--7.3-3.sql
cat $^ > $@
$(EXTENSION)--7.4-1.sql: $(EXTENSION)--7.3-3.sql $(EXTENSION)--7.3-3--7.4-1.sql
cat $^ > $@
$(EXTENSION)--7.4-2.sql: $(EXTENSION)--7.4-1.sql $(EXTENSION)--7.4-1--7.4-2.sql
cat $^ > $@
$(EXTENSION)--7.4-3.sql: $(EXTENSION)--7.4-2.sql $(EXTENSION)--7.4-2--7.4-3.sql
cat $^ > $@
$(EXTENSION)--7.5-1.sql: $(EXTENSION)--7.4-3.sql $(EXTENSION)--7.4-3--7.5-1.sql
cat $^ > $@
$(EXTENSION)--7.5-2.sql: $(EXTENSION)--7.5-1.sql $(EXTENSION)--7.5-1--7.5-2.sql
cat $^ > $@
$(EXTENSION)--7.5-3.sql: $(EXTENSION)--7.5-2.sql $(EXTENSION)--7.5-2--7.5-3.sql
cat $^ > $@
$(EXTENSION)--7.5-4.sql: $(EXTENSION)--7.5-3.sql $(EXTENSION)--7.5-3--7.5-4.sql
cat $^ > $@
$(EXTENSION)--7.5-5.sql: $(EXTENSION)--7.5-4.sql $(EXTENSION)--7.5-4--7.5-5.sql
cat $^ > $@
$(EXTENSION)--7.5-6.sql: $(EXTENSION)--7.5-5.sql $(EXTENSION)--7.5-5--7.5-6.sql
cat $^ > $@
$(EXTENSION)--7.5-7.sql: $(EXTENSION)--7.5-6.sql $(EXTENSION)--7.5-6--7.5-7.sql
cat $^ > $@
$(EXTENSION)--8.0-1.sql: $(EXTENSION)--7.5-7.sql $(EXTENSION)--7.5-7--8.0-1.sql
cat $^ > $@
$(EXTENSION)--8.0-2.sql: $(EXTENSION)--8.0-1.sql $(EXTENSION)--8.0-1--8.0-2.sql
cat $^ > $@
$(EXTENSION)--8.0-3.sql: $(EXTENSION)--8.0-2.sql $(EXTENSION)--8.0-2--8.0-3.sql
cat $^ > $@
$(EXTENSION)--8.0-4.sql: $(EXTENSION)--8.0-3.sql $(EXTENSION)--8.0-3--8.0-4.sql
cat $^ > $@
$(EXTENSION)--8.0-5.sql: $(EXTENSION)--8.0-4.sql $(EXTENSION)--8.0-4--8.0-5.sql
cat $^ > $@
$(EXTENSION)--8.0-6.sql: $(EXTENSION)--8.0-5.sql $(EXTENSION)--8.0-5--8.0-6.sql
cat $^ > $@
$(EXTENSION)--8.0-7.sql: $(EXTENSION)--8.0-6.sql $(EXTENSION)--8.0-6--8.0-7.sql
cat $^ > $@
$(EXTENSION)--8.0-8.sql: $(EXTENSION)--8.0-7.sql $(EXTENSION)--8.0-7--8.0-8.sql
cat $^ > $@
$(EXTENSION)--8.0-9.sql: $(EXTENSION)--8.0-8.sql $(EXTENSION)--8.0-8--8.0-9.sql
cat $^ > $@
$(EXTENSION)--8.0-10.sql: $(EXTENSION)--8.0-9.sql $(EXTENSION)--8.0-9--8.0-10.sql
cat $^ > $@
$(EXTENSION)--8.0-11.sql: $(EXTENSION)--8.0-10.sql $(EXTENSION)--8.0-10--8.0-11.sql
cat $^ > $@
$(EXTENSION)--8.0-12.sql: $(EXTENSION)--8.0-11.sql $(EXTENSION)--8.0-11--8.0-12.sql
cat $^ > $@
$(EXTENSION)--8.0-13.sql: $(EXTENSION)--8.0-12.sql $(EXTENSION)--8.0-12--8.0-13.sql
cat $^ > $@
$(EXTENSION)--8.1-1.sql: $(EXTENSION)--8.0-13.sql $(EXTENSION)--8.0-13--8.1-1.sql
cat $^ > $@
$(EXTENSION)--8.2-1.sql: $(EXTENSION)--8.1-1.sql $(EXTENSION)--8.1-1--8.2-1.sql
cat $^ > $@
NO_PGXS = 1
SHLIB_LINK = $(libpq)

0
src/backend/distributed/citus.sql → src/backend/distributed/citus--5.0.sql
View File

8
src/backend/distributed/executor/multi_router_executor.c
View File

@ -92,7 +92,7 @@ static int64 ExecuteSingleModifyTask(CitusScanState *scanState, Task *task, CmdT
operation, bool alwaysThrowErrorOnFailure, bool
expectResults);
static void ExecuteSingleSelectTask(CitusScanState *scanState, Task *task);
static List * BuildPlacementAccessList(uint32 groupId, List *relationShardList,
static List * BuildPlacementAccessList(int32 groupId, List *relationShardList,
ShardPlacementAccessType accessType);
static List * GetModifyConnections(Task *task, bool markCritical);
static int64 ExecuteModifyTasks(List *taskList, bool expectResults,
@ -890,7 +890,7 @@ ExecuteSingleSelectTask(CitusScanState *scanState, Task *task)
* (e.g. in case of a broadcast join) then the shard is skipped.
*/
List *
BuildPlacementSelectList(uint32 groupId, List *relationShardList)
BuildPlacementSelectList(int32 groupId, List *relationShardList)
{
return BuildPlacementAccessList(groupId, relationShardList, PLACEMENT_ACCESS_SELECT);
}
@ -900,7 +900,7 @@ BuildPlacementSelectList(uint32 groupId, List *relationShardList)
* BuildPlacementDDLList is a warpper around BuildPlacementAccessList() for DDL access.
*/
List *
BuildPlacementDDLList(uint32 groupId, List *relationShardList)
BuildPlacementDDLList(int32 groupId, List *relationShardList)
{
return BuildPlacementAccessList(groupId, relationShardList, PLACEMENT_ACCESS_DDL);
}
@ -911,7 +911,7 @@ BuildPlacementDDLList(uint32 groupId, List *relationShardList)
* relationShardList and the access type.
*/
static List *
BuildPlacementAccessList(uint32 groupId, List *relationShardList,
BuildPlacementAccessList(int32 groupId, List *relationShardList,
ShardPlacementAccessType accessType)
{
ListCell *relationShardCell = NULL;

2
src/backend/distributed/master/master_create_shards.c
View File

@ -316,7 +316,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool
{
ShardPlacement *sourcePlacement =
(ShardPlacement *) lfirst(sourceShardPlacementCell);
uint32 groupId = sourcePlacement->groupId;
int32 groupId = sourcePlacement->groupId;
const RelayFileState shardState = FILE_FINALIZED;
const uint64 shardSize = 0;
uint64 shardPlacementId = 0;

13
src/backend/distributed/master/master_metadata_utility.c
View File

@ -237,6 +237,7 @@ DistributedTableSizeOnWorker(WorkerNode *workerNode, Oid relationId, char *sizeQ
tableSizeString = tableSizeStringInfo->data;
tableSize = atol(tableSizeString);
PQclear(result);
ClearResults(connection, raiseErrors);
return tableSize;
@ -249,7 +250,7 @@ DistributedTableSizeOnWorker(WorkerNode *workerNode, Oid relationId, char *sizeQ
* on the group.
*/
List *
GroupShardPlacementsForTableOnGroup(Oid relationId, uint32 groupId)
GroupShardPlacementsForTableOnGroup(Oid relationId, int32 groupId)
{
DistTableCacheEntry *distTableCacheEntry = DistributedTableCacheEntry(relationId);
List *resultList = NIL;
@ -633,7 +634,7 @@ ShardLength(uint64 shardId)
* NodeGroupHasShardPlacements returns whether any active shards are placed on the group
*/
bool
NodeGroupHasShardPlacements(uint32 groupId, bool onlyConsiderActivePlacements)
NodeGroupHasShardPlacements(int32 groupId, bool onlyConsiderActivePlacements)
{
const int scanKeyCount = (onlyConsiderActivePlacements ? 2 : 1);
const bool indexOK = false;
@ -648,7 +649,7 @@ NodeGroupHasShardPlacements(uint32 groupId, bool onlyConsiderActivePlacements)
AccessShareLock);
ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid,
BTEqualStrategyNumber, F_INT4EQ, UInt32GetDatum(groupId));
BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId));
if (onlyConsiderActivePlacements)
{
ScanKeyInit(&scanKey[1], Anum_pg_dist_placement_shardstate,
@ -851,7 +852,7 @@ TupleToGroupShardPlacement(TupleDesc tupleDescriptor, HeapTuple heapTuple)
datumArray[Anum_pg_dist_placement_shardlength - 1]);
shardPlacement->shardState = DatumGetUInt32(
datumArray[Anum_pg_dist_placement_shardstate - 1]);
shardPlacement->groupId = DatumGetUInt32(
shardPlacement->groupId = DatumGetInt32(
datumArray[Anum_pg_dist_placement_groupid - 1]);
return shardPlacement;
@ -921,7 +922,7 @@ InsertShardRow(Oid relationId, uint64 shardId, char storageType,
uint64
InsertShardPlacementRow(uint64 shardId, uint64 placementId,
char shardState, uint64 shardLength,
uint32 groupId)
int32 groupId)
{
Relation pgDistPlacement = NULL;
TupleDesc tupleDescriptor = NULL;
@ -941,7 +942,7 @@ InsertShardPlacementRow(uint64 shardId, uint64 placementId,
values[Anum_pg_dist_placement_shardid - 1] = Int64GetDatum(shardId);
values[Anum_pg_dist_placement_shardstate - 1] = CharGetDatum(shardState);
values[Anum_pg_dist_placement_shardlength - 1] = Int64GetDatum(shardLength);
values[Anum_pg_dist_placement_groupid - 1] = Int64GetDatum(groupId);
values[Anum_pg_dist_placement_groupid - 1] = Int32GetDatum(groupId);
/* open shard placement relation and insert new tuple */
pgDistPlacement = heap_open(DistPlacementRelationId(), RowExclusiveLock);

2
src/backend/distributed/master/master_stage_protocol.c
View File

@ -766,7 +766,7 @@ UpdateShardStatistics(int64 shardId)
{
ShardPlacement *placement = (ShardPlacement *) lfirst(shardPlacementCell);
uint64 placementId = placement->placementId;
uint32 groupId = placement->groupId;
int32 groupId = placement->groupId;
DeleteShardPlacementRow(placementId);
InsertShardPlacementRow(shardId, placementId, FILE_FINALIZED, shardSize,

6
src/backend/distributed/metadata/metadata_sync.c
View File

@ -51,7 +51,7 @@
#include "utils/tqual.h"
static char * LocalGroupIdUpdateCommand(uint32 groupId);
static char * LocalGroupIdUpdateCommand(int32 groupId);
static void MarkNodeHasMetadata(char *nodeName, int32 nodePort, bool hasMetadata);
static List * SequenceDDLCommandsForTable(Oid relationId);
static void EnsureSupportedSequenceColumnType(Oid sequenceOid);
@ -824,7 +824,7 @@ ColocationIdUpdateCommand(Oid relationId, uint32 colocationId)
*/
char *
PlacementUpsertCommand(uint64 shardId, uint64 placementId, int shardState,
uint64 shardLength, uint32 groupId)
uint64 shardLength, int32 groupId)
{
StringInfo command = makeStringInfo();
@ -840,7 +840,7 @@ PlacementUpsertCommand(uint64 shardId, uint64 placementId, int shardState,
* of a worker and returns the command in a string.
*/
static char *
LocalGroupIdUpdateCommand(uint32 groupId)
LocalGroupIdUpdateCommand(int32 groupId)
{
StringInfo updateCommand = makeStringInfo();

18
src/backend/distributed/planner/distributed_planner.c
View File

@ -64,7 +64,6 @@ static DistributedPlan * CreateDistributedPlan(uint64 planId, Query *originalQue
plannerRestrictionContext);
static DeferredErrorMessage * DeferErrorIfPartitionTableNotSingleReplicated(Oid
relationId);
static Node * ResolveExternalParams(Node *inputNode, ParamListInfo boundParams);
static void AssignRTEIdentities(Query *queryTree);
static void AssignRTEIdentity(RangeTblEntry *rangeTableEntry, int rteIdentifier);
@ -147,11 +146,22 @@ distributed_planner(Query *parse, int cursorOptions, ParamListInfo boundParams)
PG_TRY();
{
/*
* First call into standard planner. This is required because the Citus
* planner relies on parse tree transformations made by postgres' planner.
* For trivial queries, we're skipping the standard_planner() in
* order to eliminate its overhead.
*
* Otherwise, call into standard planner. This is required because the Citus
* planner relies on both the restriction information per table and parse tree
* transformations made by postgres' planner.
*/
if (needsDistributedPlanning && FastPathRouterQuery(originalQuery))
{
result = FastPathPlanner(originalQuery, parse, boundParams);
}
else
{
result = standard_planner(parse, cursorOptions, boundParams);
}
if (needsDistributedPlanning)
{
@ -831,7 +841,7 @@ DeferErrorIfPartitionTableNotSingleReplicated(Oid relationId)
* Note that this function is inspired by eval_const_expr() on Postgres.
* We cannot use that function because it requires access to PlannerInfo.
*/
static Node *
Node *
ResolveExternalParams(Node *inputNode, ParamListInfo boundParams)
{
/* consider resolving external parameters only when boundParams exists */

416
src/backend/distributed/planner/fast_path_router_planner.c Normal file
View File

@ -0,0 +1,416 @@
/*-------------------------------------------------------------------------
*
* fast_path_router_planner.c
*
* Planning logic for fast path router planner queries. In this context,
* we define "Fast Path Planning" as trivial queries where Citus
* can skip relying on the standard_planner() and handle all the planning.
*
* For router planner, standard_planner() is mostly important to generate
* the necessary restriction information. Later, the restriction information
* generated by the standard_planner is used to decide whether all the shards
* that a distributed query touches reside on a single worker node. However,
* standard_planner() does a lot of extra things such as cost estimation and
* execution path generations which are completely unnecessary in the context
* of distributed planning.
*
* There are certain types of queries where Citus could skip relying on
* standard_planner() to generate the restriction information. For queries
* in the following format, Citus does not need any information that the
* standard_planner() generates:
* SELECT ... FROM single_table WHERE distribution_key = X; or
* DELETE FROM single_table WHERE distribution_key = X; or
* UPDATE single_table SET value_1 = value_2 + 1 WHERE distribution_key = X;
*
* Note that the queries might not be as simple as the above such that
* GROUP BY, WINDOW FUNCIONS, ORDER BY or HAVING etc. are all acceptable. The
* only rule is that the query is on a single distributed (or reference) table
* and there is a "distribution_key = X;" in the WHERE clause. With that, we
* could use to decide the shard that a distributed query touches reside on
* a worker node.
*
* Copyright (c) 2019, Citus Data, Inc.
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "distributed/distributed_planner.h"
#include "distributed/multi_physical_planner.h" /* only to use some utility functions */
#include "distributed/metadata_cache.h"
#include "distributed/multi_router_planner.h"
#include "distributed/pg_dist_partition.h"
#include "distributed/shardinterval_utils.h"
#include "distributed/shard_pruning.h"
#include "nodes/nodeFuncs.h"
#include "nodes/parsenodes.h"
#include "nodes/pg_list.h"
#include "optimizer/clauses.h"
bool EnableFastPathRouterPlanner = true;
static bool ColumnAppearsMultipleTimes(Node *quals, Var *distributionKey);
static bool ConjunctionContainsColumnFilter(Node *node, Var *column);
static bool DistKeyInSimpleOpExpression(Expr *clause, Var *distColumn);
/*
* FastPathPlanner is intended to be used instead of standard_planner() for trivial
* queries defined by FastPathRouterQuery().
*
* The basic idea is that we need a very little of what standard_planner() does for
* the trivial queries. So skip calling standard_planner() to save CPU cycles.
*
*/
PlannedStmt *
FastPathPlanner(Query *originalQuery, Query *parse, ParamListInfo boundParams)
{
PlannedStmt *result = NULL;
/*
* To support prepared statements for fast-path queries, we resolve the
* external parameters at this point. Note that this is normally done by
* eval_const_expr() in standard planner when the boundParams are avaliable.
* If not avaliable, as does for all other types of queries, Citus goes
* through the logic of increasing the cost of the plan and forcing
* PostgreSQL to pick custom plans.
*
* We're also only interested in resolving the quals since we'd want to
* do shard pruning based on the filter on the distribution column.
*/
originalQuery->jointree->quals =
ResolveExternalParams((Node *) originalQuery->jointree->quals,
copyParamList(boundParams));
/*
* Citus planner relies on some of the transformations on constant
* evaluation on the parse tree.
*/
parse->targetList =
(List *) eval_const_expressions(NULL, (Node *) parse->targetList);
parse->jointree->quals =
(Node *) eval_const_expressions(NULL, (Node *) parse->jointree->quals);
result = GeneratePlaceHolderPlannedStmt(originalQuery);
return result;
}
/*
* GeneratePlaceHolderPlannedStmt creates a planned statement which contains
* a sequential scan on the relation that is accessed by the input query.
* The returned PlannedStmt is not proper (e.g., set_plan_references() is
* not called on the plan or the quals are not set), so should not be
* passed to the executor directly. This is only useful to have a
* placeholder PlannedStmt where target list is properly set. Note that
* this is what router executor relies on.
*
* This function makes the assumption (and the assertion) that
* the input query is in the form defined by FastPathRouterQuery().
*/
PlannedStmt *
GeneratePlaceHolderPlannedStmt(Query *parse)
{
PlannedStmt *result = makeNode(PlannedStmt);
SeqScan *seqScanNode = makeNode(SeqScan);
Plan *plan = &seqScanNode->plan;
Oid relationId = InvalidOid;
AssertArg(FastPathRouterQuery(parse));
/* there is only a single relation rte */
seqScanNode->scanrelid = 1;
plan->targetlist = copyObject(parse->targetList);
plan->qual = NULL;
plan->lefttree = NULL;
plan->righttree = NULL;
plan->plan_node_id = 1;
/* rtable is used for access permission checks */
result->commandType = parse->commandType;
result->queryId = parse->queryId;
result->stmt_len = parse->stmt_len;
result->rtable = copyObject(parse->rtable);
result->planTree = (Plan *) plan;
relationId = ExtractFirstDistributedTableId(parse);
result->relationOids = list_make1_oid(relationId);
return result;
}
/*
* FastPathRouterQuery gets a query and returns true if the query is eligable for
* being a fast path router query.
* The requirements for the fast path query can be listed below:
*
* - SELECT query without CTES, sublinks-subqueries, set operations
* - The query should touch only a single hash distributed or reference table
* - The distribution with equality operator should be in the WHERE clause
* and it should be ANDed with any other filters. Also, the distribution
* key should only exists once in the WHERE clause. So basically,
* SELECT ... FROM dist_table WHERE dist_key = X
* - No returning for UPDATE/DELETE queries
*/
bool
FastPathRouterQuery(Query *query)
{
RangeTblEntry *rangeTableEntry = NULL;
FromExpr *joinTree = query->jointree;
Node *quals = NULL;
Oid distributedTableId = InvalidOid;
Var *distributionKey = NULL;
DistTableCacheEntry *cacheEntry = NULL;
if (!EnableFastPathRouterPlanner)
{
return false;
}
if (!(query->commandType == CMD_SELECT || query->commandType == CMD_UPDATE ||
query->commandType == CMD_DELETE))
{
return false;
}
/*
* We want to deal with only very simple select queries. Some of the
* checks might be too restrictive, still we prefer this way.
*/
if (query->cteList != NIL || query->returningList != NIL ||
query->hasSubLinks || query->setOperations != NULL ||
query->hasTargetSRFs || query->hasModifyingCTE)
{
return false;
}
/* make sure that the only range table in FROM clause */
if (list_length(query->rtable) != 1)
{
return false;
}
rangeTableEntry = (RangeTblEntry *) linitial(query->rtable);
if (rangeTableEntry->rtekind != RTE_RELATION)
{
return false;
}
/* we don't want to deal with append/range distributed tables */
distributedTableId = rangeTableEntry->relid;
cacheEntry = DistributedTableCacheEntry(distributedTableId);
if (!(cacheEntry->partitionMethod == DISTRIBUTE_BY_HASH ||
cacheEntry->partitionMethod == DISTRIBUTE_BY_NONE))
{
return false;
}
/* WHERE clause should not be empty for distributed tables */
if (joinTree == NULL ||
(cacheEntry->partitionMethod != DISTRIBUTE_BY_NONE && joinTree->quals == NULL))
{
return false;
}
/* if that's a reference table, we don't need to check anything further */
distributionKey = PartitionColumn(distributedTableId, 1);
if (!distributionKey)
{
return true;
}
/* convert list of expressions into expression tree for further processing */
quals = joinTree->quals;
if (quals != NULL && IsA(quals, List))
{
quals = (Node *) make_ands_explicit((List *) quals);
}
/*
* Distribution column must be used in a simple equality match check and it must be
* place at top level conjustion operator. In simple words, we should have
* WHERE dist_key = VALUE [AND ....];
*
* We're also not allowing any other appearances of the distribution key in the quals.
*
* Overall the logic is might sound fuzzy since it involves two individual checks:
* (a) Check for top level AND operator with one side being "dist_key = const"
* (b) Only allow single appearance of "dist_key" in the quals
*
* This is to simplify both of the individual checks and omit various edge cases
* that might arise with multiple distribution keys in the quals.
*/
if (ConjunctionContainsColumnFilter(quals, distributionKey) &&
!ColumnAppearsMultipleTimes(quals, distributionKey))
{
return true;
}
return false;
}
/*
* ColumnAppearsMultipleTimes returns true if the given input
* appears more than once in the quals.
*/
static bool
ColumnAppearsMultipleTimes(Node *quals, Var *distributionKey)
{
ListCell *varClauseCell = NULL;
List *varClauseList = NIL;
int partitionColumnReferenceCount = 0;
/* make sure partition column is used only once in the quals */
varClauseList = pull_var_clause_default(quals);
foreach(varClauseCell, varClauseList)
{
Var *column = (Var *) lfirst(varClauseCell);
if (equal(column, distributionKey))
{
partitionColumnReferenceCount++;
if (partitionColumnReferenceCount > 1)
{
return true;
}
}
}
return false;
}
/*
* ConjunctionContainsColumnFilter returns true if the query contains an exact
* match (equal) expression on the provided column. The function returns true only
* if the match expression has an AND relation with the rest of the expression tree.
*/
static bool
ConjunctionContainsColumnFilter(Node *node, Var *column)
{
if (node == NULL)
{
return false;
}
if (IsA(node, OpExpr))
{
OpExpr *opExpr = (OpExpr *) node;
bool distKeyInSimpleOpExpression =
DistKeyInSimpleOpExpression((Expr *) opExpr, column);
if (!distKeyInSimpleOpExpression)
{
return false;
}
return OperatorImplementsEquality(opExpr->opno);
}
else if (IsA(node, BoolExpr))
{
BoolExpr *boolExpr = (BoolExpr *) node;
List *argumentList = boolExpr->args;
ListCell *argumentCell = NULL;
/*
* We do not descend into boolean expressions other than AND.
* If the column filter appears in an OR clause, we do not
* consider it even if it is logically the same as a single value
* comparison (e.g. `<column> = <Const> OR false`)
*/
if (boolExpr->boolop != AND_EXPR)
{
return false;
}
foreach(argumentCell, argumentList)
{
Node *argumentNode = (Node *) lfirst(argumentCell);
if (ConjunctionContainsColumnFilter(argumentNode, column))
{
return true;
}
}
}
return false;
}
/*
* DistKeyInSimpleOpExpression checks whether given expression is a simple operator
* expression with either (dist_key = param) or (dist_key = const). Note that the
* operands could be in the reverse order as well.
*/
static bool
DistKeyInSimpleOpExpression(Expr *clause, Var *distColumn)
{
Node *leftOperand = NULL;
Node *rightOperand = NULL;
Param *paramClause = NULL;
Const *constantClause = NULL;
Var *columnInExpr = NULL;
if (is_opclause(clause) && list_length(((OpExpr *) clause)->args) == 2)
{
leftOperand = get_leftop(clause);
rightOperand = get_rightop(clause);
}
else
{
return false; /* not a binary opclause */
}
/* strip coercions before doing check */
leftOperand = strip_implicit_coercions(leftOperand);
rightOperand = strip_implicit_coercions(rightOperand);
if (IsA(rightOperand, Param) && IsA(leftOperand, Var))
{
paramClause = (Param *) rightOperand;
columnInExpr = (Var *) leftOperand;
}
else if (IsA(leftOperand, Param) && IsA(rightOperand, Var))
{
paramClause = (Param *) leftOperand;
columnInExpr = (Var *) rightOperand;
}
else if (IsA(rightOperand, Const) && IsA(leftOperand, Var))
{
constantClause = (Const *) rightOperand;
columnInExpr = (Var *) leftOperand;
}
else if (IsA(leftOperand, Const) && IsA(rightOperand, Var))
{
constantClause = (Const *) leftOperand;
columnInExpr = (Var *) rightOperand;
}
else
{
return false;
}
if (paramClause && paramClause->paramkind != PARAM_EXTERN)
{
/* we can only handle param_externs */
return false;
}
else if (constantClause && constantClause->constisnull)
{
/* we can only handle non-null constants */
return false;
}
/* at this point we should have the columnInExpr */
Assert(columnInExpr);
return equal(distColumn, columnInExpr);
}

27
src/backend/distributed/planner/multi_physical_planner.c
View File

@ -29,6 +29,7 @@
#include "catalog/pg_type.h"
#include "commands/defrem.h"
#include "commands/sequence.h"
#include "distributed/backend_data.h"
#include "distributed/listutils.h"
#include "distributed/citus_nodefuncs.h"
#include "distributed/citus_nodes.h"
@ -2005,13 +2006,11 @@ BuildJobTreeTaskList(Job *jobTree, PlannerRestrictionContext *plannerRestriction
if (job->subqueryPushdown)
{
bool isMultiShardQuery = false;
List *prunedRelationShardList = TargetShardIntervalsForQuery(job->jobQuery,
plannerRestrictionContext
->
List *prunedRelationShardList =
TargetShardIntervalsForRestrictInfo(plannerRestrictionContext->
relationRestrictionContext,
&
isMultiShardQuery,
NULL);
&isMultiShardQuery, NULL);
sqlTaskList = QueryPushdownSqlTaskList(job->jobQuery, job->jobId,
plannerRestrictionContext->
relationRestrictionContext,
@ -5078,14 +5077,24 @@ RoundRobinAssignTaskList(List *taskList)
/*
* RoundRobinReorder implements the core of the round-robin assignment policy.
* It takes a task and placement list and rotates a copy of the placement list
* based on the task's jobId. The rotated copy is returned.
* based on the latest stable transaction id provided by PostgreSQL.
*
* We prefer to use transactionId as the seed for the rotation to use the replicas
* in the same worker node within the same transaction. This becomes more important
* when we're reading from (the same or multiple) reference tables within a
* transaction. With this approach, we can prevent reads to expand the worker nodes
* that participate in a distributed transaction.
*
* Note that we prefer PostgreSQL's transactionId over distributed transactionId that
* Citus generates since the distributed transactionId is generated during the execution
* where as task-assignment happens duing the planning.
*/
static List *
RoundRobinReorder(Task *task, List *placementList)
{
uint64 jobId = task->jobId;
TransactionId transactionId = GetMyProcLocalTransactionId();
uint32 activePlacementCount = list_length(placementList);
uint32 roundRobinIndex = (jobId % activePlacementCount);
uint32 roundRobinIndex = (transactionId % activePlacementCount);
placementList = LeftRotateList(placementList, roundRobinIndex);

120
src/backend/distributed/planner/multi_router_planner.c
View File

@ -137,8 +137,7 @@ static void NormalizeMultiRowInsertTargetList(Query *query);
static List * BuildRoutesForInsert(Query *query, DeferredErrorMessage **planningError);
static List * GroupInsertValuesByShardId(List *insertValuesList);
static List * ExtractInsertValuesList(Query *query, Var *partitionColumn);
static bool MultiRouterPlannableQuery(Query *query,
RelationRestrictionContext *restrictionContext);
static bool MultiRouterPlannableQuery(Query *query);
static DeferredErrorMessage * ErrorIfQueryHasModifyingCTE(Query *queryTree);
static RangeTblEntry * GetUpdateOrDeleteRTE(Query *query);
static bool SelectsFromDistributedTable(List *rangeTableList, Query *query);
@ -146,6 +145,9 @@ static List * get_all_actual_clauses(List *restrictinfo_list);
static int CompareInsertValuesByShardId(const void *leftElement,
const void *rightElement);
static uint64 GetInitialShardId(List *relationShardList);
static List * TargetShardIntervalForFastPathQuery(Query *query,
Const **partitionValueConst,
bool *isMultiShardQuery);
static List * SingleShardSelectTaskList(Query *query, uint64 jobId,
List *relationShardList, List *placementList,
uint64 shardId);
@ -166,8 +168,7 @@ DistributedPlan *
CreateRouterPlan(Query *originalQuery, Query *query,
PlannerRestrictionContext *plannerRestrictionContext)
{
if (MultiRouterPlannableQuery(query,
plannerRestrictionContext->relationRestrictionContext))
if (MultiRouterPlannableQuery(query))
{
return CreateSingleTaskRouterPlan(originalQuery, query,
plannerRestrictionContext);
@ -1886,11 +1887,46 @@ PlanRouterQuery(Query *originalQuery,
*placementList = NIL;
prunedRelationShardList = TargetShardIntervalsForQuery(originalQuery,
plannerRestrictionContext->
/*
* When FastPathRouterQuery() returns true, we know that standard_planner() has
* not been called. Thus, restriction information is not avaliable and we do the
* shard pruning based on the distribution column in the quals of the query.
*/
if (FastPathRouterQuery(originalQuery))
{
List *shardIntervalList =
TargetShardIntervalForFastPathQuery(originalQuery, partitionValueConst,
&isMultiShardQuery);
/*
* This could only happen when there is a parameter on the distribution key.
* We defer error here, later the planner is forced to use a generic plan
* by assigning arbitrarily high cost to the plan.
*/
if (UpdateOrDeleteQuery(originalQuery) && isMultiShardQuery)
{
planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
"Router planner cannot handle multi-shard "
"modify queries", NULL, NULL);
return planningError;
}
prunedRelationShardList = list_make1(shardIntervalList);
if (!isMultiShardQuery)
{
ereport(DEBUG2, (errmsg("Distributed planning for a fast-path router "
"query")));
}
}
else
{
prunedRelationShardList =
TargetShardIntervalsForRestrictInfo(plannerRestrictionContext->
relationRestrictionContext,
&isMultiShardQuery,
partitionValueConst);
}
if (isMultiShardQuery)
{
@ -2065,18 +2101,58 @@ GetInitialShardId(List *relationShardList)
/*
* TargetShardIntervalsForQuery performs shard pruning for all referenced relations
* in the query and returns list of shards per relation. Shard pruning is done based
* on provided restriction context per relation. The function sets multiShardQuery
* to true if any of the relations pruned down to more than one active shard. It
* also records pruned shard intervals in relation restriction context to be used
* later on. Some queries may have contradiction clauses like 'and false' or
* 'and 1=0', such queries are treated as if all of the shards of joining
* relations are pruned out.
* TargetShardIntervalForFastPathQuery gets a query which is in
* the form defined by FastPathRouterQuery() and returns exactly
* one shard interval (see FastPathRouterQuery() for the detail).
*
* Also set the outgoing partition column value if requested via
* partitionValueConst
*/
static List *
TargetShardIntervalForFastPathQuery(Query *query, Const **partitionValueConst,
bool *isMultiShardQuery)
{
Const *queryPartitionValueConst = NULL;
Oid relationId = ExtractFirstDistributedTableId(query);
Node *quals = query->jointree->quals;
int relationIndex = 1;
List *prunedShardIntervalList =
PruneShards(relationId, relationIndex, make_ands_implicit((Expr *) quals),
&queryPartitionValueConst);
/* we're only expecting single shard from a single table */
Assert(FastPathRouterQuery(query));
if (list_length(prunedShardIntervalList) > 1)
{
*isMultiShardQuery = true;
}
else if (list_length(prunedShardIntervalList) == 1 &&
partitionValueConst != NULL)
{
/* set the outgoing partition column value if requested */
*partitionValueConst = queryPartitionValueConst;
}
return prunedShardIntervalList;
}
/*
* TargetShardIntervalsForRestrictInfo performs shard pruning for all referenced
* relations in the relation restriction context and returns list of shards per
* relation. Shard pruning is done based on provided restriction context per relation.
* The function sets multiShardQuery to true if any of the relations pruned down to
* more than one active shard. It also records pruned shard intervals in relation
* restriction context to be used later on. Some queries may have contradiction
* clauses like 'and false' or 'and 1=0', such queries are treated as if all of
* the shards of joining relations are pruned out.
*/
List *
TargetShardIntervalsForQuery(Query *query,
RelationRestrictionContext *restrictionContext,
TargetShardIntervalsForRestrictInfo(RelationRestrictionContext *restrictionContext,
bool *multiShardQuery, Const **partitionValueConst)
{
List *prunedRelationShardList = NIL;
@ -2832,10 +2908,11 @@ ExtractInsertPartitionKeyValue(Query *query)
* flag to false.
*/
static bool
MultiRouterPlannableQuery(Query *query, RelationRestrictionContext *restrictionContext)
MultiRouterPlannableQuery(Query *query)
{
CmdType commandType = query->commandType;
ListCell *relationRestrictionContextCell = NULL;
List *rangeTableRelationList = NIL;
ListCell *rangeTableRelationCell = NULL;
if (commandType == CMD_INSERT || commandType == CMD_UPDATE ||
commandType == CMD_DELETE)
@ -2850,11 +2927,10 @@ MultiRouterPlannableQuery(Query *query, RelationRestrictionContext *restrictionC
return false;
}
foreach(relationRestrictionContextCell, restrictionContext->relationRestrictionList)
ExtractRangeTableRelationWalker((Node *) query, &rangeTableRelationList);
foreach(rangeTableRelationCell, rangeTableRelationList)
{
RelationRestriction *relationRestriction =
(RelationRestriction *) lfirst(relationRestrictionContextCell);
RangeTblEntry *rte = relationRestriction->rte;
RangeTblEntry *rte = (RangeTblEntry *) lfirst(rangeTableRelationCell);
if (rte->rtekind == RTE_RELATION)
{
/* only hash partitioned tables are supported */

19
src/backend/distributed/planner/planner_readme.md
View File

@ -2,14 +2,23 @@
The distributed query planner is entered through the `distributed_planner` function in `distributed_planner.c`. This is the hook that Postgres calls instead of `standard_planner`.
We always first call `standard_planner` to build a `PlannedStmt`. For queries containing a distributed table or reference table, we then proceed with distributed planning, which overwrites the `planTree` in the `PlannedStmt`.
If the input query is trivial (e.g., no joins, no subqueries/ctes, single table and single shard), we create a very simple `PlannedStmt`. If the query is not trivial, call `standard_planner` to build a `PlannedStmt`. For queries containing a distributed table or reference table, we then proceed with distributed planning, which overwrites the `planTree` in the `PlannedStmt`.
Distributed planning (`CreateDistributedPlan`) tries several different methods to plan the query:
1. Router planner, proceed if the query prunes down to a single set of co-located shards
2. Modification planning, proceed if the query is a DML command and all joins are co-located
3. Recursive planning, find CTEs and subqueries that cannot be pushed down and go back to 1
4. Logical planner, constructs a multi-relational algebra tree to find a distributed execution plan
1. Fast-path router planner, proceed if the query prunes down to a single shard of a single table
2. Router planner, proceed if the query prunes down to a single set of co-located shards
3. Modification planning, proceed if the query is a DML command and all joins are co-located
4. Recursive planning, find CTEs and subqueries that cannot be pushed down and go back to 1
5. Logical planner, constructs a multi-relational algebra tree to find a distributed execution plan
## Fast-path router planner
By examining the query tree, if we can decide that the query hits only a single shard of a single table, we can skip calling `standard_planner()`. Later on the execution, we simply fetch the filter on the distribution key and do the pruning.
As the name reveals, this can be considered as a sub-item of Router planner described below. The only difference is that fast-path planner doesn't rely on `standard_planner()` for collecting restriction information.
## Router planner

154
src/backend/distributed/planner/query_pushdown_planning.c
View File

@ -79,9 +79,12 @@ static bool IsRecurringRTE(RangeTblEntry *rangeTableEntry,
static bool IsRecurringRangeTable(List *rangeTable, RecurringTuplesType *recurType);
static bool HasRecurringTuples(Node *node, RecurringTuplesType *recurType);
static MultiNode * SubqueryPushdownMultiNodeTree(Query *queryTree);
static void FlattenJoinVars(List *columnList, Query *queryTree);
static List * FlattenJoinVars(List *columnList, Query *queryTree);
static void UpdateVarMappingsForExtendedOpNode(List *columnList,
List *flattenedColumnList,
List *subqueryTargetEntryList);
static void UpdateColumnToMatchingTargetEntry(Var *column, Node *flattenedExpr,
List *targetEntryList);
static MultiTable * MultiSubqueryPushdownTable(Query *subquery);
static List * CreateSubqueryTargetEntryList(List *columnList);
static bool RelationInfoContainsOnlyRecurringTuples(PlannerInfo *plannerInfo,
@ -1413,6 +1416,7 @@ SubqueryPushdownMultiNodeTree(Query *queryTree)
{
List *targetEntryList = queryTree->targetList;
List *columnList = NIL;
List *flattenedExprList = NIL;
List *targetColumnList = NIL;
MultiCollect *subqueryCollectNode = CitusMakeNode(MultiCollect);
MultiTable *subqueryNode = NULL;
@ -1472,25 +1476,26 @@ SubqueryPushdownMultiNodeTree(Query *queryTree)
*/
/*
* uniqueColumnList contains all columns returned by subquery. Subquery target
* columnList contains all columns returned by subquery. Subquery target
* entry list, subquery range table entry's column name list are derived from
* uniqueColumnList. Columns mentioned in multiProject node and multiExtendedOp
* node are indexed with their respective position in uniqueColumnList.
* columnList. Columns mentioned in multiProject node and multiExtendedOp
* node are indexed with their respective position in columnList.
*/
targetColumnList = pull_var_clause_default((Node *) targetEntryList);
havingClauseColumnList = pull_var_clause_default(queryTree->havingQual);
columnList = list_concat(targetColumnList, havingClauseColumnList);
FlattenJoinVars(columnList, queryTree);
flattenedExprList = FlattenJoinVars(columnList, queryTree);
/* create a target entry for each unique column */
subqueryTargetEntryList = CreateSubqueryTargetEntryList(columnList);
subqueryTargetEntryList = CreateSubqueryTargetEntryList(flattenedExprList);
/*
* Update varno/varattno fields of columns in columnList to
* point to corresponding target entry in subquery target entry list.
*/
UpdateVarMappingsForExtendedOpNode(columnList, subqueryTargetEntryList);
UpdateVarMappingsForExtendedOpNode(columnList, flattenedExprList,
subqueryTargetEntryList);
/* new query only has target entries, join tree, and rtable*/
pushedDownQuery = makeNode(Query);
@ -1553,7 +1558,8 @@ SubqueryPushdownMultiNodeTree(Query *queryTree)
/*
* FlattenJoinVars iterates over provided columnList to identify
* Var's that are referenced from join RTE, and reverts back to their
* original RTEs.
* original RTEs. Then, returns a new list with reverted types. Note that,
* length of the original list and created list must be equal.
*
* This is required because Postgres allows columns to be referenced using
* a join alias. Therefore the same column from a table could be referenced
@ -1568,12 +1574,16 @@ SubqueryPushdownMultiNodeTree(Query *queryTree)
* Only exception is that, if a join is given an alias name, we do not want to
* flatten those var's. If we do, deparsing fails since it expects to see a join
* alias, and cannot access the RTE in the join tree by their names.
*
* Also note that in case of full outer joins, a column could be flattened to a
* coalesce expression if the column appears in the USING clause.
*/
static void
static List *
FlattenJoinVars(List *columnList, Query *queryTree)
{
ListCell *columnCell = NULL;
List *rteList = queryTree->rtable;
List *flattenedExprList = NIL;
foreach(columnCell, columnList)
{
@ -1595,7 +1605,7 @@ FlattenJoinVars(List *columnList, Query *queryTree)
columnRte = rt_fetch(column->varno, rteList);
if (columnRte->rtekind == RTE_JOIN && columnRte->alias == NULL)
{
Var *normalizedVar = NULL;
Node *normalizedNode = NULL;
if (root == NULL)
{
@ -1605,15 +1615,18 @@ FlattenJoinVars(List *columnList, Query *queryTree)
root->hasJoinRTEs = true;
}
normalizedVar = (Var *) flatten_join_alias_vars(root, (Node *) column);
normalizedNode = strip_implicit_coercions(flatten_join_alias_vars(root,
(Node *)
column));
flattenedExprList = lappend(flattenedExprList, copyObject(normalizedNode));
}
else
{
flattenedExprList = lappend(flattenedExprList, copyObject(column));
}
}
/*
* We need to copy values over existing one to make sure it is updated on
* respective places.
*/
memcpy(column, normalizedVar, sizeof(Var));
}
}
return flattenedExprList;
}
@ -1622,28 +1635,28 @@ FlattenJoinVars(List *columnList, Query *queryTree)
* in the column list and returns the target entry list.
*/
static List *
CreateSubqueryTargetEntryList(List *columnList)
CreateSubqueryTargetEntryList(List *exprList)
{
AttrNumber resNo = 1;
ListCell *columnCell = NULL;
List *uniqueColumnList = NIL;
ListCell *exprCell = NULL;
List *uniqueExprList = NIL;
List *subqueryTargetEntryList = NIL;
foreach(columnCell, columnList)
foreach(exprCell, exprList)
{
Var *column = (Var *) lfirst(columnCell);
uniqueColumnList = list_append_unique(uniqueColumnList, copyObject(column));
Node *expr = (Node *) lfirst(exprCell);
uniqueExprList = list_append_unique(uniqueExprList, expr);
}
foreach(columnCell, uniqueColumnList)
foreach(exprCell, uniqueExprList)
{
Var *column = (Var *) lfirst(columnCell);
Node *expr = (Node *) lfirst(exprCell);
TargetEntry *newTargetEntry = makeNode(TargetEntry);
StringInfo columnNameString = makeStringInfo();
StringInfo exprNameString = makeStringInfo();
newTargetEntry->expr = (Expr *) copyObject(column);
appendStringInfo(columnNameString, WORKER_COLUMN_FORMAT, resNo);
newTargetEntry->resname = columnNameString->data;
newTargetEntry->expr = (Expr *) copyObject(expr);
appendStringInfo(exprNameString, WORKER_COLUMN_FORMAT, resNo);
newTargetEntry->resname = exprNameString->data;
newTargetEntry->resjunk = false;
newTargetEntry->resno = resNo;
@ -1661,28 +1674,89 @@ CreateSubqueryTargetEntryList(List *columnList)
* list.
*/
static void
UpdateVarMappingsForExtendedOpNode(List *columnList, List *subqueryTargetEntryList)
UpdateVarMappingsForExtendedOpNode(List *columnList, List *flattenedExprList,
List *subqueryTargetEntryList)
{
ListCell *columnCell = NULL;
foreach(columnCell, columnList)
ListCell *flattenedExprCell = NULL;
Assert(list_length(columnList) == list_length(flattenedExprList));
forboth(columnCell, columnList, flattenedExprCell, flattenedExprList)
{
Var *columnOnTheExtendedNode = (Var *) lfirst(columnCell);
Node *flattenedExpr = (Node *) lfirst(flattenedExprCell);
/*
* As an optimization, subqueryTargetEntryList only consists of
* distinct elements. In other words, any duplicate entries in the
* target list consolidated into a single element to prevent pulling
* unnecessary data from the worker nodes (e.g. SELECT a,a,a,b,b,b FROM x;
* is turned into SELECT a,b FROM x_102008).
*
* Thus, at this point we should iterate on the subqueryTargetEntryList
* and ensure that the column on the extended op node points to the
* correct target entry.
*/
UpdateColumnToMatchingTargetEntry(columnOnTheExtendedNode, flattenedExpr,
subqueryTargetEntryList);
}
}
/*
* UpdateColumnToMatchingTargetEntry sets the variable of given column entry to
* the matching entry of the targetEntryList. Since data type of the column can
* be different from the types of the elements of targetEntryList, we use flattenedExpr.
*/
static void
UpdateColumnToMatchingTargetEntry(Var *column, Node *flattenedExpr, List *targetEntryList)
{
ListCell *targetEntryCell = NULL;
foreach(targetEntryCell, subqueryTargetEntryList)
foreach(targetEntryCell, targetEntryList)
{
TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell);
Var *targetColumn = NULL;
Assert(IsA(targetEntry->expr, Var));
targetColumn = (Var *) targetEntry->expr;
if (columnOnTheExtendedNode->varno == targetColumn->varno &&
columnOnTheExtendedNode->varattno == targetColumn->varattno)
if (IsA(targetEntry->expr, Var))
{
columnOnTheExtendedNode->varno = 1;
columnOnTheExtendedNode->varattno = targetEntry->resno;
Var *targetEntryVar = (Var *) targetEntry->expr;
if (IsA(flattenedExpr, Var) && equal(flattenedExpr, targetEntryVar))
{
column->varno = 1;
column->varattno = targetEntry->resno;
break;
}
}
else if (IsA(targetEntry->expr, CoalesceExpr))
{
/*
* flatten_join_alias_vars() flattens full oter joins' columns that is
* in the USING part into COALESCE(left_col, right_col)
*/
CoalesceExpr *targetCoalesceExpr = (CoalesceExpr *) targetEntry->expr;
if (IsA(flattenedExpr, CoalesceExpr) && equal(flattenedExpr,
targetCoalesceExpr))
{
Oid expressionType = exprType(flattenedExpr);
int32 expressionTypmod = exprTypmod(flattenedExpr);
Oid expressionCollation = exprCollation(flattenedExpr);
column->varno = 1;
column->varattno = targetEntry->resno;
column->vartype = expressionType;
column->vartypmod = expressionTypmod;
column->varcollid = expressionCollation;
break;
}
}
else
{
elog(ERROR, "unrecognized node type on the target list: %d",
nodeTag(targetEntry->expr));
}
}
}

237
src/backend/distributed/planner/recursive_planning.c
View File

@ -50,6 +50,7 @@
*/
#include "postgres.h"
#include "funcapi.h"
#include "catalog/pg_type.h"
#include "catalog/pg_class.h"
@ -66,6 +67,7 @@
#include "distributed/query_colocation_checker.h"
#include "distributed/recursive_planning.h"
#include "distributed/relation_restriction_equivalence.h"
#include "distributed/version_compat.h"
#include "lib/stringinfo.h"
#include "optimizer/planner.h"
#include "optimizer/prep.h"
@ -162,7 +164,9 @@ static bool CteReferenceListWalker(Node *node, CteReferenceWalkerContext *contex
static bool ContainsReferencesToOuterQuery(Query *query);
static bool ContainsReferencesToOuterQueryWalker(Node *node,
VarLevelsUpWalkerContext *context);
static void WrapFunctionsInSubqueries(Query *query);
static void TransformFunctionRTE(RangeTblEntry *rangeTblEntry);
static bool ShouldTransformRTE(RangeTblEntry *rangeTableEntry);
/*
* GenerateSubplansForSubqueriesAndCTEs is a wrapper around RecursivelyPlanSubqueriesAndCTEs.
@ -263,6 +267,9 @@ RecursivelyPlanSubqueriesAndCTEs(Query *query, RecursivePlanningContext *context
return NULL;
}
/* make sure function calls in joins are executed in the coordinator */
WrapFunctionsInSubqueries(query);
/* descend into subqueries */
query_tree_walker(query, RecursivelyPlanSubqueryWalker, context, 0);
@ -1305,6 +1312,234 @@ ContainsReferencesToOuterQueryWalker(Node *node, VarLevelsUpWalkerContext *conte
}
/*
* WrapFunctionsInSubqueries iterates over all the immediate Range Table Entries
* of a query and wraps the functions inside (SELECT * FROM fnc() f)
* subqueries, so that those functions will be executed on the coordinator if
* necessary.
*
* We wrap all the functions that are used in joins except the ones that are
* laterally joined or have WITH ORDINALITY clauses.
* */
static void
WrapFunctionsInSubqueries(Query *query)
{
List *rangeTableList = query->rtable;
ListCell *rangeTableCell = NULL;
/*
* If we have only one function call in a query without any joins, we can
* easily decide where to execute it.
*
* If there are some subqueries and/or functions that are joined with a
* function, it is not trivial to decide whether we should run this
* function in the coordinator or in workers and therefore we may need to
* wrap some of those functions in subqueries.
*
* If we have only one RTE, we leave the parsed query tree as it is. This
* also makes sure we do not wrap an already wrapped function call
* because we know that there will always be 1 RTE in a wrapped function.
* */
if (list_length(rangeTableList) < 2)
{
return;
}
/* iterate over all RTEs and wrap them if necessary */
foreach(rangeTableCell, rangeTableList)
{
RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell);
if (ShouldTransformRTE(rangeTableEntry))
{
TransformFunctionRTE(rangeTableEntry);
}
}
}
/*
* TransformFunctionRTE wraps a given function RangeTableEntry
* inside a (SELECT * from function() f) subquery.
*
* The said RangeTableEntry is modified and now points to the new subquery.
* */
static void
TransformFunctionRTE(RangeTblEntry *rangeTblEntry)
{
Query *subquery = makeNode(Query);
RangeTblRef *newRangeTableRef = makeNode(RangeTblRef);
RangeTblEntry *newRangeTableEntry = NULL;
Var *targetColumn = NULL;
TargetEntry *targetEntry = NULL;
RangeTblFunction *rangeTblFunction = NULL;
AttrNumber targetColumnIndex = 0;
TupleDesc tupleDesc = NULL;
rangeTblFunction = linitial(rangeTblEntry->functions);
subquery->commandType = CMD_SELECT;
/* copy the input rangeTblEntry to prevent cycles */
newRangeTableEntry = copyObject(rangeTblEntry);
/* set the FROM expression to the subquery */
subquery->rtable = list_make1(newRangeTableEntry);
newRangeTableRef->rtindex = 1;
subquery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL);
/* Determine the result type of the function.
*
* If function return type is not composite or rowtype can't be determined,
* tupleDesc is set to null here
*/
tupleDesc = (TupleDesc) get_expr_result_tupdesc(rangeTblFunction->funcexpr,
true);
/*
* If tupleDesc is not null, we iterate over all the attributes and
* create targetEntries
* */
if (tupleDesc)
{
/*
* A sample function join that end up here:
*
* CREATE FUNCTION f(..) RETURNS TABLE(c1 int, c2 text) AS .. ;
* SELECT .. FROM table JOIN f(..) ON ( .. ) ;
*
* We will iterate over Tuple Description attributes. i.e (c1 int, c2 text)
*/
for (targetColumnIndex = 0; targetColumnIndex < tupleDesc->natts;
targetColumnIndex++)
{
FormData_pg_attribute *attribute = TupleDescAttr(tupleDesc,
targetColumnIndex);
Oid columnType = attribute->atttypid;
char *columnName = attribute->attname.data;
/*
* The indexing of attributes and TupleDesc and varattno differ
*
* varattno=0 corresponds to whole row
* varattno=1 corresponds to first column that is stored in tupDesc->attrs[0]
*
* That's why we need to add one to the targetColumnIndex
* */
targetColumn = makeVar(1, targetColumnIndex + 1, columnType, -1, InvalidOid,
0);
targetEntry = makeTargetEntry((Expr *) targetColumn, targetColumnIndex + 1,
columnName, false);
subquery->targetList = lappend(subquery->targetList, targetEntry);
}
}
/*
* If tupleDesc is NULL we have 2 different cases:
*
* 1. The function returns a record but the attributes can not be
* determined just by looking at the function definition. In this case the
* column names and types must be defined explicitly in the query
*
* 2. The function returns a non-composite type (e.g. int, text, jsonb ..)
* */
else
{
/* create target entries for all columns returned by the function */
List *functionColumnNames = NULL;
ListCell *functionColumnName = NULL;
functionColumnNames = rangeTblEntry->eref->colnames;
foreach(functionColumnName, functionColumnNames)
{
char *columnName = strVal(lfirst(functionColumnName));
Oid columnType = InvalidOid;
/*
* If the function returns a set of records, the query needs
* to explicitly name column names and types
*
* Use explicitly defined types in the query if they are
* available
* */
if (list_length(rangeTblFunction->funccoltypes) > 0)
{
/*
* A sample function join that end up here:
*
* CREATE FUNCTION get_set_of_records() RETURNS SETOF RECORD AS
* $cmd$
* SELECT x, x+1 FROM generate_series(0,4) f(x)
* $cmd$
* LANGUAGE SQL;
*
* SELECT *
* FROM table1 JOIN get_set_of_records() AS t2(x int, y int)
* ON (id = x);
*
* Note that the function definition does not have column
* names and types. Therefore the user needs to explicitly
* state them in the query
* */
columnType = list_nth_oid(rangeTblFunction->funccoltypes,
targetColumnIndex);
}
/* use the types in the function definition otherwise */
else
{
/*
* Only functions returning simple types end up here.
* A sample function:
*
* CREATE FUNCTION add(integer, integer) RETURNS integer AS
* 'SELECT $1 + $2;'
* LANGUAGE SQL;
* SELECT * FROM table JOIN add(3,5) sum ON ( .. ) ;
* */
FuncExpr *funcExpr = (FuncExpr *) rangeTblFunction->funcexpr;
columnType = funcExpr->funcresulttype;
}
/* Note that the column k is associated with varattno/resno of k+1 */
targetColumn = makeVar(1, targetColumnIndex + 1, columnType, -1,
InvalidOid, 0);
targetEntry = makeTargetEntry((Expr *) targetColumn,
targetColumnIndex + 1, columnName, false);
subquery->targetList = lappend(subquery->targetList, targetEntry);
targetColumnIndex++;
}
}
/* replace the function with the constructed subquery */
rangeTblEntry->rtekind = RTE_SUBQUERY;
rangeTblEntry->subquery = subquery;
}
/*
* ShouldTransformRTE determines whether a given RTE should bne wrapped in a
* subquery.
*
* Not all functions should be wrapped in a subquery for now. As we support more
* functions to be used in joins, the constraints here will be relaxed.
* */
static bool
ShouldTransformRTE(RangeTblEntry *rangeTableEntry)
{
/*
* We should wrap only function rtes that are not LATERAL and
* without WITH CARDINALITY clause
* */
if (rangeTableEntry->rtekind != RTE_FUNCTION ||
rangeTableEntry->lateral ||
rangeTableEntry->funcordinality)
{
return false;
}
return true;
}
/*
* BuildSubPlanResultQuery returns a query of the form:
*

10
src/backend/distributed/shared_library_init.c
View File

@ -411,6 +411,16 @@ RegisterCitusConfigVariables(void)
GUC_NO_SHOW_ALL,
NULL, NULL, NULL);
DefineCustomBoolVariable(
"citus.enable_fast_path_router_planner",
gettext_noop("Enables fast path router planner"),
NULL,
&EnableFastPathRouterPlanner,
true,
PGC_USERSET,
GUC_NO_SHOW_ALL,
NULL, NULL, NULL);
DefineCustomBoolVariable(
"citus.override_table_visibility",
gettext_noop("Enables replacing occurencens of pg_catalog.pg_table_visible() "

6
src/backend/distributed/test/run_from_same_connection.c
View File

@ -196,6 +196,7 @@ GetRemoteProcessId(MultiConnection *connection)
StringInfo queryStringInfo = makeStringInfo();
PGresult *result = NULL;
int64 rowCount = 0;
int64 resultValue = 0;
appendStringInfo(queryStringInfo, GET_PROCESS_ID);
@ -208,7 +209,10 @@ GetRemoteProcessId(MultiConnection *connection)
PG_RETURN_VOID();
}
resultValue = ParseIntField(result, 0, 0);
PQclear(result);
ClearResults(connection, false);
return ParseIntField(result, 0, 0);
return resultValue;
}

11
src/backend/distributed/transaction/backend_data.c
View File

@ -993,3 +993,14 @@ ActiveDistributedTransactionNumbers(void)
return activeTransactionNumberList;
}
/*
* GetMyProcLocalTransactionId() is a wrapper for
* getting lxid of MyProc.
*/
LocalTransactionId
GetMyProcLocalTransactionId(void)
{
return MyProc->lxid;
}

2
src/backend/distributed/transaction/remote_transaction.c
View File

@ -1323,7 +1323,7 @@ Assign2PCIdentifier(MultiConnection *connection)
*/
bool
ParsePreparedTransactionName(char *preparedTransactionName,
int *groupId, int *procId,
int32 *groupId, int *procId,
uint64 *transactionNumber,
uint32 *connectionNumber)
{

6
src/backend/distributed/transaction/transaction_recovery.c
View File

@ -78,7 +78,7 @@ recover_prepared_transactions(PG_FUNCTION_ARGS)
* prepared transaction should be committed.
*/
void
LogTransactionRecord(int groupId, char *transactionName)
LogTransactionRecord(int32 groupId, char *transactionName)
{
Relation pgDistTransaction = NULL;
TupleDesc tupleDescriptor = NULL;
@ -141,7 +141,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode)
{
int recoveredTransactionCount = 0;
int groupId = workerNode->groupId;
int32 groupId = workerNode->groupId;
char *nodeName = workerNode->workerName;
int nodePort = workerNode->workerPort;
@ -461,7 +461,7 @@ PendingWorkerTransactionList(MultiConnection *connection)
static bool
IsTransactionInProgress(HTAB *activeTransactionNumberSet, char *preparedTransactionName)
{
int groupId = 0;
int32 groupId = 0;
int procId = 0;
uint32 connectionNumber = 0;
uint64 transactionNumber = 0;

2
src/backend/distributed/utils/citus_copyfuncs.c
View File

@ -33,7 +33,7 @@ CitusSetTag(Node *node, int tag)
#define DECLARE_FROM_AND_NEW_NODE(nodeTypeName) \
nodeTypeName * newnode = (nodeTypeName *) \
nodeTypeName *newnode = (nodeTypeName *) \
CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \
nodeTypeName *from = (nodeTypeName *) source_node

4
src/backend/distributed/utils/citus_outfuncs.c
View File

@ -402,7 +402,7 @@ OutShardPlacement(OUTFUNC_ARGS)
WRITE_UINT64_FIELD(shardId);
WRITE_UINT64_FIELD(shardLength);
WRITE_ENUM_FIELD(shardState, RelayFileState);
WRITE_UINT_FIELD(groupId);
WRITE_INT_FIELD(groupId);
WRITE_STRING_FIELD(nodeName);
WRITE_UINT_FIELD(nodePort);
/* so we can deal with 0 */
@ -422,7 +422,7 @@ OutGroupShardPlacement(OUTFUNC_ARGS)
WRITE_UINT64_FIELD(shardId);
WRITE_UINT64_FIELD(shardLength);
WRITE_ENUM_FIELD(shardState, RelayFileState);
WRITE_UINT_FIELD(groupId);
WRITE_INT_FIELD(groupId);
}

4
src/backend/distributed/utils/citus_readfuncs.c
View File

@ -312,7 +312,7 @@ ReadShardPlacement(READFUNC_ARGS)
READ_UINT64_FIELD(shardId);
READ_UINT64_FIELD(shardLength);
READ_ENUM_FIELD(shardState, RelayFileState);
READ_UINT_FIELD(groupId);
READ_INT_FIELD(groupId);
READ_STRING_FIELD(nodeName);
READ_UINT_FIELD(nodePort);
/* so we can deal with 0 */
@ -333,7 +333,7 @@ ReadGroupShardPlacement(READFUNC_ARGS)
READ_UINT64_FIELD(shardId);
READ_UINT64_FIELD(shardLength);
READ_ENUM_FIELD(shardState, RelayFileState);
READ_UINT_FIELD(groupId);
READ_INT_FIELD(groupId);
READ_DONE();
}

24
src/backend/distributed/utils/metadata_cache.c
View File

@ -160,7 +160,7 @@ static int WorkerNodeCount = 0;
static bool workerNodeHashValid = false;
/* default value is -1, for coordinator it's 0 and for worker nodes > 0 */
static int LocalGroupId = -1;
static int32 LocalGroupId = -1;
/* built first time through in InitializePartitionCache */
static ScanKeyData DistPartitionScanKey[1];
@ -210,7 +210,7 @@ static ShardInterval * TupleToShardInterval(HeapTuple heapTuple,
static void CachedRelationLookup(const char *relationName, Oid *cachedOid);
static ShardPlacement * ResolveGroupShardPlacement(
GroupShardPlacement *groupShardPlacement, ShardCacheEntry *shardEntry);
static WorkerNode * LookupNodeForGroup(uint32 groupid);
static WorkerNode * LookupNodeForGroup(int32 groupId);
static Oid LookupEnumValueId(Oid typeId, char *valueName);
static void InvalidateEntireDistCache(void);
@ -474,7 +474,7 @@ LoadShardPlacement(uint64 shardId, uint64 placementId)
* on the group.
*/
ShardPlacement *
FindShardPlacementOnGroup(uint32 groupId, uint64 shardId)
FindShardPlacementOnGroup(int32 groupId, uint64 shardId)
{
ShardCacheEntry *shardEntry = NULL;
DistTableCacheEntry *tableEntry = NULL;
@ -516,7 +516,7 @@ ResolveGroupShardPlacement(GroupShardPlacement *groupShardPlacement,
ShardInterval *shardInterval = tableEntry->sortedShardIntervalArray[shardIndex];
ShardPlacement *shardPlacement = CitusMakeNode(ShardPlacement);
uint32 groupId = groupShardPlacement->groupId;
int32 groupId = groupShardPlacement->groupId;
WorkerNode *workerNode = LookupNodeForGroup(groupId);
/* copy everything into shardPlacement but preserve the header */
@ -583,7 +583,7 @@ LookupNodeByNodeId(uint32 nodeId)
* appropriate error message.
*/
static WorkerNode *
LookupNodeForGroup(uint32 groupId)
LookupNodeForGroup(int32 groupId)
{
bool foundAnyNodes = false;
int workerNodeIndex = 0;
@ -593,7 +593,7 @@ LookupNodeForGroup(uint32 groupId)
for (workerNodeIndex = 0; workerNodeIndex < WorkerNodeCount; workerNodeIndex++)
{
WorkerNode *workerNode = WorkerNodeArray[workerNodeIndex];
uint32 workerNodeGroupId = workerNode->groupId;
int32 workerNodeGroupId = workerNode->groupId;
if (workerNodeGroupId != groupId)
{
continue;
@ -609,7 +609,7 @@ LookupNodeForGroup(uint32 groupId)
if (!foundAnyNodes)
{
ereport(ERROR, (errmsg("there is a shard placement in node group %u but "
ereport(ERROR, (errmsg("there is a shard placement in node group %d but "
"there are no nodes in that group", groupId)));
}
@ -617,13 +617,13 @@ LookupNodeForGroup(uint32 groupId)
{
case USE_SECONDARY_NODES_NEVER:
{
ereport(ERROR, (errmsg("node group %u does not have a primary node",
ereport(ERROR, (errmsg("node group %d does not have a primary node",
groupId)));
}
case USE_SECONDARY_NODES_ALWAYS:
{
ereport(ERROR, (errmsg("node group %u does not have a secondary node",
ereport(ERROR, (errmsg("node group %d does not have a secondary node",
groupId)));
}
@ -2801,7 +2801,7 @@ RegisterWorkerNodeCacheCallbacks(void)
* that pg_dist_local_node_group has exactly one row and has at least one column.
* Otherwise, the function errors out.
*/
int
int32
GetLocalGroupId(void)
{
SysScanDesc scanDescriptor = NULL;
@ -2809,7 +2809,7 @@ GetLocalGroupId(void)
int scanKeyCount = 0;
HeapTuple heapTuple = NULL;
TupleDesc tupleDescriptor = NULL;
Oid groupId = InvalidOid;
int32 groupId = 0;
Relation pgDistLocalGroupId = NULL;
Oid localGroupTableOid = InvalidOid;
@ -2846,7 +2846,7 @@ GetLocalGroupId(void)
Anum_pg_dist_local_groupid,
tupleDescriptor, &isNull);
groupId = DatumGetUInt32(groupIdDatum);
groupId = DatumGetInt32(groupIdDatum);
}
else
{

16
src/backend/distributed/utils/node_metadata.c
View File

@ -63,7 +63,7 @@ static HeapTuple GetNodeTuple(char *nodeName, int32 nodePort);
static Datum GenerateNodeTuple(WorkerNode *workerNode);
static int32 GetNextGroupId(void);
static int GetNextNodeId(void);
static void InsertNodeRow(int nodeid, char *nodename, int32 nodeport, uint32 groupId,
static void InsertNodeRow(int nodeid, char *nodename, int32 nodeport, int32 groupId,
char *nodeRack, bool hasMetadata, bool isActive, Oid nodeRole,
char *nodeCluster);
static void DeleteNodeRow(char *nodename, int32 nodeport);
@ -395,7 +395,7 @@ WorkerNodeIsReadable(WorkerNode *workerNode)
* it will set the bool groupContainsNodes references to true.
*/
WorkerNode *
PrimaryNodeForGroup(uint32 groupId, bool *groupContainsNodes)
PrimaryNodeForGroup(int32 groupId, bool *groupContainsNodes)
{
WorkerNode *workerNode = NULL;
HASH_SEQ_STATUS status;
@ -405,7 +405,7 @@ PrimaryNodeForGroup(uint32 groupId, bool *groupContainsNodes)
while ((workerNode = hash_seq_search(&status)) != NULL)
{
uint32 workerNodeGroupId = workerNode->groupId;
int32 workerNodeGroupId = workerNode->groupId;
if (workerNodeGroupId != groupId)
{
continue;
@ -1116,7 +1116,7 @@ GenerateNodeTuple(WorkerNode *workerNode)
memset(isNulls, false, sizeof(isNulls));
values[Anum_pg_dist_node_nodeid - 1] = UInt32GetDatum(workerNode->nodeId);
values[Anum_pg_dist_node_groupid - 1] = UInt32GetDatum(workerNode->groupId);
values[Anum_pg_dist_node_groupid - 1] = Int32GetDatum(workerNode->groupId);
values[Anum_pg_dist_node_nodename - 1] = CStringGetTextDatum(workerNode->workerName);
values[Anum_pg_dist_node_nodeport - 1] = UInt32GetDatum(workerNode->workerPort);
values[Anum_pg_dist_node_noderack - 1] = CStringGetTextDatum(workerNode->workerRack);
@ -1166,7 +1166,7 @@ GetNextGroupId()
SetUserIdAndSecContext(savedUserId, savedSecurityContext);
groupId = DatumGetUInt32(groupIdDatum);
groupId = DatumGetInt32(groupIdDatum);
return groupId;
}
@ -1232,7 +1232,7 @@ EnsureCoordinator(void)
* an existing group. If you don't it's possible for the metadata to become inconsistent.
*/
static void
InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, uint32 groupId, char *nodeRack,
InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, int32 groupId, char *nodeRack,
bool hasMetadata, bool isActive, Oid nodeRole, char *nodeCluster)
{
Relation pgDistNode = NULL;
@ -1249,7 +1249,7 @@ InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, uint32 groupId, char *
memset(isNulls, false, sizeof(isNulls));
values[Anum_pg_dist_node_nodeid - 1] = UInt32GetDatum(nodeid);
values[Anum_pg_dist_node_groupid - 1] = UInt32GetDatum(groupId);
values[Anum_pg_dist_node_groupid - 1] = Int32GetDatum(groupId);
values[Anum_pg_dist_node_nodename - 1] = CStringGetTextDatum(nodeName);
values[Anum_pg_dist_node_nodeport - 1] = UInt32GetDatum(nodePort);
values[Anum_pg_dist_node_noderack - 1] = CStringGetTextDatum(nodeRack);
@ -1500,7 +1500,7 @@ TupleToWorkerNode(TupleDesc tupleDescriptor, HeapTuple heapTuple)
workerNode = (WorkerNode *) palloc0(sizeof(WorkerNode));
workerNode->nodeId = DatumGetUInt32(datumArray[Anum_pg_dist_node_nodeid - 1]);
workerNode->workerPort = DatumGetUInt32(datumArray[Anum_pg_dist_node_nodeport - 1]);
workerNode->groupId = DatumGetUInt32(datumArray[Anum_pg_dist_node_groupid - 1]);
workerNode->groupId = DatumGetInt32(datumArray[Anum_pg_dist_node_groupid - 1]);
strlcpy(workerNode->workerName, TextDatumGetCString(nodeName), WORKER_LENGTH);
strlcpy(workerNode->workerRack, TextDatumGetCString(nodeRack), WORKER_LENGTH);
workerNode->hasMetadata = DatumGetBool(datumArray[Anum_pg_dist_node_hasmetadata - 1]);

4
src/backend/distributed/utils/reference_table_utils.c
View File

@ -306,7 +306,7 @@ ReplicateShardToNode(ShardInterval *shardInterval, char *nodeName, int nodePort)
if (targetPlacement == NULL || targetPlacement->shardState != FILE_FINALIZED)
{
uint64 placementId = 0;
uint32 groupId = 0;
int32 groupId = 0;
ereport(NOTICE, (errmsg("Replicating reference table \"%s\" to the node %s:%d",
get_rel_name(shardInterval->relationId), nodeName,
@ -410,7 +410,7 @@ CreateReferenceTableColocationId()
* group of reference tables. It is caller's responsibility to do that if it is necessary.
*/
void
DeleteAllReferenceTablePlacementsFromNodeGroup(uint32 groupId)
DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId)
{
List *referenceTableList = ReferenceTableOidList();
List *referenceShardIntervalList = NIL;

6
src/backend/distributed/utils/ruleutils_10.c
View File

@ -7134,6 +7134,12 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context)
/* Else print column aliases as needed */
get_column_alias_list(colinfo, context);
}
/* check if column's are given aliases in distributed tables */
else if (colinfo->parentUsing != NIL)
{
Assert(colinfo->printaliases);
get_column_alias_list(colinfo, context);
}
/* Tablesample clause must go after any alias */
if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) &&

6
src/backend/distributed/utils/ruleutils_11.c
View File

@ -7151,6 +7151,12 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context)
/* Else print column aliases as needed */
get_column_alias_list(colinfo, context);
}
/* check if column's are given aliases in distributed tables */
else if (colinfo->parentUsing != NIL)
{
Assert(colinfo->printaliases);
get_column_alias_list(colinfo, context);
}
/* Tablesample clause must go after any alias */
if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) &&

1
src/backend/distributed/worker/worker_data_fetch_protocol.c
View File

@ -563,6 +563,7 @@ TableDDLCommandList(const char *nodeName, uint32 nodePort, const char *tableName
ExecuteOptionalRemoteCommand(connection, queryString->data, &result);
ddlCommandList = ReadFirstColumnAsText(result);
PQclear(result);
ForgetResults(connection);
CloseConnection(connection);

1
src/include/distributed/backend_data.h
View File

@ -67,5 +67,6 @@ extern void GetBackendDataForProc(PGPROC *proc, BackendData *result);
extern void CancelTransactionDueToDeadlock(PGPROC *proc);
extern bool MyBackendGotCancelledDueToDeadlock(void);
extern List * ActiveDistributedTransactionNumbers(void);
LocalTransactionId GetMyProcLocalTransactionId(void);
#endif /* BACKEND_DATA_H */

1
src/include/distributed/distributed_planner.h
View File

@ -97,6 +97,7 @@ extern bool IsModifyCommand(Query *query);
extern bool IsUpdateOrDelete(struct DistributedPlan *distributedPlan);
extern bool IsModifyDistributedPlan(struct DistributedPlan *distributedPlan);
extern void EnsurePartitionTableNotReplicated(Oid relationId);
extern Node * ResolveExternalParams(Node *inputNode, ParamListInfo boundParams);
extern bool IsMultiTaskPlan(struct DistributedPlan *distributedPlan);
extern bool IsMultiShardModifyPlan(struct DistributedPlan *distributedPlan);
extern RangeTblEntry * RemoteScanRangeTableEntry(List *columnNameList);

10
src/include/distributed/master_metadata_utility.h
View File

@ -79,7 +79,7 @@ typedef struct GroupShardPlacement
uint64 shardId;
uint64 shardLength;
RelayFileState shardState;
uint32 groupId;
int32 groupId;
} GroupShardPlacement;
@ -94,7 +94,7 @@ typedef struct ShardPlacement
uint64 shardId;
uint64 shardLength;
RelayFileState shardState;
uint32 groupId;
int32 groupId;
/* the rest of the fields aren't from pg_dist_placement */
char *nodeName;
@ -122,13 +122,13 @@ extern void CopyShardInterval(ShardInterval *srcInterval, ShardInterval *destInt
extern void CopyShardPlacement(ShardPlacement *srcPlacement,
ShardPlacement *destPlacement);
extern uint64 ShardLength(uint64 shardId);
extern bool NodeGroupHasShardPlacements(uint32 groupId,
extern bool NodeGroupHasShardPlacements(int32 groupId,
bool onlyConsiderActivePlacements);
extern List * FinalizedShardPlacementList(uint64 shardId);
extern ShardPlacement * FinalizedShardPlacement(uint64 shardId, bool missingOk);
extern List * BuildShardPlacementList(ShardInterval *shardInterval);
extern List * AllShardPlacementsOnNodeGroup(int32 groupId);
extern List * GroupShardPlacementsForTableOnGroup(Oid relationId, uint32 groupId);
extern List * GroupShardPlacementsForTableOnGroup(Oid relationId, int32 groupId);
/* Function declarations to modify shard and shard placement data */
extern void InsertShardRow(Oid relationId, uint64 shardId, char storageType,
@ -136,7 +136,7 @@ extern void InsertShardRow(Oid relationId, uint64 shardId, char storageType,
extern void DeleteShardRow(uint64 shardId);
extern uint64 InsertShardPlacementRow(uint64 shardId, uint64 placementId,
char shardState, uint64 shardLength,
uint32 groupId);
int32 groupId);
extern void InsertIntoPgDistPartition(Oid relationId, char distributionMethod,
Var *distributionColumn, uint32 colocationId,
char replicationModel);

4
src/include/distributed/metadata_cache.h
View File

@ -93,11 +93,11 @@ extern List * DistributedTableList(void);
extern ShardInterval * LoadShardInterval(uint64 shardId);
extern Oid RelationIdForShard(uint64 shardId);
extern bool ReferenceTableShardId(uint64 shardId);
extern ShardPlacement * FindShardPlacementOnGroup(uint32 groupId, uint64 shardId);
extern ShardPlacement * FindShardPlacementOnGroup(int32 groupId, uint64 shardId);
extern GroupShardPlacement * LoadGroupShardPlacement(uint64 shardId, uint64 placementId);
extern ShardPlacement * LoadShardPlacement(uint64 shardId, uint64 placementId);
extern DistTableCacheEntry * DistributedTableCacheEntry(Oid distributedRelationId);
extern int GetLocalGroupId(void);
extern int32 GetLocalGroupId(void);
extern List * DistTableOidList(void);
extern Oid LookupShardRelation(int64 shardId, bool missing_ok);
extern List * ShardPlacementList(uint64 shardId);

2
src/include/distributed/metadata_sync.h
View File

@ -35,7 +35,7 @@ extern char * NodeStateUpdateCommand(uint32 nodeId, bool isActive);
extern char * ColocationIdUpdateCommand(Oid relationId, uint32 colocationId);
extern char * CreateSchemaDDLCommand(Oid schemaId);
extern char * PlacementUpsertCommand(uint64 shardId, uint64 placementId, int shardState,
uint64 shardLength, uint32 groupId);
uint64 shardLength, int32 groupId);
extern void CreateTableMetadataOnWorkers(Oid relationId);

1
src/include/distributed/multi_physical_planner.h
View File

@ -351,6 +351,7 @@ extern List * TaskListDifference(const List *list1, const List *list2);
extern List * AssignAnchorShardTaskList(List *taskList);
extern List * FirstReplicaAssignTaskList(List *taskList);
extern List * RoundRobinAssignTaskList(List *taskList);
extern List * RoundRobinPerTransactionAssignTaskList(List *taskList);
extern int CompareTasksByTaskId(const void *leftElement, const void *rightElement);
/* function declaration for creating Task */

4
src/include/distributed/multi_router_executor.h
View File

@ -53,7 +53,7 @@ extern int64 ExecuteModifyTasksSequentiallyWithoutResults(List *taskList,
/* helper functions */
extern bool TaskListRequires2PC(List *taskList);
extern List * BuildPlacementSelectList(uint32 groupId, List *relationShardList);
extern List * BuildPlacementDDLList(uint32 groupId, List *relationShardList);
extern List * BuildPlacementSelectList(int32 groupId, List *relationShardList);
extern List * BuildPlacementDDLList(int32 groupId, List *relationShardList);
#endif /* MULTI_ROUTER_EXECUTOR_H_ */

13
src/include/distributed/multi_router_planner.h
View File

@ -25,6 +25,7 @@
#define CITUS_TABLE_ALIAS "citus_table_alias"
extern bool EnableRouterExecution;
extern bool EnableFastPathRouterPlanner;
extern DistributedPlan * CreateRouterPlan(Query *originalQuery, Query *query,
PlannerRestrictionContext *
@ -42,8 +43,8 @@ extern DeferredErrorMessage * PlanRouterQuery(Query *originalQuery,
Const **partitionValueConst);
extern List * RouterInsertTaskList(Query *query, DeferredErrorMessage **planningError);
extern Const * ExtractInsertPartitionKeyValue(Query *query);
extern List * TargetShardIntervalsForQuery(Query *query,
RelationRestrictionContext *restrictionContext,
extern List * TargetShardIntervalsForRestrictInfo(RelationRestrictionContext *
restrictionContext,
bool *multiShardQuery,
Const **partitionValueConst);
extern List * WorkersContainingAllShards(List *prunedShardIntervalsList);
@ -68,5 +69,13 @@ extern void AddShardIntervalRestrictionToSelect(Query *subqery,
extern bool UpdateOrDeleteQuery(Query *query);
extern List * WorkersContainingAllShards(List *prunedShardIntervalsList);
/*
* FastPathPlanner is a subset of router planner, that's why we prefer to
* keep the external function here.
*/extern PlannedStmt * GeneratePlaceHolderPlannedStmt(Query *parse);
extern PlannedStmt * FastPathPlanner(Query *originalQuery, Query *parse, ParamListInfo
boundParams);
extern bool FastPathRouterQuery(Query *query);
#endif /* MULTI_ROUTER_PLANNER_H */

2
src/include/distributed/reference_table_utils.h
View File

@ -14,7 +14,7 @@
extern uint32 CreateReferenceTableColocationId(void);
extern void ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort);
extern void DeleteAllReferenceTablePlacementsFromNodeGroup(uint32 groupId);
extern void DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId);
extern List * ReferenceTableOidList(void);
extern int CompareOids(const void *leftElement, const void *rightElement);

2
src/include/distributed/remote_transaction.h
View File

@ -81,7 +81,7 @@ typedef struct RemoteTransaction
/* utility functions for dealing with remote transactions */
extern bool ParsePreparedTransactionName(char *preparedTransactionName, int *groupId,
extern bool ParsePreparedTransactionName(char *preparedTransactionName, int32 *groupId,
int *procId, uint64 *transactionNumber,
uint32 *connectionNumber);

2
src/include/distributed/transaction_recovery.h
View File

@ -17,7 +17,7 @@ extern int Recover2PCInterval;
/* Functions declarations for worker transactions */
extern void LogTransactionRecord(int groupId, char *transactionName);
extern void LogTransactionRecord(int32 groupId, char *transactionName);
extern int RecoverTwoPhaseCommits(void);

28
src/include/distributed/version_compat.h
View File

@ -23,6 +23,7 @@
#include "optimizer/prep.h"
#include "postmaster/bgworker.h"
#include "utils/memutils.h"
#include "funcapi.h"
/* PostgreSQL 11 splits hash procs into "standard" and "extended" */
#define HASHSTANDARD_PROC HASHPROC
@ -134,6 +135,33 @@ canonicalize_qual_compat(Expr *qual, bool is_check)
}
/*
* A convenient wrapper around get_expr_result_type() that is added on PG11
*
* Note that this function ignores the second parameter and behaves
* slightly differently than the PG11 version.
*
* 1. The original function throws errors if noError flag is not set, we ignore
* this flag here and return NULL in that case
* 2. TYPEFUNC_COMPOSITE_DOMAIN is introduced in PG11, and references to this
* macro is removed
* */
static inline TupleDesc
get_expr_result_tupdesc(Node *expr, bool noError)
{
TupleDesc tupleDesc;
TypeFuncClass functypclass;
functypclass = get_expr_result_type(expr, NULL, &tupleDesc);
if (functypclass == TYPEFUNC_COMPOSITE)
{
return tupleDesc;
}
return NULL;
}
#endif
#if (PG_VERSION_NUM >= 110000)

4
src/include/distributed/worker_manager.h
View File

@ -41,7 +41,7 @@ typedef struct WorkerNode
uint32 nodeId; /* node's unique id, key of the hash table */
uint32 workerPort; /* node's port */
char workerName[WORKER_LENGTH]; /* node's name */
uint32 groupId; /* node's groupId; same for the nodes that are in the same group */
int32 groupId; /* node's groupId; same for the nodes that are in the same group */
char workerRack[WORKER_LENGTH]; /* node's network location */
bool hasMetadata; /* node gets metadata changes */
bool isActive; /* node's state */
@ -72,7 +72,7 @@ extern WorkerNode * FindWorkerNodeAnyCluster(char *nodeName, int32 nodePort);
extern List * ReadWorkerNodes(bool includeNodesFromOtherClusters);
extern void EnsureCoordinator(void);
extern uint32 GroupForNode(char *nodeName, int32 nodePorT);
extern WorkerNode * PrimaryNodeForGroup(uint32 groupId, bool *groupContainsNodes);
extern WorkerNode * PrimaryNodeForGroup(int32 groupId, bool *groupContainsNodes);
extern bool WorkerNodeIsPrimary(WorkerNode *worker);
extern bool WorkerNodeIsSecondary(WorkerNode *worker);
extern bool WorkerNodeIsReadable(WorkerNode *worker);

2
src/test/regress/Makefile
View File

@ -15,7 +15,7 @@ endif
##
MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install
pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)"
MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags)
MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times"
# XXX: Can't actually do useful testruns against install - $libdir
# etc will point to the directory configured during postgres'

362
src/test/regress/expected/fast_path_router_modify.out Normal file
View File

@ -0,0 +1,362 @@
CREATE SCHEMA fast_path_router_modify;
SET search_path TO fast_path_router_modify;
SET citus.next_shard_id TO 1840000;
-- all the tests in this file is intended for testing fast-path
-- router planner, so we're explicitly enabling itin this file.
-- We've bunch of other tests that triggers non-fast-path-router
-- planner (note this is already true by default)
SET citus.enable_fast_path_router_planner TO true;
SET citus.shard_replication_factor TO 1;
CREATE TABLE modify_fast_path(key int, value_1 int, value_2 text);
SELECT create_distributed_table('modify_fast_path', 'key');
create_distributed_table
--------------------------
(1 row)
SET citus.shard_replication_factor TO 2;
CREATE TABLE modify_fast_path_replication_2(key int, value_1 int, value_2 text);
SELECT create_distributed_table('modify_fast_path_replication_2', 'key');
create_distributed_table
--------------------------
(1 row)
CREATE TABLE modify_fast_path_reference(key int, value_1 int, value_2 text);
SELECT create_reference_table('modify_fast_path_reference');
create_reference_table
------------------------
(1 row)
-- show the output
SET client_min_messages TO DEBUG;
-- very simple queries goes through fast-path planning
DELETE FROM modify_fast_path WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
UPDATE modify_fast_path SET value_1 = 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
UPDATE modify_fast_path SET value_1 = value_1 + 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
UPDATE modify_fast_path SET value_1 = value_1 + value_2::int WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 AND value_2 = 'citus');
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
DELETE FROM modify_fast_path WHERE key = 1 and FALSE;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
-- UPDATE may include complex target entries
UPDATE modify_fast_path SET value_1 = value_1 + 12 * value_1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
UPDATE modify_fast_path SET value_1 = abs(-19) WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
-- cannot go through fast-path because there are multiple keys
DELETE FROM modify_fast_path WHERE key = 1 AND key = 2;
DEBUG: Creating router plan
DEBUG: Plan is router executable
DELETE FROM modify_fast_path WHERE key = 1 AND (key = 2 AND value_1 = 15);
DEBUG: Creating router plan
DEBUG: Plan is router executable
-- cannot go through fast-path because key is not on the top level
DELETE FROM modify_fast_path WHERE value_1 = 15 OR (key = 1 AND value_2 = 'citus');
DEBUG: Creating router plan
DEBUG: Plan is router executable
DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 OR value_2 = 'citus');
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
-- goes through fast-path planning even if the key is updated to the same value
UPDATE modify_fast_path SET key = 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
UPDATE modify_fast_path SET key = 1::float WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
-- cannot support if key changes
UPDATE modify_fast_path SET key = 2 WHERE key = 1;
DEBUG: modifying the partition value of rows is not allowed
ERROR: modifying the partition value of rows is not allowed
UPDATE modify_fast_path SET key = 2::numeric WHERE key = 1;
DEBUG: modifying the partition value of rows is not allowed
ERROR: modifying the partition value of rows is not allowed
-- returning is not supported via fast-path
DELETE FROM modify_fast_path WHERE key = 1 RETURNING *;
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
key | value_1 | value_2
-----+---------+---------
(0 rows)
-- modifying ctes are not supported via fast-path
WITH t1 AS (DELETE FROM modify_fast_path WHERE key = 1), t2 AS (SELECT * FROM modify_fast_path) SELECT * FROM t2;
DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries
DEBUG: generating subplan 18_1 for CTE t1: DELETE FROM fast_path_router_modify.modify_fast_path WHERE (key OPERATOR(pg_catalog.=) 1)
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
DEBUG: generating subplan 18_2 for CTE t2: SELECT key, value_1, value_2 FROM fast_path_router_modify.modify_fast_path
DEBUG: Plan 18 query after replacing subqueries and CTEs: SELECT key, value_1, value_2 FROM (SELECT intermediate_result.key, intermediate_result.value_1, intermediate_result.value_2 FROM read_intermediate_result('18_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value_1 integer, value_2 text)) t2
DEBUG: Creating router plan
DEBUG: Plan is router executable
key | value_1 | value_2
-----+---------+---------
(0 rows)
-- for update/share is supported via fast-path when replication factor = 1 or reference table
SELECT * FROM modify_fast_path WHERE key = 1 FOR UPDATE;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
key | value_1 | value_2
-----+---------+---------
(0 rows)
SELECT * FROM modify_fast_path WHERE key = 1 FOR SHARE;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
key | value_1 | value_2
-----+---------+---------
(0 rows)
SELECT * FROM modify_fast_path_reference WHERE key = 1 FOR UPDATE;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
key | value_1 | value_2
-----+---------+---------
(0 rows)
SELECT * FROM modify_fast_path_reference WHERE key = 1 FOR SHARE;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
key | value_1 | value_2
-----+---------+---------
(0 rows)
-- for update/share is not supported via fast-path wen replication factor > 1
SELECT * FROM modify_fast_path_replication_2 WHERE key = 1 FOR UPDATE;
ERROR: could not run distributed query with FOR UPDATE/SHARE commands
HINT: Consider using an equality filter on the distributed table's partition column.
SELECT * FROM modify_fast_path_replication_2 WHERE key = 1 FOR SHARE;
ERROR: could not run distributed query with FOR UPDATE/SHARE commands
HINT: Consider using an equality filter on the distributed table's partition column.
-- very simple queries on reference tables goes through fast-path planning
DELETE FROM modify_fast_path_reference WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
UPDATE modify_fast_path_reference SET value_1 = 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
UPDATE modify_fast_path_reference SET value_1 = value_1 + 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
UPDATE modify_fast_path_reference SET value_1 = value_1 + value_2::int WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
-- joins are not supported via fast-path
UPDATE modify_fast_path
SET value_1 = 1
FROM modify_fast_path_reference
WHERE
modify_fast_path.key = modify_fast_path_reference.key AND
modify_fast_path.key = 1 AND
modify_fast_path_reference.key = 1;
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
PREPARE p1 (int, int, int) AS
UPDATE modify_fast_path SET value_1 = value_1 + $1 WHERE key = $2 AND value_1 = $3;
EXECUTE p1(1,1,1);
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
EXECUTE p1(2,2,2);
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 2
EXECUTE p1(3,3,3);
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 3
EXECUTE p1(4,4,4);
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 4
EXECUTE p1(5,5,5);
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 5
EXECUTE p1(6,6,6);
DEBUG: Router planner cannot handle multi-shard modify queries
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 6
CREATE FUNCTION modify_fast_path_plpsql(int, int) RETURNS void as $$
BEGIN
DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2;
END;
$$ LANGUAGE plpgsql;
SELECT modify_fast_path_plpsql(1,1);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(2,2);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 2
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(3,3);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 3
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(4,4);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 4
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(5,5);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 5
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(6,6);
DEBUG: Router planner cannot handle multi-shard modify queries
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 6
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
SELECT modify_fast_path_plpsql(6,6);
DEBUG: Distributed planning for a fast-path router query
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Creating router plan
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
DEBUG: Plan is router executable
DETAIL: distribution column value: 6
CONTEXT: SQL statement "DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2"
PL/pgSQL function modify_fast_path_plpsql(integer,integer) line 3 at SQL statement
modify_fast_path_plpsql
-------------------------
(1 row)
RESET client_min_messages;
DROP SCHEMA fast_path_router_modify CASCADE;
NOTICE: drop cascades to 4 other objects
DETAIL: drop cascades to table modify_fast_path
drop cascades to table modify_fast_path_replication_2
drop cascades to table modify_fast_path_reference
drop cascades to function modify_fast_path_plpsql(integer,integer)

250
src/test/regress/expected/full_join.out Normal file
View File

@ -0,0 +1,250 @@
--
-- Full join with subquery pushdown support
--
SET citus.next_shard_id TO 9000000;
CREATE SCHEMA full_join;
SET search_path TO full_join, public;
CREATE TABLE test_table_1(id int, val1 int);
CREATE TABLE test_table_2(id bigint, val1 int);
CREATE TABLE test_table_3(id int, val1 bigint);
SELECT create_distributed_table('test_table_1', 'id');
create_distributed_table
--------------------------
(1 row)
SELECT create_distributed_table('test_table_2', 'id');
create_distributed_table
--------------------------
(1 row)
SELECT create_distributed_table('test_table_3', 'id');
create_distributed_table
--------------------------
(1 row)
INSERT INTO test_table_1 VALUES(1,1),(2,2),(3,3);
INSERT INTO test_table_2 VALUES(2,2),(3,3),(4,4);
INSERT INTO test_table_3 VALUES(1,1),(3,3),(4,5);
-- Simple full outer join
SELECT id FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
id
----
1
2
3
4
(4 rows)
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
id | val1 | val1
----+------+------
1 | 1 | 1
2 | 2 |
3 | 3 | 3
4 | | 5
(4 rows)
-- Join subqueries using single column
SELECT * FROM
(SELECT test_table_1.id FROM test_table_1 FULL JOIN test_table_3 using(id)) as j1
FULL JOIN
(SELECT test_table_1.id FROM test_table_1 FULL JOIN test_table_3 using(id)) as j2
USING(id)
ORDER BY 1;
id
----
1
2
3
(5 rows)
-- Join subqueries using multiple columns
SELECT * FROM
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_3 using(id)) as j1
FULL JOIN
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_3 using(id)) as j2
USING(id, val1)
ORDER BY 1;
id | val1
----+------
1 | 1
2 | 2
3 | 3
|
|
(5 rows)
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_3 USING(id, val1) ORDER BY 1;
id | val1
----+------
1 | 1
2 | 2
3 | 3
4 | 5
(4 rows)
-- Full join with complicated target lists
SELECT count(DISTINCT id), (avg(test_table_1.val1) + id * id)::integer as avg_value, id::numeric IS NOT NULL as not_null
FROM test_table_1 FULL JOIN test_table_3 using(id)
WHERE id::bigint < 55
GROUP BY id
ORDER BY 2
ASC LIMIT 3;
count | avg_value | not_null
-------+-----------+----------
1 | 2 | t
1 | 6 | t
1 | 12 | t
(3 rows)
SELECT max(val1)
FROM test_table_1 FULL JOIN test_table_3 USING(id, val1)
GROUP BY test_table_1.id
ORDER BY 1;
max
-----
1
2
3
5
(4 rows)
-- Test the left join as well
SELECT max(val1)
FROM test_table_1 LEFT JOIN test_table_3 USING(id, val1)
GROUP BY test_table_1.id
ORDER BY 1;
max
-----
1
2
3
(3 rows)
-- Full outer join with different distribution column types, should error out
SELECT * FROM test_table_1 full join test_table_2 using(id);
ERROR: cannot push down this subquery
DETAIL: Shards of relations in subquery need to have 1-to-1 shard partitioning
-- Test when the non-distributed column has the value of NULL
INSERT INTO test_table_1 VALUES(7, NULL);
INSERT INTO test_table_2 VALUES(7, NULL);
INSERT INTO test_table_3 VALUES(7, NULL);
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
id | val1 | val1
----+------+------
1 | 1 | 1
2 | 2 |
3 | 3 | 3
4 | | 5
7 | |
(5 rows)
-- Get the same result (with multiple id)
SELECT * FROM test_table_1 FULL JOIN test_table_3 ON (test_table_1.id = test_table_3.id) ORDER BY 1;
id | val1 | id | val1
----+------+----+------
1 | 1 | 1 | 1
2 | 2 | |
3 | 3 | 3 | 3
7 | | 7 |
| | 4 | 5
(5 rows)
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_3 USING(id, val1) ORDER BY 1;
id | val1
----+------
1 | 1
2 | 2
3 | 3
4 | 5
7 |
7 |
(6 rows)
-- In order to make the same test with different data types use text-varchar pair
-- instead of using int-bigint pair.
DROP TABLE test_table_1;
DROP TABLE test_table_2;
DROP TABLE test_table_3;
CREATE TABLE test_table_1(id int, val1 text);
CREATE TABLE test_table_2(id int, val1 varchar(30));
SELECT create_distributed_table('test_table_1', 'id');
create_distributed_table
--------------------------
(1 row)
SELECT create_distributed_table('test_table_2', 'id');
create_distributed_table
--------------------------
(1 row)
INSERT INTO test_table_1 VALUES(1,'val_1'),(2,'val_2'),(3,'val_3'), (4, NULL);
INSERT INTO test_table_2 VALUES(2,'val_2'),(3,'val_3'),(4,'val_4'), (5, NULL);
-- Simple full outer join
SELECT id FROM test_table_1 FULL JOIN test_table_2 using(id) ORDER BY 1;
id
----
1
2
3
4
5
(5 rows)
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_2 using(id) ORDER BY 1;
id | val1 | val1
----+-------+-------
1 | val_1 |
2 | val_2 | val_2
3 | val_3 | val_3
4 | | val_4
5 | |
(5 rows)
-- Join subqueries using multiple columns
SELECT * FROM
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_2 using(id)) as j1
FULL JOIN
(SELECT test_table_2.id, test_table_2.val1 FROM test_table_1 FULL JOIN test_table_2 using(id)) as j2
USING(id, val1)
ORDER BY 1,2;
id | val1
----+-------
1 | val_1
2 | val_2
3 | val_3
4 | val_4
4 |
5 |
|
|
(8 rows)
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_2 USING(id, val1) ORDER BY 1,2;
id | val1
----+-------
1 | val_1
2 | val_2
3 | val_3
4 | val_4
4 |
5 |
(6 rows)
DROP SCHEMA full_join CASCADE;
NOTICE: drop cascades to 2 other objects
DETAIL: drop cascades to table test_table_1
drop cascades to table test_table_2

4
src/test/regress/expected/multi_function_evaluation.out
View File

@ -2,6 +2,10 @@
-- MULTI_FUNCTION_EVALUATION
--
SET citus.next_shard_id TO 1200000;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- nextval() works (no good way to test DEFAULT, or, by extension, SERIAL)
CREATE TABLE example (key INT, value INT);
SELECT master_create_distributed_table('example', 'key', 'hash');

229
src/test/regress/expected/multi_function_in_join.out Normal file
View File

@ -0,0 +1,229 @@
--
-- multi function in join queries aims to test the function calls that are
-- used in joins.
--
-- These functions are supposed to be executed on the worker and to ensure
-- that we wrap those functions inside (SELECT * FROM fnc()) sub queries.
--
-- We do not yet support those functions that:
-- - have lateral joins
-- - have WITH ORDINALITY clause
-- - are user-defined and immutable
CREATE SCHEMA functions_in_joins;
SET search_path TO 'functions_in_joins';
SET citus.next_shard_id TO 2500000;
CREATE TABLE table1 (id int, data int);
SELECT create_distributed_table('table1','id');
create_distributed_table
--------------------------
(1 row)
INSERT INTO table1
SELECT x, x*x
from generate_series(1, 100) as f (x);
-- Verbose messages for observing the subqueries that wrapped function calls
SET client_min_messages TO DEBUG1;
-- Check joins on a sequence
CREATE SEQUENCE numbers;
SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n) ORDER BY id ASC;
DEBUG: generating subplan 2_1 for subquery SELECT n FROM nextval('functions_in_joins.numbers'::regclass) n(n)
DEBUG: Plan 2 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.n FROM read_intermediate_result('2_1'::text, 'binary'::citus_copy_format) intermediate_result(n bigint)) n ON ((table1.id OPERATOR(pg_catalog.=) n.n))) ORDER BY table1.id
id | data | n
----+------+---
1 | 1 | 1
(1 row)
-- Check joins of a function that returns a single integer
CREATE FUNCTION add(integer, integer) RETURNS integer
AS 'SELECT $1 + $2;'
LANGUAGE SQL;
SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC;
DEBUG: generating subplan 3_1 for subquery SELECT sum FROM functions_in_joins.add(3, 5) sum(sum)
DEBUG: Plan 3 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, sum.sum FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.sum FROM read_intermediate_result('3_1'::text, 'binary'::citus_copy_format) intermediate_result(sum integer)) sum ON ((table1.id OPERATOR(pg_catalog.=) sum.sum))) ORDER BY table1.id
id | data | sum
----+------+-----
8 | 64 | 8
(1 row)
-- Check join of plpgsql functions
-- a function returning a single integer
CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$
BEGIN
RETURN i + 1;
END;
$$ LANGUAGE plpgsql;
SELECT * FROM table1 JOIN increment(2) val ON (id = val) ORDER BY id ASC;
DEBUG: generating subplan 4_1 for subquery SELECT val FROM functions_in_joins.increment(2) val(val)
DEBUG: Plan 4 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, val.val FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.val FROM read_intermediate_result('4_1'::text, 'binary'::citus_copy_format) intermediate_result(val integer)) val ON ((table1.id OPERATOR(pg_catalog.=) val.val))) ORDER BY table1.id
id | data | val
----+------+-----
3 | 9 | 3
(1 row)
-- a function that returns a set of integers
CREATE OR REPLACE FUNCTION next_k_integers(IN first_value INTEGER,
IN k INTEGER DEFAULT 3,
OUT result INTEGER)
RETURNS SETOF INTEGER AS $$
BEGIN
RETURN QUERY SELECT x FROM generate_series(first_value, first_value+k-1) f(x);
END;
$$ LANGUAGE plpgsql;
SELECT *
FROM table1 JOIN next_k_integers(3,2) next_integers ON (id = next_integers.result)
ORDER BY id ASC;
DEBUG: generating subplan 5_1 for subquery SELECT result FROM functions_in_joins.next_k_integers(3, 2) next_integers(result)
DEBUG: Plan 5 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, next_integers.result FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.result FROM read_intermediate_result('5_1'::text, 'binary'::citus_copy_format) intermediate_result(result integer)) next_integers ON ((table1.id OPERATOR(pg_catalog.=) next_integers.result))) ORDER BY table1.id
id | data | result
----+------+--------
3 | 9 | 3
4 | 16 | 4
(2 rows)
-- a function returning set of records
CREATE FUNCTION get_set_of_records() RETURNS SETOF RECORD AS $cmd$
SELECT x, x+1 FROM generate_series(0,4) f(x)
$cmd$
LANGUAGE SQL;
SELECT * FROM table1 JOIN get_set_of_records() AS t2(x int, y int) ON (id = x) ORDER BY id ASC;
DEBUG: generating subplan 6_1 for subquery SELECT x, y FROM functions_in_joins.get_set_of_records() t2(x integer, y integer)
DEBUG: Plan 6 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, t2.x, t2.y FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('6_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) t2 ON ((table1.id OPERATOR(pg_catalog.=) t2.x))) ORDER BY table1.id
id | data | x | y
----+------+---+---
1 | 1 | 1 | 2
2 | 4 | 2 | 3
3 | 9 | 3 | 4
4 | 16 | 4 | 5
(4 rows)
-- a function returning table
CREATE FUNCTION dup(int) RETURNS TABLE(f1 int, f2 text)
AS $$ SELECT $1, CAST($1 AS text) || ' is text' $$
LANGUAGE SQL;
SELECT f.* FROM table1 t JOIN dup(32) f ON (f1 = id);
DEBUG: generating subplan 7_1 for subquery SELECT f1, f2 FROM functions_in_joins.dup(32) f(f1, f2)
DEBUG: Plan 7 query after replacing subqueries and CTEs: SELECT f.f1, f.f2 FROM (functions_in_joins.table1 t JOIN (SELECT intermediate_result.f1, intermediate_result.f2 FROM read_intermediate_result('7_1'::text, 'binary'::citus_copy_format) intermediate_result(f1 integer, f2 text)) f ON ((f.f1 OPERATOR(pg_catalog.=) t.id)))
f1 | f2
----+------------
32 | 32 is text
(1 row)
-- a stable function
CREATE OR REPLACE FUNCTION the_minimum_id()
RETURNS INTEGER STABLE AS 'SELECT min(id) FROM table1' LANGUAGE SQL;
SELECT * FROM table1 JOIN the_minimum_id() min_id ON (id = min_id);
DEBUG: generating subplan 8_1 for subquery SELECT min_id FROM functions_in_joins.the_minimum_id() min_id(min_id)
DEBUG: Plan 8 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, min_id.min_id FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.min_id FROM read_intermediate_result('8_1'::text, 'binary'::citus_copy_format) intermediate_result(min_id integer)) min_id ON ((table1.id OPERATOR(pg_catalog.=) min_id.min_id)))
id | data | min_id
----+------+--------
1 | 1 | 1
(1 row)
-- a built-in immutable function
SELECT * FROM table1 JOIN abs(100) as hundred ON (id = hundred) ORDER BY id ASC;
id | data | hundred
-----+-------+---------
100 | 10000 | 100
(1 row)
-- function joins inside a CTE
WITH next_row_to_process AS (
SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n)
)
SELECT *
FROM table1, next_row_to_process
WHERE table1.data <= next_row_to_process.data
ORDER BY 1,2 ASC;
DEBUG: generating subplan 11_1 for CTE next_row_to_process: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN nextval('functions_in_joins.numbers'::regclass) n(n) ON ((table1.id OPERATOR(pg_catalog.=) n.n)))
DEBUG: generating subplan 12_1 for subquery SELECT n FROM nextval('functions_in_joins.numbers'::regclass) n(n)
DEBUG: Plan 12 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.n FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(n bigint)) n ON ((table1.id OPERATOR(pg_catalog.=) n.n)))
DEBUG: Plan 11 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, next_row_to_process.id, next_row_to_process.data, next_row_to_process.n FROM functions_in_joins.table1, (SELECT intermediate_result.id, intermediate_result.data, intermediate_result.n FROM read_intermediate_result('11_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, data integer, n bigint)) next_row_to_process WHERE (table1.data OPERATOR(pg_catalog.<=) next_row_to_process.data) ORDER BY table1.id, table1.data
id | data | id | data | n
----+------+----+------+---
1 | 1 | 2 | 4 | 2
2 | 4 | 2 | 4 | 2
(2 rows)
-- Multiple functions in an RTE
SELECT * FROM ROWS FROM (next_k_integers(5), next_k_integers(10)) AS f(a, b),
table1 WHERE id = a ORDER BY id ASC;
DEBUG: generating subplan 13_1 for subquery SELECT a, b FROM ROWS FROM(functions_in_joins.next_k_integers(5), functions_in_joins.next_k_integers(10)) f(a, b)
DEBUG: Plan 13 query after replacing subqueries and CTEs: SELECT f.a, f.b, table1.id, table1.data FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('13_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) f(a, b), functions_in_joins.table1 WHERE (table1.id OPERATOR(pg_catalog.=) f.a) ORDER BY table1.id
a | b | id | data
---+----+----+------
5 | 10 | 5 | 25
6 | 11 | 6 | 36
7 | 12 | 7 | 49
(3 rows)
-- Custom Type returning function used in a join
CREATE TYPE min_and_max AS (
minimum INT,
maximum INT
);
CREATE OR REPLACE FUNCTION max_and_min () RETURNS
min_and_max AS $$
DECLARE
result min_and_max%rowtype;
begin
select into result min(data) as minimum, max(data) as maximum from table1;
return result;
end;
$$ language plpgsql;
SELECT * FROM table1 JOIN max_and_min() m ON (m.maximum = data OR m.minimum = data);
DEBUG: generating subplan 14_1 for subquery SELECT minimum, maximum FROM functions_in_joins.max_and_min() m(minimum, maximum)
DEBUG: Plan 14 query after replacing subqueries and CTEs: SELECT table1.id, table1.data, m.minimum, m.maximum FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.minimum, intermediate_result.maximum FROM read_intermediate_result('14_1'::text, 'binary'::citus_copy_format) intermediate_result(minimum integer, maximum integer)) m ON (((m.maximum OPERATOR(pg_catalog.=) table1.data) OR (m.minimum OPERATOR(pg_catalog.=) table1.data))))
id | data | minimum | maximum
-----+-------+---------+---------
1 | 1 | 1 | 10000
100 | 10000 | 1 | 10000
(2 rows)
-- The following tests will fail as we do not support all joins on
-- all kinds of functions
SET client_min_messages TO ERROR;
-- function joins in CTE results can create lateral joins that are not supported
SELECT public.raise_failed_execution($cmd$
WITH one_row AS (
SELECT * FROM table1 WHERE id=52
)
SELECT table1.id, table1.data
FROM one_row, table1, next_k_integers(one_row.id, 5) next_five_ids
WHERE table1.id = next_five_ids;
$cmd$);
ERROR: Task failed to execute
CONTEXT: PL/pgSQL function public.raise_failed_execution(text) line 6 at RAISE
-- a user-defined immutable function
CREATE OR REPLACE FUNCTION the_answer_to_life()
RETURNS INTEGER IMMUTABLE AS 'SELECT 42' LANGUAGE SQL;
SELECT public.raise_failed_execution($cmd$
SELECT * FROM table1 JOIN the_answer_to_life() the_answer ON (id = the_answer)
$cmd$);
ERROR: Task failed to execute
CONTEXT: PL/pgSQL function public.raise_failed_execution(text) line 6 at RAISE
-- WITH ORDINALITY clause
SELECT public.raise_failed_execution($cmd$
SELECT *
FROM table1
JOIN next_k_integers(10,5) WITH ORDINALITY next_integers
ON (id = next_integers.result)
ORDER BY id ASC;
$cmd$);
ERROR: Task failed to execute
CONTEXT: PL/pgSQL function public.raise_failed_execution(text) line 6 at RAISE
RESET client_min_messages;
DROP SCHEMA functions_in_joins CASCADE;
NOTICE: drop cascades to 11 other objects
DETAIL: drop cascades to table table1
drop cascades to sequence numbers
drop cascades to function add(integer,integer)
drop cascades to function increment(integer)
drop cascades to function next_k_integers(integer,integer)
drop cascades to function get_set_of_records()
drop cascades to function dup(integer)
drop cascades to function the_minimum_id()
drop cascades to type min_and_max
drop cascades to function max_and_min()
drop cascades to function the_answer_to_life()
SET search_path TO DEFAULT;

4
src/test/regress/expected/multi_hash_pruning.out
View File

@ -5,6 +5,10 @@
SET citus.next_shard_id TO 630000;
SET citus.shard_count to 4;
SET citus.shard_replication_factor to 1;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- Create a table partitioned on integer column and update partition type to
-- hash. Then load data into this table and update shard min max values with
-- hashed ones. Hash value of 1, 2, 3 and 4 are consecutively -1905060026,

1
src/test/regress/expected/multi_insert_select.out
View File

@ -1179,6 +1179,7 @@ FROM
DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match
DETAIL: The target table's partition column should correspond to a partition column in the subquery.
DEBUG: Collecting INSERT ... SELECT results on coordinator
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
-- foo2 is recursively planned and INSERT...SELECT is done via coordinator

4
src/test/regress/expected/multi_mx_router_planner.out
View File

@ -64,6 +64,10 @@ DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 10
-- single-shard tests
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- test simple select for a single row
SELECT * FROM articles_hash_mx WHERE author_id = 10 AND id = 50;
DEBUG: Creating router plan

4
src/test/regress/expected/multi_prepare_plsql.out
View File

@ -4,6 +4,10 @@
-- Many of the queries are taken from other regression test files
-- and converted into both plain SQL and PL/pgsql functions, which
-- use prepared statements internally.
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
CREATE FUNCTION plpgsql_test_1() RETURNS TABLE(count bigint) AS $$
DECLARE
BEGIN

4
src/test/regress/expected/multi_prepare_sql.out
View File

@ -1,6 +1,10 @@
--
-- MULTI_PREPARE_SQL
--
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- Tests covering PREPARE statements. Many of the queries are
-- taken from other regression test files and converted into
-- prepared statements.

4
src/test/regress/expected/multi_router_planner.out
View File

@ -2,6 +2,10 @@ SET citus.next_shard_id TO 840000;
-- ===================================================================
-- test router planner functionality for single shard select queries
-- ===================================================================
-- all the tests in this file is intended for testing non-fast-path
-- router planner, so we're disabling it in this file. We've bunch of
-- other tests that triggers fast-path-router planner
SET citus.enable_fast_path_router_planner TO false;
CREATE TABLE articles_hash (
id bigint NOT NULL,
author_id bigint NOT NULL,

2204
src/test/regress/expected/multi_router_planner_fast_path.out Normal file
View File

File diff suppressed because it is too large Load Diff

52
src/test/regress/expected/multi_simple_queries.out
View File

@ -1,4 +1,8 @@
SET citus.next_shard_id TO 850000;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- ===================================================================
-- test end-to-end query functionality
-- ===================================================================
@ -658,4 +662,52 @@ DETAIL: distribution column value: 1
41 | 1 | aznavour | 11814
(5 rows)
-- test tablesample with fast path as well
SET citus.enable_fast_path_router_planner TO true;
SELECT * FROM articles TABLESAMPLE SYSTEM (0) WHERE author_id = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+-------+------------
(0 rows)
SELECT * FROM articles TABLESAMPLE BERNOULLI (0) WHERE author_id = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+-------+------------
(0 rows)
SELECT * FROM articles TABLESAMPLE SYSTEM (100) WHERE author_id = 1 ORDER BY id;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+--------------+------------
1 | 1 | arsenous | 9572
11 | 1 | alamo | 1347
21 | 1 | arcading | 5890
31 | 1 | athwartships | 7271
41 | 1 | aznavour | 11814
(5 rows)
SELECT * FROM articles TABLESAMPLE BERNOULLI (100) WHERE author_id = 1 ORDER BY id;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+--------------+------------
1 | 1 | arsenous | 9572
11 | 1 | alamo | 1347
21 | 1 | arcading | 5890
31 | 1 | athwartships | 7271
41 | 1 | aznavour | 11814
(5 rows)
SET client_min_messages to 'NOTICE';

52
src/test/regress/expected/multi_simple_queries_0.out
View File

@ -1,4 +1,8 @@
SET citus.next_shard_id TO 850000;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- ===================================================================
-- test end-to-end query functionality
-- ===================================================================
@ -602,4 +606,52 @@ DETAIL: distribution column value: 1
41 | 1 | aznavour | 11814
(5 rows)
-- test tablesample with fast path as well
SET citus.enable_fast_path_router_planner TO true;
SELECT * FROM articles TABLESAMPLE SYSTEM (0) WHERE author_id = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+-------+------------
(0 rows)
SELECT * FROM articles TABLESAMPLE BERNOULLI (0) WHERE author_id = 1;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+-------+------------
(0 rows)
SELECT * FROM articles TABLESAMPLE SYSTEM (100) WHERE author_id = 1 ORDER BY id;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+--------------+------------
1 | 1 | arsenous | 9572
11 | 1 | alamo | 1347
21 | 1 | arcading | 5890
31 | 1 | athwartships | 7271
41 | 1 | aznavour | 11814
(5 rows)
SELECT * FROM articles TABLESAMPLE BERNOULLI (100) WHERE author_id = 1 ORDER BY id;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
id | author_id | title | word_count
----+-----------+--------------+------------
1 | 1 | arsenous | 9572
11 | 1 | alamo | 1347
21 | 1 | arcading | 5890
31 | 1 | athwartships | 7271
41 | 1 | aznavour | 11814
(5 rows)
SET client_min_messages to 'NOTICE';

59
src/test/regress/expected/multi_subquery_complex_queries.out
View File

@ -2690,3 +2690,62 @@ ORDER BY 1;
5 |
(4 rows)
-- queries where column aliases are used
-- the query is not very complex. join is given an alias with aliases
-- for each output column
SELECT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
k1
----
1
1
1
1
1
(5 rows)
SELECT DISTINCT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
k1
----
1
2
3
4
5
(5 rows)
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e ON (u.user_id = e.user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;
x1 | x3 | value_2
----+----+---------
1 | 1 | 1
1 | 1 | 1
1 | 1 | 1
1 | 1 | 2
1 | 1 | 2
(5 rows)
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e USING (user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;
x1 | x3 | value_2
----+----+---------
1 | 1 | 1
1 | 1 | 1
1 | 1 | 1
1 | 1 | 2
1 | 1 | 2
(5 rows)

44
src/test/regress/expected/multi_subquery_complex_reference_clause.out
View File

@ -281,10 +281,8 @@ SET client_min_messages TO DEBUG;
SELECT count(*) FROM
(SELECT random() FROM user_buy_test_table JOIN random() AS users_ref_test_table(id)
ON user_buy_test_table.item_id > users_ref_test_table.id) subquery_1;
DEBUG: generating subplan 30_1 for subquery SELECT random() AS random FROM (public.user_buy_test_table JOIN random() users_ref_test_table(id) ON (((user_buy_test_table.item_id)::double precision OPERATOR(pg_catalog.>) users_ref_test_table.id)))
DEBUG: Plan 30 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.random FROM read_intermediate_result('30_1'::text, 'binary'::citus_copy_format) intermediate_result(random double precision)) subquery_1
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 30_1 for subquery SELECT id FROM random() users_ref_test_table(id)
DEBUG: Plan 30 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT random() AS random FROM (public.user_buy_test_table JOIN (SELECT intermediate_result.id FROM read_intermediate_result('30_1'::text, 'binary'::citus_copy_format) intermediate_result(id double precision)) users_ref_test_table(id) ON (((user_buy_test_table.item_id)::double precision OPERATOR(pg_catalog.>) users_ref_test_table.id)))) subquery_1
count
-------
4
@ -295,10 +293,8 @@ SELECT count(*) FROM
(SELECT item_id FROM user_buy_test_table JOIN generate_series(random()::int,10) AS users_ref_test_table(id)
ON user_buy_test_table.item_id > users_ref_test_table.id) subquery_1
WHERE item_id = 6;
DEBUG: generating subplan 32_1 for subquery SELECT user_buy_test_table.item_id FROM (public.user_buy_test_table JOIN generate_series((random())::integer, 10) users_ref_test_table(id) ON ((user_buy_test_table.item_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))
DEBUG: Plan 32 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.item_id FROM read_intermediate_result('32_1'::text, 'binary'::citus_copy_format) intermediate_result(item_id integer)) subquery_1 WHERE (item_id OPERATOR(pg_catalog.=) 6)
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 31_1 for subquery SELECT id FROM generate_series((random())::integer, 10) users_ref_test_table(id)
DEBUG: Plan 31 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT user_buy_test_table.item_id FROM (public.user_buy_test_table JOIN (SELECT intermediate_result.id FROM read_intermediate_result('31_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) users_ref_test_table(id) ON ((user_buy_test_table.item_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))) subquery_1 WHERE (item_id OPERATOR(pg_catalog.=) 6)
count
-------
0
@ -309,11 +305,11 @@ SELECT count(*) FROM
(SELECT user_id FROM user_buy_test_table
UNION ALL
SELECT id FROM generate_series(1,10) AS users_ref_test_table(id)) subquery_1;
DEBUG: generating subplan 34_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: generating subplan 32_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 34_2 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('34_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT users_ref_test_table.id FROM generate_series(1, 10) users_ref_test_table(id)
DEBUG: Plan 34 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('34_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) subquery_1
DEBUG: generating subplan 32_2 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('32_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT users_ref_test_table.id FROM generate_series(1, 10) users_ref_test_table(id)
DEBUG: Plan 32 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('32_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) subquery_1
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
@ -359,11 +355,11 @@ SELECT count(*) FROM
(SELECT user_id FROM user_buy_test_table
UNION ALL
SELECT id FROM (SELECT 5 AS id) users_ref_test_table) subquery_1;
DEBUG: generating subplan 41_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: generating subplan 39_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 41_2 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('41_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT users_ref_test_table.id FROM (SELECT 5 AS id) users_ref_test_table
DEBUG: Plan 41 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('41_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) subquery_1
DEBUG: generating subplan 39_2 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('39_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT users_ref_test_table.id FROM (SELECT 5 AS id) users_ref_test_table
DEBUG: Plan 39 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('39_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) subquery_1
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
@ -379,11 +375,11 @@ SELECT * FROM
UNION
SELECT user_id FROM user_buy_test_table) sub
ORDER BY 1 DESC;
DEBUG: generating subplan 44_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: generating subplan 42_1 for subquery SELECT user_id FROM public.user_buy_test_table
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 44_2 for subquery SELECT users_ref_test_table.id FROM public.users_ref_test_table UNION SELECT intermediate_result.user_id FROM read_intermediate_result('44_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)
DEBUG: Plan 44 query after replacing subqueries and CTEs: SELECT id FROM (SELECT intermediate_result.id FROM read_intermediate_result('44_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) sub ORDER BY id DESC
DEBUG: generating subplan 42_2 for subquery SELECT users_ref_test_table.id FROM public.users_ref_test_table UNION SELECT intermediate_result.user_id FROM read_intermediate_result('42_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)
DEBUG: Plan 42 query after replacing subqueries and CTEs: SELECT id FROM (SELECT intermediate_result.id FROM read_intermediate_result('42_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) sub ORDER BY id DESC
DEBUG: Creating router plan
DEBUG: Plan is router executable
id
@ -403,11 +399,11 @@ SELECT * FROM
UNION
SELECT user_id, random() * 0 FROM (SELECT user_id FROM user_buy_test_table) sub2) sub
ORDER BY 1 DESC;
DEBUG: generating subplan 47_1 for subquery SELECT user_id, (random() OPERATOR(pg_catalog.*) (0)::double precision) FROM (SELECT user_buy_test_table.user_id FROM public.user_buy_test_table) sub2
DEBUG: generating subplan 45_1 for subquery SELECT user_id, (random() OPERATOR(pg_catalog.*) (0)::double precision) FROM (SELECT user_buy_test_table.user_id FROM public.user_buy_test_table) sub2
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 47_2 for subquery SELECT sub1.id, (random() OPERATOR(pg_catalog.*) (0)::double precision) FROM (SELECT users_ref_test_table.id FROM public.users_ref_test_table) sub1 UNION SELECT intermediate_result.user_id, intermediate_result."?column?" FROM read_intermediate_result('47_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "?column?" double precision)
DEBUG: Plan 47 query after replacing subqueries and CTEs: SELECT id, "?column?" FROM (SELECT intermediate_result.id, intermediate_result."?column?" FROM read_intermediate_result('47_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer, "?column?" double precision)) sub ORDER BY id DESC
DEBUG: generating subplan 45_2 for subquery SELECT sub1.id, (random() OPERATOR(pg_catalog.*) (0)::double precision) FROM (SELECT users_ref_test_table.id FROM public.users_ref_test_table) sub1 UNION SELECT intermediate_result.user_id, intermediate_result."?column?" FROM read_intermediate_result('45_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "?column?" double precision)
DEBUG: Plan 45 query after replacing subqueries and CTEs: SELECT id, "?column?" FROM (SELECT intermediate_result.id, intermediate_result."?column?" FROM read_intermediate_result('45_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer, "?column?" double precision)) sub ORDER BY id DESC
DEBUG: Creating router plan
DEBUG: Plan is router executable
id | ?column?
@ -1334,8 +1330,8 @@ SELECT count(*) FROM
(SELECT user_buy_test_table.user_id, random() FROM user_buy_test_table LEFT JOIN users_ref_test_table
ON user_buy_test_table.user_id > users_ref_test_table.id) subquery_2
WHERE subquery_1.user_id != subquery_2.user_id ;
DEBUG: generating subplan 86_1 for subquery SELECT user_buy_test_table.user_id, random() AS random FROM (public.user_buy_test_table LEFT JOIN public.users_ref_test_table ON ((user_buy_test_table.user_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))
DEBUG: Plan 86 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT user_buy_test_table.user_id, random() AS random FROM (public.user_buy_test_table LEFT JOIN public.users_ref_test_table ON ((user_buy_test_table.item_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))) subquery_1, (SELECT intermediate_result.user_id, intermediate_result.random FROM read_intermediate_result('86_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, random double precision)) subquery_2 WHERE (subquery_1.user_id OPERATOR(pg_catalog.<>) subquery_2.user_id)
DEBUG: generating subplan 84_1 for subquery SELECT user_buy_test_table.user_id, random() AS random FROM (public.user_buy_test_table LEFT JOIN public.users_ref_test_table ON ((user_buy_test_table.user_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))
DEBUG: Plan 84 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT user_buy_test_table.user_id, random() AS random FROM (public.user_buy_test_table LEFT JOIN public.users_ref_test_table ON ((user_buy_test_table.item_id OPERATOR(pg_catalog.>) users_ref_test_table.id)))) subquery_1, (SELECT intermediate_result.user_id, intermediate_result.random FROM read_intermediate_result('84_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, random double precision)) subquery_2 WHERE (subquery_1.user_id OPERATOR(pg_catalog.<>) subquery_2.user_id)
count
-------
67
@ -1380,8 +1376,8 @@ count(*) AS cnt, "generated_group_field"
ORDER BY
cnt DESC, generated_group_field ASC
LIMIT 10;
DEBUG: generating subplan 88_1 for subquery SELECT user_id, value_2 AS generated_group_field FROM public.users_table users
DEBUG: Plan 88 query after replacing subqueries and CTEs: SELECT count(*) AS cnt, generated_group_field FROM (SELECT "eventQuery".user_id, random() AS random, "eventQuery".generated_group_field FROM (SELECT multi_group_wrapper_1."time", multi_group_wrapper_1.event_user_id, multi_group_wrapper_1.user_id, left_group_by_1.generated_group_field, random() AS random FROM ((SELECT temp_data_queries."time", temp_data_queries.event_user_id, user_filters_1.user_id FROM ((SELECT events."time", events.user_id AS event_user_id FROM public.events_table events WHERE (events.user_id OPERATOR(pg_catalog.>) 2)) temp_data_queries JOIN (SELECT users.user_id FROM public.users_reference_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 2) AND (users.value_2 OPERATOR(pg_catalog.=) 5))) user_filters_1 ON ((temp_data_queries.event_user_id OPERATOR(pg_catalog.<) user_filters_1.user_id)))) multi_group_wrapper_1 RIGHT JOIN (SELECT intermediate_result.user_id, intermediate_result.generated_group_field FROM read_intermediate_result('88_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, generated_group_field integer)) left_group_by_1 ON ((left_group_by_1.user_id OPERATOR(pg_catalog.>) multi_group_wrapper_1.event_user_id)))) "eventQuery") "pushedDownQuery" GROUP BY generated_group_field ORDER BY (count(*)) DESC, generated_group_field LIMIT 10
DEBUG: generating subplan 86_1 for subquery SELECT user_id, value_2 AS generated_group_field FROM public.users_table users
DEBUG: Plan 86 query after replacing subqueries and CTEs: SELECT count(*) AS cnt, generated_group_field FROM (SELECT "eventQuery".user_id, random() AS random, "eventQuery".generated_group_field FROM (SELECT multi_group_wrapper_1."time", multi_group_wrapper_1.event_user_id, multi_group_wrapper_1.user_id, left_group_by_1.generated_group_field, random() AS random FROM ((SELECT temp_data_queries."time", temp_data_queries.event_user_id, user_filters_1.user_id FROM ((SELECT events."time", events.user_id AS event_user_id FROM public.events_table events WHERE (events.user_id OPERATOR(pg_catalog.>) 2)) temp_data_queries JOIN (SELECT users.user_id FROM public.users_reference_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 2) AND (users.value_2 OPERATOR(pg_catalog.=) 5))) user_filters_1 ON ((temp_data_queries.event_user_id OPERATOR(pg_catalog.<) user_filters_1.user_id)))) multi_group_wrapper_1 RIGHT JOIN (SELECT intermediate_result.user_id, intermediate_result.generated_group_field FROM read_intermediate_result('86_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, generated_group_field integer)) left_group_by_1 ON ((left_group_by_1.user_id OPERATOR(pg_catalog.>) multi_group_wrapper_1.event_user_id)))) "eventQuery") "pushedDownQuery" GROUP BY generated_group_field ORDER BY (count(*)) DESC, generated_group_field LIMIT 10
ERROR: cannot pushdown the subquery
DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join
RESET client_min_messages;

185
src/test/regress/expected/multi_task_assignment_policy.out
View File

@ -10,6 +10,27 @@ SELECT substring(:'server_version', '\d+')::int > 9 AS version_above_nine;
t
(1 row)
-- the function simply parses the results and returns 'shardId@worker'
-- for all the explain task outputs
CREATE OR REPLACE FUNCTION parse_explain_output(in qry text, in table_name text, out r text)
RETURNS SETOF TEXT AS $$
DECLARE
portOfTheTask text;
shardOfTheTask text;
begin
for r in execute qry loop
IF r LIKE '%port%' THEN
portOfTheTask = substring(r, '([0-9]{1,10})');
END IF;
IF r LIKE '%' || table_name || '%' THEN
shardOfTheTask = substring(r, '([0-9]{1,10})');
return QUERY SELECT shardOfTheTask || '@' || portOfTheTask ;
END IF;
end loop;
return;
end; $$ language plpgsql;
SET citus.explain_distributed_queries TO off;
-- Check that our policies for assigning tasks to worker nodes run as expected.
-- To test this, we first create a shell table, and then manually insert shard
@ -102,48 +123,7 @@ DEBUG: assigned task 1 to node localhost:57638
explain statements for distributed queries are not enabled
(3 rows)
-- Finally test the round-robin task assignment policy
SET citus.task_assignment_policy TO 'round-robin';
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
DEBUG: assigned task 3 to node localhost:57638
DEBUG: assigned task 2 to node localhost:57638
DEBUG: assigned task 1 to node localhost:57637
QUERY PLAN
-----------------------------------------------------------------------
Aggregate (cost=0.00..0.00 rows=0 width=0)
-> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0)
explain statements for distributed queries are not enabled
(3 rows)
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
DEBUG: assigned task 3 to node localhost:57637
DEBUG: assigned task 2 to node localhost:57637
DEBUG: assigned task 1 to node localhost:57638
QUERY PLAN
-----------------------------------------------------------------------
Aggregate (cost=0.00..0.00 rows=0 width=0)
-> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0)
explain statements for distributed queries are not enabled
(3 rows)
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
DEBUG: assigned task 3 to node localhost:57638
DEBUG: assigned task 2 to node localhost:57638
DEBUG: assigned task 1 to node localhost:57637
QUERY PLAN
-----------------------------------------------------------------------
Aggregate (cost=0.00..0.00 rows=0 width=0)
-> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0)
explain statements for distributed queries are not enabled
(3 rows)
RESET citus.task_assignment_policy;
RESET client_min_messages;
COMMIT;
BEGIN;
SET LOCAL client_min_messages TO DEBUG3;
SET LOCAL citus.explain_distributed_queries TO off;
-- Check how task_assignment_policy impact planning decisions for reference tables
CREATE TABLE task_assignment_reference_table (test_id integer);
SELECT create_reference_table('task_assignment_reference_table');
create_reference_table
@ -151,8 +131,13 @@ SELECT create_reference_table('task_assignment_reference_table');
(1 row)
BEGIN;
SET LOCAL client_min_messages TO DEBUG3;
SET LOCAL citus.explain_distributed_queries TO off;
-- Check how task_assignment_policy impact planning decisions for reference tables
SET LOCAL citus.task_assignment_policy TO 'greedy';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
@ -162,6 +147,7 @@ DEBUG: Plan is router executable
(2 rows)
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
@ -172,6 +158,7 @@ DEBUG: Plan is router executable
SET LOCAL citus.task_assignment_policy TO 'first-replica';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
@ -181,28 +168,7 @@ DEBUG: Plan is router executable
(2 rows)
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
--------------------------------------------------------------
Custom Scan (Citus Router)
explain statements for distributed queries are not enabled
(2 rows)
-- here we expect debug output showing two different hosts for subsequent queries
SET LOCAL citus.task_assignment_policy TO 'round-robin';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: assigned task 0 to node localhost:57637
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
--------------------------------------------------------------
Custom Scan (Citus Router)
explain statements for distributed queries are not enabled
(2 rows)
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: assigned task 0 to node localhost:57638
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
QUERY PLAN
@ -212,6 +178,98 @@ DEBUG: Plan is router executable
(2 rows)
ROLLBACK;
RESET client_min_messages;
-- Now, lets test round-robin policy
-- round-robin policy relies on PostgreSQL's local transactionId,
-- which might change and we don't have any control over it.
-- the important thing that we look for is that round-robin policy
-- should give the same output for executions in the same transaction
-- and different output for executions that are not insdie the
-- same transaction. To ensure that, we define a helper function
BEGIN;
SET LOCAL citus.explain_distributed_queries TO on;
CREATE TEMPORARY TABLE explain_outputs (value text);
SET LOCAL citus.task_assignment_policy TO 'round-robin';
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
-- given that we're in the same transaction, the count should be 1
SELECT count(DISTINCT value) FROM explain_outputs;
count
-------
1
(1 row)
DROP TABLE explain_outputs;
COMMIT;
-- now test round-robin policy outside
-- a transaction, we should see the assignements
-- change on every execution
CREATE TEMPORARY TABLE explain_outputs (value text);
SET citus.task_assignment_policy TO 'round-robin';
SET citus.explain_distributed_queries TO ON;
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
-- given that we're in the same transaction, the count should be 2
-- since there are two different worker nodes
SELECT count(DISTINCT value) FROM explain_outputs;
count
-------
2
(1 row)
TRUNCATE explain_outputs;
-- same test with a distributed table
-- we keep this test because as of this commit, the code
-- paths for reference tables and distributed tables are
-- not the same
SET citus.shard_replication_factor TO 2;
CREATE TABLE task_assignment_replicated_hash (test_id integer);
SELECT create_distributed_table('task_assignment_replicated_hash', 'test_id');
create_distributed_table
--------------------------
(1 row)
BEGIN;
SET LOCAL citus.explain_distributed_queries TO on;
SET LOCAL citus.task_assignment_policy TO 'round-robin';
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
-- given that we're in the same transaction, the count should be 1
SELECT count(DISTINCT value) FROM explain_outputs;
count
-------
1
(1 row)
DROP TABLE explain_outputs;
COMMIT;
-- now test round-robin policy outside
-- a transaction, we should see the assignements
-- change on every execution
CREATE TEMPORARY TABLE explain_outputs (value text);
SET citus.task_assignment_policy TO 'round-robin';
SET citus.explain_distributed_queries TO ON;
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
-- given that we're in the same transaction, the count should be 2
-- since there are two different worker nodes
SELECT count(DISTINCT value) FROM explain_outputs;
count
-------
2
(1 row)
RESET citus.task_assignment_policy;
RESET client_min_messages;
-- we should be able to use round-robin with router queries that
-- only contains intermediate results
BEGIN;
@ -234,3 +292,4 @@ WITH q1 AS (SELECT * FROM task_assignment_test_table_2) SELECT * FROM q1;
(0 rows)
ROLLBACK;
DROP TABLE task_assignment_replicated_hash, task_assignment_reference_table;

2
src/test/regress/expected/set_operation_and_local_tables.out
View File

@ -139,6 +139,7 @@ FROM
WHERE test.y = foo.x;
DEBUG: generating subplan 19_1 for CTE cte_1: SELECT x FROM recursive_set_local.test
DEBUG: generating subplan 19_2 for CTE cte_1: SELECT a FROM recursive_set_local.ref
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 19_3 for subquery SELECT x FROM recursive_set_local.local_test
@ -165,6 +166,7 @@ FROM
WHERE ref.a = foo.x;
DEBUG: generating subplan 23_1 for CTE cte_1: SELECT x FROM recursive_set_local.test
DEBUG: generating subplan 23_2 for CTE cte_1: SELECT a FROM recursive_set_local.ref
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 23_3 for subquery SELECT x FROM recursive_set_local.local_test

2
src/test/regress/expected/set_operations.out
View File

@ -509,7 +509,7 @@ DEBUG: generating subplan 100_2 for subquery SELECT x, y FROM recursive_union.t
DEBUG: Creating router plan
DEBUG: Plan is router executable
DEBUG: generating subplan 100_3 for subquery SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('100_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer) EXCEPT SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('100_2'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)
DEBUG: Plan 100 query after replacing subqueries and CTEs: SELECT u.x, u.y FROM ((SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('100_3'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) u JOIN generate_series(1, 10) x(x) USING (x)) ORDER BY u.x, u.y
DEBUG: Plan 100 query after replacing subqueries and CTEs: SELECT u.x, u.y FROM ((SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('100_3'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) u JOIN (SELECT x_1.x FROM generate_series(1, 10) x_1(x)) x USING (x)) ORDER BY u.x, u.y
DEBUG: Creating router plan
DEBUG: Plan is router executable
x | y

4
src/test/regress/log_test_times Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
export TIMEFORMAT="${PG_MAJOR}/${PGAPPNAME} %6R"
{ { time "$@" 1>&3- 2>&4-; } 2>> test_times.log; } 3>&1 4>&2

8
src/test/regress/multi_schedule
View File

@ -64,8 +64,8 @@ test: multi_deparse_shard_query multi_distributed_transaction_id multi_real_time
test: multi_explain
test: multi_basic_queries multi_complex_expressions multi_subquery multi_subquery_complex_queries multi_subquery_behavioral_analytics
test: multi_subquery_complex_reference_clause multi_subquery_window_functions multi_view multi_sql_function multi_prepare_sql
test: sql_procedure
test: multi_subquery_in_where_reference_clause
test: sql_procedure multi_function_in_join
test: multi_subquery_in_where_reference_clause full_join
test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc
test: multi_agg_distinct multi_agg_approximate_distinct multi_limit_clause_approximate multi_outer_join_reference multi_single_relation_subquery multi_prepare_plsql
test: multi_reference_table multi_select_for_update relation_access_tracking
@ -184,8 +184,8 @@ test: multi_transaction_recovery
# multi_copy creates hash and range-partitioned tables and performs COPY
# multi_router_planner creates hash partitioned tables.
# ---------
test: multi_copy
test: multi_router_planner
test: multi_copy fast_path_router_modify
test: multi_router_planner multi_router_planner_fast_path
# ----------
# multi_large_shardid loads more lineitem data using high shard identifiers

23
src/test/regress/pg_regress_multi.pl
View File

@ -16,6 +16,7 @@ use warnings;
use Fcntl;
use Getopt::Long;
use File::Basename;
use File::Spec::Functions;
use File::Path qw(make_path remove_tree);
use Config;
@ -151,6 +152,11 @@ else
$plainRegress = "$pgxsdir/src/test/regress/pg_regress";
$isolationRegress = "${postgresBuilddir}/src/test/isolation/pg_isolation_regress";
$pgConfig = "$bindir/pg_config";
if (-x "$pgxsdir/src/test/isolation/pg_isolation_regress")
{
$isolationRegress = "$pgxsdir/src/test/isolation/pg_isolation_regress";
}
}
if ($isolationtester && ! -f "$isolationRegress")
@ -171,7 +177,9 @@ MESSAGE
}
my $vanillaRegress = catfile("${postgresBuilddir}", "src", "test", "regress", "pg_regress");
if ($vanillatest && ! -f "$vanillaRegress")
my $vanillaSchedule = catfile(dirname("${pgxsdir}"), "regress", "parallel_schedule");
if ($vanillatest && ! (-f "$vanillaRegress" or -f "$vanillaSchedule"))
{
die <<"MESSAGE";
@ -796,8 +804,21 @@ if ($vanillatest)
$ENV{PGPORT} = $masterPort;
$ENV{PGUSER} = $user;
if (-f "$vanillaSchedule")
{
rmdir "./testtablespace";
mkdir "./testtablespace";
my $pgregressdir=catfile(dirname("$pgxsdir"), "regress");
system("$plainRegress", ("--inputdir", $pgregressdir),
("--schedule", catfile("$pgregressdir", "parallel_schedule"))) == 0
or die "Could not run vanilla tests";
}
else
{
system("make", ("-C", catfile("$postgresBuilddir", "src", "test", "regress"), "installcheck-parallel")) == 0
or die "Could not run vanilla tests";
}
}
elsif ($isolationtester)
{

113
src/test/regress/sql/fast_path_router_modify.sql Normal file
View File

@ -0,0 +1,113 @@
CREATE SCHEMA fast_path_router_modify;
SET search_path TO fast_path_router_modify;
SET citus.next_shard_id TO 1840000;
-- all the tests in this file is intended for testing fast-path
-- router planner, so we're explicitly enabling itin this file.
-- We've bunch of other tests that triggers non-fast-path-router
-- planner (note this is already true by default)
SET citus.enable_fast_path_router_planner TO true;
SET citus.shard_replication_factor TO 1;
CREATE TABLE modify_fast_path(key int, value_1 int, value_2 text);
SELECT create_distributed_table('modify_fast_path', 'key');
SET citus.shard_replication_factor TO 2;
CREATE TABLE modify_fast_path_replication_2(key int, value_1 int, value_2 text);
SELECT create_distributed_table('modify_fast_path_replication_2', 'key');
CREATE TABLE modify_fast_path_reference(key int, value_1 int, value_2 text);
SELECT create_reference_table('modify_fast_path_reference');
-- show the output
SET client_min_messages TO DEBUG;
-- very simple queries goes through fast-path planning
DELETE FROM modify_fast_path WHERE key = 1;
UPDATE modify_fast_path SET value_1 = 1 WHERE key = 1;
UPDATE modify_fast_path SET value_1 = value_1 + 1 WHERE key = 1;
UPDATE modify_fast_path SET value_1 = value_1 + value_2::int WHERE key = 1;
DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 AND value_2 = 'citus');
DELETE FROM modify_fast_path WHERE key = 1 and FALSE;
-- UPDATE may include complex target entries
UPDATE modify_fast_path SET value_1 = value_1 + 12 * value_1 WHERE key = 1;
UPDATE modify_fast_path SET value_1 = abs(-19) WHERE key = 1;
-- cannot go through fast-path because there are multiple keys
DELETE FROM modify_fast_path WHERE key = 1 AND key = 2;
DELETE FROM modify_fast_path WHERE key = 1 AND (key = 2 AND value_1 = 15);
-- cannot go through fast-path because key is not on the top level
DELETE FROM modify_fast_path WHERE value_1 = 15 OR (key = 1 AND value_2 = 'citus');
DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 OR value_2 = 'citus');
-- goes through fast-path planning even if the key is updated to the same value
UPDATE modify_fast_path SET key = 1 WHERE key = 1;
UPDATE modify_fast_path SET key = 1::float WHERE key = 1;
-- cannot support if key changes
UPDATE modify_fast_path SET key = 2 WHERE key = 1;
UPDATE modify_fast_path SET key = 2::numeric WHERE key = 1;
-- returning is not supported via fast-path
DELETE FROM modify_fast_path WHERE key = 1 RETURNING *;
-- modifying ctes are not supported via fast-path
WITH t1 AS (DELETE FROM modify_fast_path WHERE key = 1), t2 AS (SELECT * FROM modify_fast_path) SELECT * FROM t2;
-- for update/share is supported via fast-path when replication factor = 1 or reference table
SELECT * FROM modify_fast_path WHERE key = 1 FOR UPDATE;
SELECT * FROM modify_fast_path WHERE key = 1 FOR SHARE;
SELECT * FROM modify_fast_path_reference WHERE key = 1 FOR UPDATE;
SELECT * FROM modify_fast_path_reference WHERE key = 1 FOR SHARE;
-- for update/share is not supported via fast-path wen replication factor > 1
SELECT * FROM modify_fast_path_replication_2 WHERE key = 1 FOR UPDATE;
SELECT * FROM modify_fast_path_replication_2 WHERE key = 1 FOR SHARE;
-- very simple queries on reference tables goes through fast-path planning
DELETE FROM modify_fast_path_reference WHERE key = 1;
UPDATE modify_fast_path_reference SET value_1 = 1 WHERE key = 1;
UPDATE modify_fast_path_reference SET value_1 = value_1 + 1 WHERE key = 1;
UPDATE modify_fast_path_reference SET value_1 = value_1 + value_2::int WHERE key = 1;
-- joins are not supported via fast-path
UPDATE modify_fast_path
SET value_1 = 1
FROM modify_fast_path_reference
WHERE
modify_fast_path.key = modify_fast_path_reference.key AND
modify_fast_path.key = 1 AND
modify_fast_path_reference.key = 1;
PREPARE p1 (int, int, int) AS
UPDATE modify_fast_path SET value_1 = value_1 + $1 WHERE key = $2 AND value_1 = $3;
EXECUTE p1(1,1,1);
EXECUTE p1(2,2,2);
EXECUTE p1(3,3,3);
EXECUTE p1(4,4,4);
EXECUTE p1(5,5,5);
EXECUTE p1(6,6,6);
CREATE FUNCTION modify_fast_path_plpsql(int, int) RETURNS void as $$
BEGIN
DELETE FROM modify_fast_path WHERE key = $1 AND value_1 = $2;
END;
$$ LANGUAGE plpgsql;
SELECT modify_fast_path_plpsql(1,1);
SELECT modify_fast_path_plpsql(2,2);
SELECT modify_fast_path_plpsql(3,3);
SELECT modify_fast_path_plpsql(4,4);
SELECT modify_fast_path_plpsql(5,5);
SELECT modify_fast_path_plpsql(6,6);
SELECT modify_fast_path_plpsql(6,6);
RESET client_min_messages;
DROP SCHEMA fast_path_router_modify CASCADE;

115
src/test/regress/sql/full_join.sql Normal file
View File

@ -0,0 +1,115 @@
--
-- Full join with subquery pushdown support
--
SET citus.next_shard_id TO 9000000;
CREATE SCHEMA full_join;
SET search_path TO full_join, public;
CREATE TABLE test_table_1(id int, val1 int);
CREATE TABLE test_table_2(id bigint, val1 int);
CREATE TABLE test_table_3(id int, val1 bigint);
SELECT create_distributed_table('test_table_1', 'id');
SELECT create_distributed_table('test_table_2', 'id');
SELECT create_distributed_table('test_table_3', 'id');
INSERT INTO test_table_1 VALUES(1,1),(2,2),(3,3);
INSERT INTO test_table_2 VALUES(2,2),(3,3),(4,4);
INSERT INTO test_table_3 VALUES(1,1),(3,3),(4,5);
-- Simple full outer join
SELECT id FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
-- Join subqueries using single column
SELECT * FROM
(SELECT test_table_1.id FROM test_table_1 FULL JOIN test_table_3 using(id)) as j1
FULL JOIN
(SELECT test_table_1.id FROM test_table_1 FULL JOIN test_table_3 using(id)) as j2
USING(id)
ORDER BY 1;
-- Join subqueries using multiple columns
SELECT * FROM
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_3 using(id)) as j1
FULL JOIN
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_3 using(id)) as j2
USING(id, val1)
ORDER BY 1;
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_3 USING(id, val1) ORDER BY 1;
-- Full join with complicated target lists
SELECT count(DISTINCT id), (avg(test_table_1.val1) + id * id)::integer as avg_value, id::numeric IS NOT NULL as not_null
FROM test_table_1 FULL JOIN test_table_3 using(id)
WHERE id::bigint < 55
GROUP BY id
ORDER BY 2
ASC LIMIT 3;
SELECT max(val1)
FROM test_table_1 FULL JOIN test_table_3 USING(id, val1)
GROUP BY test_table_1.id
ORDER BY 1;
-- Test the left join as well
SELECT max(val1)
FROM test_table_1 LEFT JOIN test_table_3 USING(id, val1)
GROUP BY test_table_1.id
ORDER BY 1;
-- Full outer join with different distribution column types, should error out
SELECT * FROM test_table_1 full join test_table_2 using(id);
-- Test when the non-distributed column has the value of NULL
INSERT INTO test_table_1 VALUES(7, NULL);
INSERT INTO test_table_2 VALUES(7, NULL);
INSERT INTO test_table_3 VALUES(7, NULL);
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_3 using(id) ORDER BY 1;
-- Get the same result (with multiple id)
SELECT * FROM test_table_1 FULL JOIN test_table_3 ON (test_table_1.id = test_table_3.id) ORDER BY 1;
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_3 USING(id, val1) ORDER BY 1;
-- In order to make the same test with different data types use text-varchar pair
-- instead of using int-bigint pair.
DROP TABLE test_table_1;
DROP TABLE test_table_2;
DROP TABLE test_table_3;
CREATE TABLE test_table_1(id int, val1 text);
CREATE TABLE test_table_2(id int, val1 varchar(30));
SELECT create_distributed_table('test_table_1', 'id');
SELECT create_distributed_table('test_table_2', 'id');
INSERT INTO test_table_1 VALUES(1,'val_1'),(2,'val_2'),(3,'val_3'), (4, NULL);
INSERT INTO test_table_2 VALUES(2,'val_2'),(3,'val_3'),(4,'val_4'), (5, NULL);
-- Simple full outer join
SELECT id FROM test_table_1 FULL JOIN test_table_2 using(id) ORDER BY 1;
-- Get all columns as the result of the full join
SELECT * FROM test_table_1 FULL JOIN test_table_2 using(id) ORDER BY 1;
-- Join subqueries using multiple columns
SELECT * FROM
(SELECT test_table_1.id, test_table_1.val1 FROM test_table_1 FULL JOIN test_table_2 using(id)) as j1
FULL JOIN
(SELECT test_table_2.id, test_table_2.val1 FROM test_table_1 FULL JOIN test_table_2 using(id)) as j2
USING(id, val1)
ORDER BY 1,2;
-- Full join using multiple columns
SELECT * FROM test_table_1 FULL JOIN test_table_2 USING(id, val1) ORDER BY 1,2;
DROP SCHEMA full_join CASCADE;

5
src/test/regress/sql/multi_function_evaluation.sql
View File

@ -4,6 +4,11 @@
SET citus.next_shard_id TO 1200000;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- nextval() works (no good way to test DEFAULT, or, by extension, SERIAL)
CREATE TABLE example (key INT, value INT);

146
src/test/regress/sql/multi_function_in_join.sql Normal file
View File

@ -0,0 +1,146 @@
--
-- multi function in join queries aims to test the function calls that are
-- used in joins.
--
-- These functions are supposed to be executed on the worker and to ensure
-- that we wrap those functions inside (SELECT * FROM fnc()) sub queries.
--
-- We do not yet support those functions that:
-- - have lateral joins
-- - have WITH ORDINALITY clause
-- - are user-defined and immutable
CREATE SCHEMA functions_in_joins;
SET search_path TO 'functions_in_joins';
SET citus.next_shard_id TO 2500000;
CREATE TABLE table1 (id int, data int);
SELECT create_distributed_table('table1','id');
INSERT INTO table1
SELECT x, x*x
from generate_series(1, 100) as f (x);
-- Verbose messages for observing the subqueries that wrapped function calls
SET client_min_messages TO DEBUG1;
-- Check joins on a sequence
CREATE SEQUENCE numbers;
SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n) ORDER BY id ASC;
-- Check joins of a function that returns a single integer
CREATE FUNCTION add(integer, integer) RETURNS integer
AS 'SELECT $1 + $2;'
LANGUAGE SQL;
SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC;
-- Check join of plpgsql functions
-- a function returning a single integer
CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$
BEGIN
RETURN i + 1;
END;
$$ LANGUAGE plpgsql;
SELECT * FROM table1 JOIN increment(2) val ON (id = val) ORDER BY id ASC;
-- a function that returns a set of integers
CREATE OR REPLACE FUNCTION next_k_integers(IN first_value INTEGER,
IN k INTEGER DEFAULT 3,
OUT result INTEGER)
RETURNS SETOF INTEGER AS $$
BEGIN
RETURN QUERY SELECT x FROM generate_series(first_value, first_value+k-1) f(x);
END;
$$ LANGUAGE plpgsql;
SELECT *
FROM table1 JOIN next_k_integers(3,2) next_integers ON (id = next_integers.result)
ORDER BY id ASC;
-- a function returning set of records
CREATE FUNCTION get_set_of_records() RETURNS SETOF RECORD AS $cmd$
SELECT x, x+1 FROM generate_series(0,4) f(x)
$cmd$
LANGUAGE SQL;
SELECT * FROM table1 JOIN get_set_of_records() AS t2(x int, y int) ON (id = x) ORDER BY id ASC;
-- a function returning table
CREATE FUNCTION dup(int) RETURNS TABLE(f1 int, f2 text)
AS $$ SELECT $1, CAST($1 AS text) || ' is text' $$
LANGUAGE SQL;
SELECT f.* FROM table1 t JOIN dup(32) f ON (f1 = id);
-- a stable function
CREATE OR REPLACE FUNCTION the_minimum_id()
RETURNS INTEGER STABLE AS 'SELECT min(id) FROM table1' LANGUAGE SQL;
SELECT * FROM table1 JOIN the_minimum_id() min_id ON (id = min_id);
-- a built-in immutable function
SELECT * FROM table1 JOIN abs(100) as hundred ON (id = hundred) ORDER BY id ASC;
-- function joins inside a CTE
WITH next_row_to_process AS (
SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n)
)
SELECT *
FROM table1, next_row_to_process
WHERE table1.data <= next_row_to_process.data
ORDER BY 1,2 ASC;
-- Multiple functions in an RTE
SELECT * FROM ROWS FROM (next_k_integers(5), next_k_integers(10)) AS f(a, b),
table1 WHERE id = a ORDER BY id ASC;
-- Custom Type returning function used in a join
CREATE TYPE min_and_max AS (
minimum INT,
maximum INT
);
CREATE OR REPLACE FUNCTION max_and_min () RETURNS
min_and_max AS $$
DECLARE
result min_and_max%rowtype;
begin
select into result min(data) as minimum, max(data) as maximum from table1;
return result;
end;
$$ language plpgsql;
SELECT * FROM table1 JOIN max_and_min() m ON (m.maximum = data OR m.minimum = data);
-- The following tests will fail as we do not support all joins on
-- all kinds of functions
SET client_min_messages TO ERROR;
-- function joins in CTE results can create lateral joins that are not supported
SELECT public.raise_failed_execution($cmd$
WITH one_row AS (
SELECT * FROM table1 WHERE id=52
)
SELECT table1.id, table1.data
FROM one_row, table1, next_k_integers(one_row.id, 5) next_five_ids
WHERE table1.id = next_five_ids;
$cmd$);
-- a user-defined immutable function
CREATE OR REPLACE FUNCTION the_answer_to_life()
RETURNS INTEGER IMMUTABLE AS 'SELECT 42' LANGUAGE SQL;
SELECT public.raise_failed_execution($cmd$
SELECT * FROM table1 JOIN the_answer_to_life() the_answer ON (id = the_answer)
$cmd$);
-- WITH ORDINALITY clause
SELECT public.raise_failed_execution($cmd$
SELECT *
FROM table1
JOIN next_k_integers(10,5) WITH ORDINALITY next_integers
ON (id = next_integers.result)
ORDER BY id ASC;
$cmd$);
RESET client_min_messages;
DROP SCHEMA functions_in_joins CASCADE;
SET search_path TO DEFAULT;

6
src/test/regress/sql/multi_hash_pruning.sql
View File

@ -9,6 +9,12 @@ SET citus.next_shard_id TO 630000;
SET citus.shard_count to 4;
SET citus.shard_replication_factor to 1;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- Create a table partitioned on integer column and update partition type to
-- hash. Then load data into this table and update shard min max values with
-- hashed ones. Hash value of 1, 2, 3 and 4 are consecutively -1905060026,

5
src/test/regress/sql/multi_mx_router_planner.sql
View File

@ -70,6 +70,11 @@ INSERT INTO articles_single_shard_hash_mx VALUES (50, 10, 'anjanette', 19519);
-- single-shard tests
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- test simple select for a single row
SELECT * FROM articles_hash_mx WHERE author_id = 10 AND id = 50;

4
src/test/regress/sql/multi_prepare_plsql.sql
View File

@ -6,6 +6,10 @@
-- and converted into both plain SQL and PL/pgsql functions, which
-- use prepared statements internally.
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
CREATE FUNCTION plpgsql_test_1() RETURNS TABLE(count bigint) AS $$
DECLARE

5
src/test/regress/sql/multi_prepare_sql.sql
View File

@ -2,6 +2,11 @@
-- MULTI_PREPARE_SQL
--
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- Tests covering PREPARE statements. Many of the queries are
-- taken from other regression test files and converted into
-- prepared statements.

6
src/test/regress/sql/multi_router_planner.sql
View File

@ -1,11 +1,15 @@
SET citus.next_shard_id TO 840000;
-- ===================================================================
-- test router planner functionality for single shard select queries
-- ===================================================================
-- all the tests in this file is intended for testing non-fast-path
-- router planner, so we're disabling it in this file. We've bunch of
-- other tests that triggers fast-path-router planner
SET citus.enable_fast_path_router_planner TO false;
CREATE TABLE articles_hash (
id bigint NOT NULL,
author_id bigint NOT NULL,

829
src/test/regress/sql/multi_router_planner_fast_path.sql Normal file
View File

@ -0,0 +1,829 @@
CREATE SCHEMA fast_path_router_select;
SET search_path TO fast_path_router_select;
SET citus.next_shard_id TO 1840000;
-- all the tests in this file is intended for testing fast-path
-- router planner, so we're explicitly enabling itin this file.
-- We've bunch of other tests that triggers non-fast-path-router
-- planner (note this is already true by default)
SET citus.enable_fast_path_router_planner TO true;
-- ===================================================================
-- test router planner functionality for via fast path on
-- single shard select queries
-- ===================================================================
CREATE TABLE articles_hash (
id bigint NOT NULL,
author_id bigint NOT NULL,
title varchar(20) NOT NULL,
word_count integer
);
CREATE TABLE articles_range (
id bigint NOT NULL,
author_id bigint NOT NULL,
title varchar(20) NOT NULL,
word_count integer
);
CREATE TABLE articles_append (
id bigint NOT NULL,
author_id bigint NOT NULL,
title varchar(20) NOT NULL,
word_count integer
);
-- Check for the existence of line 'DEBUG: Creating router plan'
-- to determine if router planner is used.
-- this table is used in a CTE test
CREATE TABLE authors_hash ( name varchar(20), id bigint );
CREATE TABLE authors_range ( name varchar(20), id bigint );
SET citus.shard_replication_factor TO 1;
SET citus.shard_count TO 2;
SELECT create_distributed_table('articles_hash', 'author_id');
CREATE TABLE authors_reference ( name varchar(20), id bigint );
SELECT create_reference_table('authors_reference');
-- create a bunch of test data
INSERT INTO articles_hash VALUES (1, 1, 'arsenous', 9572), (2, 2, 'abducing', 13642),( 3, 3, 'asternal', 10480),( 4, 4, 'altdorfer', 14551),( 5, 5, 'aruru', 11389),
(6, 6, 'atlases', 15459),(7, 7, 'aseptic', 12298),( 8, 8, 'agatized', 16368),(9, 9, 'alligate', 438),
(10, 10, 'aggrandize', 17277),(11, 1, 'alamo', 1347),(12, 2, 'archiblast', 18185),
(13, 3, 'aseyev', 2255),(14, 4, 'andesite', 19094),(15, 5, 'adversa', 3164),
(16, 6, 'allonym', 2),(17, 7, 'auriga', 4073),(18, 8, 'assembly', 911),(19, 9, 'aubergiste', 4981),
(20, 10, 'absentness', 1820),(21, 1, 'arcading', 5890),(22, 2, 'antipope', 2728),(23, 3, 'abhorring', 6799),
(24, 4, 'audacious', 3637),(25, 5, 'antehall', 7707),(26, 6, 'abington', 4545),(27, 7, 'arsenous', 8616),
(28, 8, 'aerophyte', 5454),(29, 9, 'amateur', 9524),(30, 10, 'andelee', 6363),(31, 1, 'athwartships', 7271),
(32, 2, 'amazon', 11342),(33, 3, 'autochrome', 8180),(34, 4, 'amnestied', 12250),(35, 5, 'aminate', 9089),
(36, 6, 'ablation', 13159),(37, 7, 'archduchies', 9997),(38, 8, 'anatine', 14067),(39, 9, 'anchises', 10906),
(40, 10, 'attemper', 14976),(41, 1, 'aznavour', 11814),(42, 2, 'ausable', 15885),(43, 3, 'affixal', 12723),
(44, 4, 'anteport', 16793),(45, 5, 'afrasia', 864),(46, 6, 'atlanta', 17702),(47, 7, 'abeyance', 1772),
(48, 8, 'alkylic', 18610),(49, 9, 'anyone', 2681),(50, 10, 'anjanette', 19519);
SET citus.task_executor_type TO 'real-time';
SET client_min_messages TO 'DEBUG2';
-- test simple select for a single row
SELECT * FROM articles_hash WHERE author_id = 10 AND id = 50;
-- get all titles by a single author
SELECT title FROM articles_hash WHERE author_id = 10;
-- try ordering them by word count
SELECT title, word_count FROM articles_hash
WHERE author_id = 10
ORDER BY word_count DESC NULLS LAST;
-- look at last two articles by an author
SELECT title, id FROM articles_hash
WHERE author_id = 5
ORDER BY id
LIMIT 2;
-- find all articles by two authors in same shard
-- but plan is not fast path router plannable due to
-- two distribution columns in the query
SELECT title, author_id FROM articles_hash
WHERE author_id = 7 OR author_id = 8
ORDER BY author_id ASC, id;
-- having clause is supported if it goes to a single shard
-- and single dist. key on the query
SELECT author_id, sum(word_count) AS corpus_size FROM articles_hash
WHERE author_id = 1
GROUP BY author_id
HAVING sum(word_count) > 1000
ORDER BY sum(word_count) DESC;
-- fast path planner only support = operator
SELECT * FROM articles_hash WHERE author_id <= 1;
SELECT * FROM articles_hash WHERE author_id IN (1, 3);
-- queries with CTEs cannot go through fast-path planning
WITH first_author AS ( SELECT id FROM articles_hash WHERE author_id = 1)
SELECT * FROM first_author;
-- two CTE joins also cannot go through fast-path planning
WITH id_author AS ( SELECT id, author_id FROM articles_hash WHERE author_id = 1),
id_title AS (SELECT id, title from articles_hash WHERE author_id = 1)
SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id;
-- this is a different case where each CTE is recursively planned and those goes
-- through the fast-path router planner, but the top level join is not
WITH id_author AS ( SELECT id, author_id FROM articles_hash WHERE author_id = 1),
id_title AS (SELECT id, title from articles_hash WHERE author_id = 2)
SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id;
CREATE TABLE company_employees (company_id int, employee_id int, manager_id int);
SELECT master_create_distributed_table('company_employees', 'company_id', 'hash');
SELECT master_create_worker_shards('company_employees', 4, 1);
INSERT INTO company_employees values(1, 1, 0);
INSERT INTO company_employees values(1, 2, 1);
INSERT INTO company_employees values(1, 3, 1);
INSERT INTO company_employees values(1, 4, 2);
INSERT INTO company_employees values(1, 5, 4);
INSERT INTO company_employees values(3, 1, 0);
INSERT INTO company_employees values(3, 15, 1);
INSERT INTO company_employees values(3, 3, 1);
-- recursive CTEs are also cannot go through fast
-- path planning
WITH RECURSIVE hierarchy as (
SELECT *, 1 AS level
FROM company_employees
WHERE company_id = 1 and manager_id = 0
UNION
SELECT ce.*, (h.level+1)
FROM hierarchy h JOIN company_employees ce
ON (h.employee_id = ce.manager_id AND
h.company_id = ce.company_id AND
ce.company_id = 1))
SELECT * FROM hierarchy WHERE LEVEL <= 2;
WITH update_article AS (
UPDATE articles_hash SET word_count = 10 WHERE id = 1 AND word_count = 9 RETURNING *
)
SELECT * FROM update_article;
WITH delete_article AS (
DELETE FROM articles_hash WHERE id = 1 AND word_count = 10 RETURNING *
)
SELECT * FROM delete_article;
-- grouping sets are supported via fast-path
SELECT
id, substring(title, 2, 1) AS subtitle, count(*)
FROM articles_hash
WHERE author_id = 1
GROUP BY GROUPING SETS ((id),(subtitle))
ORDER BY id, subtitle;
-- grouping sets are not supported with multiple quals
SELECT
id, substring(title, 2, 1) AS subtitle, count(*)
FROM articles_hash
WHERE author_id = 1 or author_id = 2
GROUP BY GROUPING SETS ((id),(subtitle))
ORDER BY id, subtitle;
-- queries which involve functions in FROM clause are not supported via fast path planning
SELECT * FROM articles_hash, position('om' in 'Thomas') WHERE author_id = 1;
-- sublinks are not supported via fast path planning
SELECT * FROM articles_hash
WHERE author_id IN (SELECT author_id FROM articles_hash WHERE author_id = 2)
ORDER BY articles_hash.id;
-- subqueries are not supported via fast path planning
SELECT articles_hash.id,test.word_count
FROM articles_hash, (SELECT id, word_count FROM articles_hash) AS test WHERE test.id = articles_hash.id
ORDER BY test.word_count DESC, articles_hash.id LIMIT 5;
SELECT articles_hash.id,test.word_count
FROM articles_hash, (SELECT id, word_count FROM articles_hash) AS test
WHERE test.id = articles_hash.id and articles_hash.author_id = 1
ORDER BY articles_hash.id;
-- subqueries are not supported in SELECT clause
SELECT a.title AS name, (SELECT a2.id FROM articles_hash a2 WHERE a.id = a2.id LIMIT 1)
AS special_price FROM articles_hash a;
-- simple lookup query just works
SELECT *
FROM articles_hash
WHERE author_id = 1;
-- below query hits a single shard but with multiple filters
-- so cannot go via fast-path
SELECT *
FROM articles_hash
WHERE author_id = 1 OR author_id = 17;
-- rename the output columns
SELECT id as article_id, word_count * id as random_value
FROM articles_hash
WHERE author_id = 1;
-- joins do not go through fast-path planning
SELECT a.author_id as first_author, b.word_count as second_word_count
FROM articles_hash a, articles_hash b
WHERE a.author_id = 10 and a.author_id = b.author_id
LIMIT 3;
-- single shard select with limit goes through fast-path planning
SELECT *
FROM articles_hash
WHERE author_id = 1
LIMIT 3;
-- single shard select with limit + offset goes through fast-path planning
SELECT *
FROM articles_hash
WHERE author_id = 1
LIMIT 2
OFFSET 1;
-- single shard select with limit + offset + order by goes through fast-path planning
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id desc
LIMIT 2
OFFSET 1;
-- single shard select with group by on non-partition column goes through fast-path planning
SELECT id
FROM articles_hash
WHERE author_id = 1
GROUP BY id
ORDER BY id;
-- single shard select with distinct goes through fast-path planning
SELECT DISTINCT id
FROM articles_hash
WHERE author_id = 1
ORDER BY id;
-- single shard aggregate goes through fast-path planning
SELECT avg(word_count)
FROM articles_hash
WHERE author_id = 2;
-- max, min, sum, count goes through fast-path planning
SELECT max(word_count) as max, min(word_count) as min,
sum(word_count) as sum, count(word_count) as cnt
FROM articles_hash
WHERE author_id = 2;
-- queries with aggregates and group by goes through fast-path planning
SELECT max(word_count)
FROM articles_hash
WHERE author_id = 1
GROUP BY author_id;
-- set operations are not supported via fast-path planning
SELECT * FROM (
SELECT * FROM articles_hash WHERE author_id = 1
UNION
SELECT * FROM articles_hash WHERE author_id = 3
) AS combination
ORDER BY id;
-- function calls in the target list is supported via fast path
SELECT LEFT(title, 1) FROM articles_hash WHERE author_id = 1
-- top-level union queries are supported through recursive planning
SET client_min_messages to 'NOTICE';
-- unions in subqueries are not supported via fast-path planning
SELECT * FROM (
(SELECT * FROM articles_hash WHERE author_id = 1)
UNION
(SELECT * FROM articles_hash WHERE author_id = 1)) uu
ORDER BY 1, 2
LIMIT 5;
-- Test various filtering options for router plannable check
SET client_min_messages to 'DEBUG2';
-- cannot go through fast-path if there is
-- explicit coercion
SELECT *
FROM articles_hash
WHERE author_id = 1::bigint;
-- can go through fast-path if there is
-- implicit coercion
-- This doesn't work see the related issue
-- reported https://github.com/citusdata/citus/issues/2605
-- SELECT *
-- FROM articles_hash
-- WHERE author_id = 1.0;
SELECT *
FROM articles_hash
WHERE author_id = 68719476736; -- this is bigint
-- cannot go through fast-path due to
-- multiple filters on the dist. key
SELECT *
FROM articles_hash
WHERE author_id = 1 and author_id >= 1;
-- cannot go through fast-path due to
-- multiple filters on the dist. key
SELECT *
FROM articles_hash
WHERE author_id = 1 or id = 1;
-- goes through fast-path planning because
-- the dist. key is ANDed with the rest of the
-- filters
SELECT *
FROM articles_hash
WHERE author_id = 1 and (id = 1 or id = 41);
-- this time there is an OR clause which prevents
-- router planning at all
SELECT *
FROM articles_hash
WHERE author_id = 1 and id = 1 or id = 41;
-- goes through fast-path planning because
-- the dist. key is ANDed with the rest of the
-- filters
SELECT *
FROM articles_hash
WHERE author_id = 1 and (id = random()::int * 0);
-- not router plannable due to function call on the right side
SELECT *
FROM articles_hash
WHERE author_id = (random()::int * 0 + 1);
-- Citus does not qualify this as a fast-path because
-- dist_key = func()
SELECT *
FROM articles_hash
WHERE author_id = abs(-1);
-- Citus does not qualify this as a fast-path because
-- dist_key = func()
SELECT *
FROM articles_hash
WHERE 1 = abs(author_id);
-- Citus does not qualify this as a fast-path because
-- dist_key = func()
SELECT *
FROM articles_hash
WHERE author_id = abs(author_id - 2);
-- the function is not on the dist. key, so qualify as
-- fast-path
SELECT *
FROM articles_hash
WHERE author_id = 1 and (id = abs(id - 2));
-- not router plannable due to is true
SELECT *
FROM articles_hash
WHERE (author_id = 1) is true;
-- router plannable, (boolean expression) = true is collapsed to (boolean expression)
SELECT *
FROM articles_hash
WHERE (author_id = 1) = true;
-- some more complex quals
SELECT count(*) FROM articles_hash WHERE (author_id = 15) AND (id = 1 OR word_count > 5);
SELECT count(*) FROM articles_hash WHERE (author_id = 15) OR (id = 1 AND word_count > 5);
SELECT count(*) FROM articles_hash WHERE (id = 15) OR (author_id = 1 AND word_count > 5);
SELECT count(*) FROM articles_hash WHERE (id = 15) AND (author_id = 1 OR word_count > 5);
SELECT count(*) FROM articles_hash WHERE (id = 15) AND (author_id = 1 AND (word_count > 5 OR id = 2));
SELECT count(*) FROM articles_hash WHERE (id = 15) AND (title ilike 'a%' AND (word_count > 5 OR author_id = 2));
SELECT count(*) FROM articles_hash WHERE (id = 15) AND (title ilike 'a%' AND (word_count > 5 AND author_id = 2));
SELECT count(*) FROM articles_hash WHERE (id = 15) AND (title ilike 'a%' AND ((word_count > 5 OR title ilike 'b%' ) AND (author_id = 2 AND word_count > 50)));
-- fast-path router plannable, between operator is on another column
SELECT *
FROM articles_hash
WHERE (author_id = 1) and id between 0 and 20;
-- fast-path router plannable, partition column expression is and'ed to rest
SELECT *
FROM articles_hash
WHERE (author_id = 1) and (id = 1 or id = 31) and title like '%s';
-- fast-path router plannable, order is changed
SELECT *
FROM articles_hash
WHERE (id = 1 or id = 31) and title like '%s' and (author_id = 1);
-- fast-path router plannable
SELECT *
FROM articles_hash
WHERE (title like '%s' or title like 'a%') and (author_id = 1);
-- fast-path router plannable
SELECT *
FROM articles_hash
WHERE (title like '%s' or title like 'a%') and (author_id = 1) and (word_count < 3000 or word_count > 8000);
-- window functions are supported with fast-path router plannable
SELECT LAG(title, 1) over (ORDER BY word_count) prev, title, word_count
FROM articles_hash
WHERE author_id = 5;
SELECT LAG(title, 1) over (ORDER BY word_count) prev, title, word_count
FROM articles_hash
WHERE author_id = 5
ORDER BY word_count DESC;
SELECT id, MIN(id) over (order by word_count)
FROM articles_hash
WHERE author_id = 1;
SELECT id, word_count, AVG(word_count) over (order by word_count)
FROM articles_hash
WHERE author_id = 1;
SELECT word_count, rank() OVER (PARTITION BY author_id ORDER BY word_count)
FROM articles_hash
WHERE author_id = 1;
-- some more tests on complex target lists
SELECT DISTINCT ON (author_id, id) author_id, id,
MIN(id) over (order by avg(word_count)) * AVG(id * 5.2 + (1.0/max(word_count))) over (order by max(word_count)) as t1,
count(*) FILTER (WHERE title LIKE 'al%') as cnt_with_filter,
count(*) FILTER (WHERE '0300030' LIKE '%3%') as cnt_with_filter_2,
avg(case when id > 2 then char_length(word_count::text) * (id * strpos(word_count::text, '1')) end) as case_cnt,
COALESCE(strpos(avg(word_count)::text, '1'), 20)
FROM articles_hash as aliased_table
WHERE author_id = 1
GROUP BY author_id, id
HAVING count(DISTINCT title) > 0
ORDER BY author_id, id, sum(word_count) - avg(char_length(title)) DESC, COALESCE(array_upper(ARRAY[max(id)],1) * 5,0) DESC;
-- where false queries are router plannable but not fast-path
SELECT *
FROM articles_hash
WHERE false;
-- fast-path with false
SELECT *
FROM articles_hash
WHERE author_id = 1 and false;
-- fast-path with false
SELECT *
FROM articles_hash
WHERE author_id = 1 and 1=0;
SELECT *
FROM articles_hash
WHERE null and author_id = 1;
-- we cannot qualify dist_key = X operator Y via
-- fast-path planning
SELECT *
FROM articles_hash
WHERE author_id = 1 + 1;
-- where false with immutable function returning false
-- goes through fast-path
SELECT *
FROM articles_hash a
WHERE a.author_id = 10 and int4eq(1, 2);
-- partition_column is null clause does not prune out any shards,
-- all shards remain after shard pruning, not router plannable
-- not fast-path router either
SELECT *
FROM articles_hash a
WHERE a.author_id is null;
-- partition_column equals to null clause prunes out all shards
-- no shards after shard pruning, router plannable
-- not fast-path router either
SELECT *
FROM articles_hash a
WHERE a.author_id = null;
-- union/difference /intersection with where false
-- this query was not originally router plannable, addition of 1=0
-- makes it router plannable but not fast-path
SELECT * FROM (
SELECT * FROM articles_hash WHERE author_id = 1
UNION
SELECT * FROM articles_hash WHERE author_id = 2 and 1=0
) AS combination
ORDER BY id;
-- same with the above, but with WHERE false
SELECT * FROM (
SELECT * FROM articles_hash WHERE author_id = 1
UNION
SELECT * FROM articles_hash WHERE author_id = 2 and 1=0
) AS combination WHERE false
ORDER BY id;
-- window functions with where false
SELECT word_count, rank() OVER (PARTITION BY author_id ORDER BY word_count)
FROM articles_hash
WHERE author_id = 1 and 1=0;
-- create a dummy function to be used in filtering
CREATE OR REPLACE FUNCTION someDummyFunction(regclass)
RETURNS text AS
$$
BEGIN
RETURN md5($1::text);
END;
$$ LANGUAGE 'plpgsql' IMMUTABLE;
-- fast path router plannable, but errors
SELECT * FROM articles_hash
WHERE
someDummyFunction('articles_hash') = md5('articles_hash') AND author_id = 1
ORDER BY
author_id, id
LIMIT 5;
-- temporarily turn off debug messages before dropping the function
SET client_min_messages TO 'NOTICE';
DROP FUNCTION someDummyFunction(regclass);
SET client_min_messages TO 'DEBUG2';
-- complex query hitting a single shard and a fast-path
SELECT
count(DISTINCT CASE
WHEN
word_count > 100
THEN
id
ELSE
NULL
END) as c
FROM
articles_hash
WHERE
author_id = 5;
-- queries inside transactions can be fast-path router plannable
BEGIN;
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id;
END;
-- queries inside read-only transactions can be fast-path router plannable
SET TRANSACTION READ ONLY;
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id;
END;
-- cursor queries are fast-path router plannable
BEGIN;
DECLARE test_cursor CURSOR FOR
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id;
FETCH test_cursor;
FETCH ALL test_cursor;
FETCH test_cursor; -- fetch one row after the last
FETCH BACKWARD test_cursor;
END;
-- queries inside copy can be router plannable
COPY (
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id) TO STDOUT;
-- table creation queries inside can be fast-path router plannable
CREATE TEMP TABLE temp_articles_hash as
SELECT *
FROM articles_hash
WHERE author_id = 1
ORDER BY id;
-- fast-path router plannable queries may include filter for aggragates
SELECT count(*), count(*) FILTER (WHERE id < 3)
FROM articles_hash
WHERE author_id = 1;
-- prepare queries can be router plannable
PREPARE author_1_articles as
SELECT *
FROM articles_hash
WHERE author_id = 1;
EXECUTE author_1_articles;
EXECUTE author_1_articles;
EXECUTE author_1_articles;
EXECUTE author_1_articles;
EXECUTE author_1_articles;
EXECUTE author_1_articles;
-- parametric prepare queries can be router plannable
PREPARE author_articles(int) as
SELECT *
FROM articles_hash
WHERE author_id = $1;
EXECUTE author_articles(1);
EXECUTE author_articles(1);
EXECUTE author_articles(1);
EXECUTE author_articles(1);
EXECUTE author_articles(1);
EXECUTE author_articles(1);
-- queries inside plpgsql functions could be router plannable
CREATE OR REPLACE FUNCTION author_articles_max_id() RETURNS int AS $$
DECLARE
max_id integer;
BEGIN
SELECT MAX(id) FROM articles_hash ah
WHERE author_id = 1
into max_id;
return max_id;
END;
$$ LANGUAGE plpgsql;
-- we don't want too many details. though we're omitting
-- "DETAIL: distribution column value:", we see it acceptable
-- since the query results verifies the correctness
\set VERBOSITY terse
SELECT author_articles_max_id();
SELECT author_articles_max_id();
SELECT author_articles_max_id();
SELECT author_articles_max_id();
SELECT author_articles_max_id();
SELECT author_articles_max_id();
-- queries inside plpgsql functions could be router plannable
CREATE OR REPLACE FUNCTION author_articles_max_id(int) RETURNS int AS $$
DECLARE
max_id integer;
BEGIN
SELECT MAX(id) FROM articles_hash ah
WHERE author_id = $1
into max_id;
return max_id;
END;
$$ LANGUAGE plpgsql;
SELECT author_articles_max_id(1);
SELECT author_articles_max_id(1);
SELECT author_articles_max_id(1);
SELECT author_articles_max_id(1);
SELECT author_articles_max_id(1);
SELECT author_articles_max_id(1);
-- check that function returning setof query are router plannable
CREATE OR REPLACE FUNCTION author_articles_id_word_count() RETURNS TABLE(id bigint, word_count int) AS $$
DECLARE
BEGIN
RETURN QUERY
SELECT ah.id, ah.word_count
FROM articles_hash ah
WHERE author_id = 1;
END;
$$ LANGUAGE plpgsql;
SELECT * FROM author_articles_id_word_count();
SELECT * FROM author_articles_id_word_count();
SELECT * FROM author_articles_id_word_count();
SELECT * FROM author_articles_id_word_count();
SELECT * FROM author_articles_id_word_count();
SELECT * FROM author_articles_id_word_count();
-- check that function returning setof query are router plannable
CREATE OR REPLACE FUNCTION author_articles_id_word_count(int) RETURNS TABLE(id bigint, word_count int) AS $$
DECLARE
BEGIN
RETURN QUERY
SELECT ah.id, ah.word_count
FROM articles_hash ah
WHERE author_id = $1;
END;
$$ LANGUAGE plpgsql;
SELECT * FROM author_articles_id_word_count(1);
SELECT * FROM author_articles_id_word_count(1);
SELECT * FROM author_articles_id_word_count(1);
SELECT * FROM author_articles_id_word_count(1);
SELECT * FROM author_articles_id_word_count(1);
SELECT * FROM author_articles_id_word_count(1);
\set VERBOSITY default
-- insert .. select via coordinator could also
-- use fast-path queries
PREPARE insert_sel(int, int) AS
INSERT INTO articles_hash
SELECT * FROM articles_hash WHERE author_id = $2 AND word_count = $1 OFFSET 0;
EXECUTE insert_sel(1,1);
EXECUTE insert_sel(1,1);
EXECUTE insert_sel(1,1);
EXECUTE insert_sel(1,1);
EXECUTE insert_sel(1,1);
EXECUTE insert_sel(1,1);
-- one final interesting preperad statement
-- where one of the filters is on the target list
PREPARE fast_path_agg_filter(int, int) AS
SELECT
count(*) FILTER (WHERE word_count=$1)
FROM
articles_hash
WHERE author_id = $2;
EXECUTE fast_path_agg_filter(1,1);
EXECUTE fast_path_agg_filter(2,2);
EXECUTE fast_path_agg_filter(3,3);
EXECUTE fast_path_agg_filter(4,4);
EXECUTE fast_path_agg_filter(5,5);
EXECUTE fast_path_agg_filter(6,6);
-- views internally become subqueries, so not fast-path router query
CREATE VIEW test_view AS
SELECT * FROM articles_hash WHERE author_id = 1;
SELECT * FROM test_view;
-- materialized views can be created for fast-path router plannable queries
CREATE MATERIALIZED VIEW mv_articles_hash_empty AS
SELECT * FROM articles_hash WHERE author_id = 1;
SELECT * FROM mv_articles_hash_empty;
-- fast-path router planner/executor is enabled for task-tracker executor
SET citus.task_executor_type to 'task-tracker';
SELECT id
FROM articles_hash
WHERE author_id = 1;
-- insert query is router plannable even under task-tracker
INSERT INTO articles_hash VALUES (51, 1, 'amateus', 1814), (52, 1, 'second amateus', 2824);
-- verify insert is successfull (not router plannable and executable)
SELECT id
FROM articles_hash
WHERE author_id = 1;
SET client_min_messages to 'NOTICE';
-- finally, some tests with partitioned tables
CREATE TABLE collections_list (
key bigint,
ts timestamptz,
collection_id integer,
value numeric
) PARTITION BY LIST (collection_id );
CREATE TABLE collections_list_1
PARTITION OF collections_list (key, ts, collection_id, value)
FOR VALUES IN ( 1 );
CREATE TABLE collections_list_2
PARTITION OF collections_list (key, ts, collection_id, value)
FOR VALUES IN ( 2 );
-- we don't need many shards
SET citus.shard_count TO 2;
SELECT create_distributed_table('collections_list', 'key');
INSERT INTO collections_list SELECT i % 10, now(), (i % 2) + 1, i*i FROM generate_series(0, 50)i;
SET client_min_messages to 'DEBUG2';
SELECT count(*) FROM collections_list WHERE key = 4;
SELECT count(*) FROM collections_list_1 WHERE key = 4;
SELECT count(*) FROM collections_list_2 WHERE key = 4;
UPDATE collections_list SET value = 15 WHERE key = 4;
SELECT count(*) FILTER (where value = 15) FROM collections_list WHERE key = 4;
SELECT count(*) FILTER (where value = 15) FROM collections_list_1 WHERE key = 4;
SELECT count(*) FILTER (where value = 15) FROM collections_list_2 WHERE key = 4;
SET client_min_messages to 'NOTICE';
DROP FUNCTION author_articles_max_id();
DROP FUNCTION author_articles_id_word_count();
DROP MATERIALIZED VIEW mv_articles_hash_empty;
DROP MATERIALIZED VIEW mv_articles_hash_data;
DROP TABLE articles_hash;
DROP TABLE authors_hash;
DROP TABLE authors_range;
DROP TABLE authors_reference;
DROP TABLE company_employees;
DROP TABLE articles_range;
DROP TABLE articles_append;
DROP TABLE collections_list;
RESET search_path;
DROP SCHEMA fast_path_router_select CASCADE;

12
src/test/regress/sql/multi_simple_queries.sql
View File

@ -1,6 +1,10 @@
SET citus.next_shard_id TO 850000;
-- many of the tests in this file is intended for testing non-fast-path
-- router planner, so we're explicitly disabling it in this file.
-- We've bunch of other tests that triggers fast-path-router
SET citus.enable_fast_path_router_planner TO false;
-- ===================================================================
-- test end-to-end query functionality
@ -302,4 +306,12 @@ SELECT * FROM articles TABLESAMPLE BERNOULLI (0) WHERE author_id = 1;
SELECT * FROM articles TABLESAMPLE SYSTEM (100) WHERE author_id = 1 ORDER BY id;
SELECT * FROM articles TABLESAMPLE BERNOULLI (100) WHERE author_id = 1 ORDER BY id;
-- test tablesample with fast path as well
SET citus.enable_fast_path_router_planner TO true;
SELECT * FROM articles TABLESAMPLE SYSTEM (0) WHERE author_id = 1;
SELECT * FROM articles TABLESAMPLE BERNOULLI (0) WHERE author_id = 1;
SELECT * FROM articles TABLESAMPLE SYSTEM (100) WHERE author_id = 1 ORDER BY id;
SELECT * FROM articles TABLESAMPLE BERNOULLI (100) WHERE author_id = 1 ORDER BY id;
SET client_min_messages to 'NOTICE';

28
src/test/regress/sql/multi_subquery_complex_queries.sql
View File

@ -2400,3 +2400,31 @@ FROM
USING (user_id)
GROUP BY a.user_id
ORDER BY 1;
-- queries where column aliases are used
-- the query is not very complex. join is given an alias with aliases
-- for each output column
SELECT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
SELECT DISTINCT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e ON (u.user_id = e.user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e USING (user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;

141
src/test/regress/sql/multi_task_assignment_policy.sql
View File

@ -9,6 +9,28 @@ SET citus.next_shard_id TO 880000;
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 9 AS version_above_nine;
-- the function simply parses the results and returns 'shardId@worker'
-- for all the explain task outputs
CREATE OR REPLACE FUNCTION parse_explain_output(in qry text, in table_name text, out r text)
RETURNS SETOF TEXT AS $$
DECLARE
portOfTheTask text;
shardOfTheTask text;
begin
for r in execute qry loop
IF r LIKE '%port%' THEN
portOfTheTask = substring(r, '([0-9]{1,10})');
END IF;
IF r LIKE '%' || table_name || '%' THEN
shardOfTheTask = substring(r, '([0-9]{1,10})');
return QUERY SELECT shardOfTheTask || '@' || portOfTheTask ;
END IF;
end loop;
return;
end; $$ language plpgsql;
SET citus.explain_distributed_queries TO off;
@ -80,31 +102,19 @@ EXPLAIN SELECT count(*) FROM task_assignment_test_table;
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
-- Finally test the round-robin task assignment policy
SET citus.task_assignment_policy TO 'round-robin';
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
EXPLAIN SELECT count(*) FROM task_assignment_test_table;
RESET citus.task_assignment_policy;
RESET client_min_messages;
COMMIT;
CREATE TABLE task_assignment_reference_table (test_id integer);
SELECT create_reference_table('task_assignment_reference_table');
BEGIN;
SET LOCAL client_min_messages TO DEBUG3;
SET LOCAL citus.explain_distributed_queries TO off;
-- Check how task_assignment_policy impact planning decisions for reference tables
CREATE TABLE task_assignment_reference_table (test_id integer);
SELECT create_reference_table('task_assignment_reference_table');
SET LOCAL citus.task_assignment_policy TO 'greedy';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
@ -113,15 +123,100 @@ SET LOCAL citus.task_assignment_policy TO 'first-replica';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
-- here we expect debug output showing two different hosts for subsequent queries
SET LOCAL citus.task_assignment_policy TO 'round-robin';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
ROLLBACK;
RESET client_min_messages;
-- Now, lets test round-robin policy
-- round-robin policy relies on PostgreSQL's local transactionId,
-- which might change and we don't have any control over it.
-- the important thing that we look for is that round-robin policy
-- should give the same output for executions in the same transaction
-- and different output for executions that are not insdie the
-- same transaction. To ensure that, we define a helper function
BEGIN;
SET LOCAL citus.explain_distributed_queries TO on;
CREATE TEMPORARY TABLE explain_outputs (value text);
SET LOCAL citus.task_assignment_policy TO 'round-robin';
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
-- given that we're in the same transaction, the count should be 1
SELECT count(DISTINCT value) FROM explain_outputs;
DROP TABLE explain_outputs;
COMMIT;
-- now test round-robin policy outside
-- a transaction, we should see the assignements
-- change on every execution
CREATE TEMPORARY TABLE explain_outputs (value text);
SET citus.task_assignment_policy TO 'round-robin';
SET citus.explain_distributed_queries TO ON;
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_reference_table;', 'task_assignment_reference_table');
-- given that we're in the same transaction, the count should be 2
-- since there are two different worker nodes
SELECT count(DISTINCT value) FROM explain_outputs;
TRUNCATE explain_outputs;
-- same test with a distributed table
-- we keep this test because as of this commit, the code
-- paths for reference tables and distributed tables are
-- not the same
SET citus.shard_replication_factor TO 2;
CREATE TABLE task_assignment_replicated_hash (test_id integer);
SELECT create_distributed_table('task_assignment_replicated_hash', 'test_id');
BEGIN;
SET LOCAL citus.explain_distributed_queries TO on;
SET LOCAL citus.task_assignment_policy TO 'round-robin';
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
-- given that we're in the same transaction, the count should be 1
SELECT count(DISTINCT value) FROM explain_outputs;
DROP TABLE explain_outputs;
COMMIT;
-- now test round-robin policy outside
-- a transaction, we should see the assignements
-- change on every execution
CREATE TEMPORARY TABLE explain_outputs (value text);
SET citus.task_assignment_policy TO 'round-robin';
SET citus.explain_distributed_queries TO ON;
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
INSERT INTO explain_outputs
SELECT parse_explain_output('EXPLAIN SELECT count(*) FROM task_assignment_replicated_hash;', 'task_assignment_replicated_hash');
-- given that we're in the same transaction, the count should be 2
-- since there are two different worker nodes
SELECT count(DISTINCT value) FROM explain_outputs;
RESET citus.task_assignment_policy;
RESET client_min_messages;
-- we should be able to use round-robin with router queries that
-- only contains intermediate results
BEGIN;
@ -133,4 +228,4 @@ SET LOCAL citus.task_assignment_policy TO 'round-robin';
WITH q1 AS (SELECT * FROM task_assignment_test_table_2) SELECT * FROM q1;
ROLLBACK;
DROP TABLE task_assignment_replicated_hash, task_assignment_reference_table;