From 1b4d7a51f824ce3fb72a08de4337e608a670079b Mon Sep 17 00:00:00 2001 From: aykut-bozkurt Date: Wed, 13 Sep 2023 14:20:21 +0300 Subject: [PATCH 001/155] bump citus into 12.1.0 --- configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configure b/configure index c98037d89..723b48e26 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1devel. +# Generated by GNU Autoconf 2.69 for Citus 12.1.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1devel' -PACKAGE_STRING='Citus 12.1devel' +PACKAGE_VERSION='12.1.0' +PACKAGE_STRING='Citus 12.1.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1devel to adapt to many kinds of systems. +\`configure' configures Citus 12.1.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1devel:";; + short | recursive ) echo "Configuration of Citus 12.1.0:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1devel +Citus configure 12.1.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1devel, which was +It was created by Citus $as_me 12.1.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1devel, which was +This file was extended by Citus $as_me 12.1.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1devel +Citus config.status 12.1.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 7e5619857..0642cbd75 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1devel]) +AC_INIT([Citus], [12.1.0]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index fe203efb5..67ec1d78b 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1devel + 12.1.0 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 9b6ffece5ea043056e4d13532c75bf04bc657792 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:23:04 +0300 Subject: [PATCH 002/155] Adds PostgreSQL 16.0 Support (#7201) This commit concludes PG16.0 Support in Citus. The main PG16 support work has been done for 16beta3 https://github.com/citusdata/citus/pull/6952 There was some extra work needed for 16rc1 https://github.com/citusdata/citus/pull/7173 And this PR yet introduces some extra work needed to 16.0 :) `pgstat_fetch_stat_local_beentry` has been renamed to `pgstat_get_local_beentry_by_index` in PG16.0 Relevant PG commit: https://github.com/postgres/postgres/commit/8dfa37b797843a83a5756ea3309055e8953e1a86 8dfa37b797843a83a5756ea3309055e8953e1a86 Sister PR https://github.com/citusdata/the-process/pull/150 (cherry picked from commit 4e46708789478d6deccd3d121f2b4da7f631ebe3) --- .circleci/config.yml | 6 +++--- src/include/pg_version_compat.h | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1c0f1553..8addafccc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: parameters: image_suffix: type: string - default: '-v641cdcd' + default: '-v87fd773' pg14_version: type: string default: '14.9' @@ -15,10 +15,10 @@ parameters: default: '15.4' pg16_version: type: string - default: '16rc1' + default: '16.0' upgrade_pg_versions: type: string - default: '14.9-15.4-16rc1' + default: '14.9-15.4-16.0' style_checker_tools_version: type: string default: '0.8.18' diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 2b0320003..1bdbae580 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -46,6 +46,8 @@ get_guc_variables_compat(int *gucCount) #define object_ownercheck(a, b, c) object_ownercheck(a, b, c) #define object_aclcheck(a, b, c, d) object_aclcheck(a, b, c, d) +#define pgstat_fetch_stat_local_beentry(a) pgstat_get_local_beentry_by_index(a) + #else #include "catalog/pg_class_d.h" From 3b908eec2aa281159966b4fa7bdee56ad92fd6de Mon Sep 17 00:00:00 2001 From: aykut-bozkurt <51649454+aykut-bozkurt@users.noreply.github.com> Date: Wed, 20 Sep 2023 16:45:04 +0300 Subject: [PATCH 003/155] Fix the changelog entry for citus_pause_node_within_txn() UDF (#7215) (cherry picked from commit 2c190d068918d1c457894adf97f550e5b3739184) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2b2ce99..02fc91d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,8 @@ * Add `citus_schema_move()` function which moves tables within a distributed schema to another node (#7180) -* Adds `citus_pause_node()` UDF that allows pausing the node with given id - (#7089) +* Adds `citus_pause_node_within_txn()` UDF that allows pausing the node with + given id (#7089) * Makes sure to enforce shard level colocation with the GUC `citus.enable_non_colocated_router_query_pushdown` (#7076) From e59ffbf54913830c1a186348d7ee453ad67dfd43 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Mon, 9 Oct 2023 13:05:51 +0200 Subject: [PATCH 004/155] Fix leaking of memory and memory contexts in Foreign Constraint Graphs (#7236) DESCRIPTION: Fix leaking of memory and memory contexts in Foreign Constraint Graphs Previously, every time we (re)created the Foreign Constraint Relationship Graph, we created a new Memory Context while loosing a reference to the previous context. This old context could still have left over memory in there causing a memory leak. With this patch we statically have one memory context that we lazily initialize the first time we create our foreign constraint relationship graph. On every subsequent creation, beside destroying our previous hashmap we also reset our memory context to remove any left over references. --- .../utils/foreign_key_relationship.c | 57 ++++++++++--------- .../distributed/foreign_key_relationship.h | 1 - 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index 2858e6ed3..d30c767df 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -28,6 +28,7 @@ #include "distributed/version_compat.h" #include "nodes/pg_list.h" #include "storage/lockdefs.h" +#include "utils/catcache.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "common/hashfn.h" @@ -96,6 +97,8 @@ static List * GetConnectedListHelper(ForeignConstraintRelationshipNode *node, bool isReferencing); static List * GetForeignConstraintRelationshipHelper(Oid relationId, bool isReferencing); +MemoryContext ForeignConstraintRelationshipMemoryContext = NULL; + /* * GetForeignKeyConnectedRelationIdList returns a list of relation id's for @@ -321,17 +324,36 @@ CreateForeignConstraintRelationshipGraph() return; } - ClearForeignConstraintRelationshipGraphContext(); + /* + * Lazily create our memory context once and reset on every reuse. + * Since we have cleared and invalidated the fConstraintRelationshipGraph, right + * before we can simply reset the context if it was already existing. + */ + if (ForeignConstraintRelationshipMemoryContext == NULL) + { + /* make sure we've initialized CacheMemoryContext */ + if (CacheMemoryContext == NULL) + { + CreateCacheMemoryContext(); + } - MemoryContext fConstraintRelationshipMemoryContext = AllocSetContextCreateInternal( - CacheMemoryContext, - "Forign Constraint Relationship Graph Context", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); + ForeignConstraintRelationshipMemoryContext = AllocSetContextCreate( + CacheMemoryContext, + "Foreign Constraint Relationship Graph Context", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + } + else + { + fConstraintRelationshipGraph = NULL; + MemoryContextReset(ForeignConstraintRelationshipMemoryContext); + } + + Assert(fConstraintRelationshipGraph == NULL); MemoryContext oldContext = MemoryContextSwitchTo( - fConstraintRelationshipMemoryContext); + ForeignConstraintRelationshipMemoryContext); fConstraintRelationshipGraph = (ForeignConstraintRelationshipGraph *) palloc( sizeof(ForeignConstraintRelationshipGraph)); @@ -631,22 +653,3 @@ CreateOrFindNode(HTAB *adjacencyLists, Oid relid) return node; } - - -/* - * ClearForeignConstraintRelationshipGraphContext clear all the allocated memory obtained - * for foreign constraint relationship graph. Since all the variables of relationship - * graph was obtained within the same context, destroying hash map is enough as - * it deletes the context. - */ -void -ClearForeignConstraintRelationshipGraphContext() -{ - if (fConstraintRelationshipGraph == NULL) - { - return; - } - - hash_destroy(fConstraintRelationshipGraph->nodeMap); - fConstraintRelationshipGraph = NULL; -} diff --git a/src/include/distributed/foreign_key_relationship.h b/src/include/distributed/foreign_key_relationship.h index ef2c5be33..fbbee831e 100644 --- a/src/include/distributed/foreign_key_relationship.h +++ b/src/include/distributed/foreign_key_relationship.h @@ -20,7 +20,6 @@ extern bool ShouldUndistributeCitusLocalTable(Oid relationId); extern List * ReferencedRelationIdList(Oid relationId); extern List * ReferencingRelationIdList(Oid relationId); extern void SetForeignConstraintRelationshipGraphInvalid(void); -extern void ClearForeignConstraintRelationshipGraphContext(void); extern bool OidVisited(HTAB *oidVisitedMap, Oid oid); extern void VisitOid(HTAB *oidVisitedMap, Oid oid); From a4fe969947f60b0ad3758acc8e0213ade5b00809 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 25 Sep 2023 11:14:35 +0300 Subject: [PATCH 005/155] Make sure to disallow creating a replicated distributed table concurrently (#7219) See explanation in https://github.com/citusdata/citus/issues/7216. Fixes https://github.com/citusdata/citus/issues/7216. DESCRIPTION: Makes sure to disallow creating a replicated distributed table concurrently (cherry picked from commit 111b4c19bc0a429abc5be56dbb6ded9f080fa110) --- .../commands/create_distributed_table.c | 13 ++++++ .../create_distributed_table_concurrently.out | 40 ++++++++----------- .../create_distributed_table_concurrently.sql | 9 +++++ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index dc06692b3..1e89c6b93 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -445,6 +445,19 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName, if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone( colocateWithTableName)) { + if (replicationModel != REPLICATION_MODEL_STREAMING) + { + ereport(ERROR, (errmsg("cannot create distributed table " + "concurrently because Citus allows " + "concurrent table distribution only when " + "citus.shard_replication_factor = 1"), + errhint("table %s is requested to be colocated " + "with %s which has " + "citus.shard_replication_factor > 1", + get_rel_name(relationId), + colocateWithTableName))); + } + EnsureColocateWithTableIsValid(relationId, distributionMethod, distributionColumnName, colocateWithTableName); diff --git a/src/test/regress/expected/create_distributed_table_concurrently.out b/src/test/regress/expected/create_distributed_table_concurrently.out index 025629efa..1bf366fb3 100644 --- a/src/test/regress/expected/create_distributed_table_concurrently.out +++ b/src/test/regress/expected/create_distributed_table_concurrently.out @@ -36,6 +36,19 @@ set citus.shard_replication_factor to 2; select create_distributed_table_concurrently('test','key', 'hash'); ERROR: cannot distribute a table concurrently when citus.shard_replication_factor > 1 set citus.shard_replication_factor to 1; +set citus.shard_replication_factor to 2; +create table dist_1(a int); +select create_distributed_table('dist_1', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +set citus.shard_replication_factor to 1; +create table dist_2(a int); +select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1'); +ERROR: cannot create distributed table concurrently because Citus allows concurrent table distribution only when citus.shard_replication_factor = 1 +HINT: table dist_2 is requested to be colocated with dist_1 which has citus.shard_replication_factor > 1 begin; select create_distributed_table_concurrently('test','key'); ERROR: create_distributed_table_concurrently cannot run inside a transaction block @@ -138,27 +151,8 @@ select count(*) from test; rollback; -- verify that we can undistribute the table begin; +set local client_min_messages to warning; select undistribute_table('test', cascade_via_foreign_keys := true); -NOTICE: converting the partitions of create_distributed_table_concurrently.test -NOTICE: creating a new table for create_distributed_table_concurrently.test -NOTICE: dropping the old create_distributed_table_concurrently.test -NOTICE: renaming the new table to create_distributed_table_concurrently.test -NOTICE: creating a new table for create_distributed_table_concurrently.ref -NOTICE: moving the data of create_distributed_table_concurrently.ref -NOTICE: dropping the old create_distributed_table_concurrently.ref -NOTICE: drop cascades to constraint test_id_fkey_1190041 on table create_distributed_table_concurrently.test_1190041 -CONTEXT: SQL statement "SELECT citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false)" -PL/pgSQL function citus_drop_trigger() line XX at PERFORM -SQL statement "DROP TABLE create_distributed_table_concurrently.ref CASCADE" -NOTICE: renaming the new table to create_distributed_table_concurrently.ref -NOTICE: creating a new table for create_distributed_table_concurrently.test_1 -NOTICE: moving the data of create_distributed_table_concurrently.test_1 -NOTICE: dropping the old create_distributed_table_concurrently.test_1 -NOTICE: renaming the new table to create_distributed_table_concurrently.test_1 -NOTICE: creating a new table for create_distributed_table_concurrently.test_2 -NOTICE: moving the data of create_distributed_table_concurrently.test_2 -NOTICE: dropping the old create_distributed_table_concurrently.test_2 -NOTICE: renaming the new table to create_distributed_table_concurrently.test_2 undistribute_table --------------------------------------------------------------------- @@ -245,7 +239,7 @@ insert into dist_table4 select s from generate_series(1,100) s; select count(*) as total from dist_table4; total --------------------------------------------------------------------- - 100 + 100 (1 row) -- verify we do not allow foreign keys from distributed table to citus local table concurrently @@ -295,13 +289,13 @@ select count(*) from test_columnar; select id from test_columnar where id = 1; id --------------------------------------------------------------------- - 1 + 1 (1 row) select id from test_columnar where id = 51; id --------------------------------------------------------------------- - 51 + 51 (1 row) select count(*) from test_columnar_1; diff --git a/src/test/regress/sql/create_distributed_table_concurrently.sql b/src/test/regress/sql/create_distributed_table_concurrently.sql index 9632eba6e..6820d782c 100644 --- a/src/test/regress/sql/create_distributed_table_concurrently.sql +++ b/src/test/regress/sql/create_distributed_table_concurrently.sql @@ -28,6 +28,14 @@ set citus.shard_replication_factor to 2; select create_distributed_table_concurrently('test','key', 'hash'); set citus.shard_replication_factor to 1; +set citus.shard_replication_factor to 2; +create table dist_1(a int); +select create_distributed_table('dist_1', 'a'); +set citus.shard_replication_factor to 1; + +create table dist_2(a int); +select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1'); + begin; select create_distributed_table_concurrently('test','key'); rollback; @@ -63,6 +71,7 @@ rollback; -- verify that we can undistribute the table begin; +set local client_min_messages to warning; select undistribute_table('test', cascade_via_foreign_keys := true); rollback; From 2502e7e754276fa7858662c233ee5e4a07e09472 Mon Sep 17 00:00:00 2001 From: Gokhan Gulbiz Date: Mon, 13 Nov 2023 14:46:31 +0300 Subject: [PATCH 006/155] Backport GHA Migration to release-12.1 (#7277) Co-authored-by: Jelte Fennema-Nio --- .circleci/config.yml | 1131 ----------------- .github/actions/parallelization/action.yml | 23 + .../actions/save_logs_and_results/action.yml | 38 + .github/actions/setup_extension/action.yml | 35 + .github/actions/upload_coverage/action.yml | 27 + .github/workflows/build_and_test.yml | 505 ++++++++ .github/workflows/flaky_test_debugging.yml | 79 ++ .../workflows/packaging-test-pipelines.yml | 12 +- ci/build-citus.sh | 5 +- ci/check_all_ci_scripts_are_run.sh | 4 +- ci/check_enterprise_merge.sh | 96 -- 11 files changed, 717 insertions(+), 1238 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/actions/parallelization/action.yml create mode 100644 .github/actions/save_logs_and_results/action.yml create mode 100644 .github/actions/setup_extension/action.yml create mode 100644 .github/actions/upload_coverage/action.yml create mode 100644 .github/workflows/build_and_test.yml create mode 100644 .github/workflows/flaky_test_debugging.yml delete mode 100755 ci/check_enterprise_merge.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8addafccc..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1131 +0,0 @@ -version: 2.1 -orbs: - codecov: codecov/codecov@1.1.1 - azure-cli: circleci/azure-cli@1.0.0 - -parameters: - image_suffix: - type: string - default: '-v87fd773' - pg14_version: - type: string - default: '14.9' - pg15_version: - type: string - default: '15.4' - pg16_version: - type: string - default: '16.0' - upgrade_pg_versions: - type: string - default: '14.9-15.4-16.0' - style_checker_tools_version: - type: string - default: '0.8.18' - flaky_test: - type: string - default: '' - flaky_test_runs_per_job: - type: integer - default: 50 - skip_flaky_tests: - type: boolean - default: false - -commands: - install_extension: - parameters: - pg_major: - description: 'postgres major version to use' - type: integer - steps: - - run: - name: 'Install Extension' - command: | - tar xfv "${CIRCLE_WORKING_DIRECTORY}/install-<< parameters.pg_major >>.tar" --directory / - - configure: - steps: - - run: - name: 'Configure' - command: | - chown -R circleci . - gosu circleci ./configure --without-pg-version-check - - enable_core: - steps: - - run: - name: 'Enable core dumps' - command: | - ulimit -c unlimited - - save_regressions: - steps: - - run: - name: 'Regressions' - command: | - if [ -f "src/test/regress/regression.diffs" ]; then - cat src/test/regress/regression.diffs - exit 1 - fi - when: on_fail - - store_artifacts: - name: 'Save regressions' - path: src/test/regress/regression.diffs - - save_logs_and_results: - steps: - - store_artifacts: - name: 'Save mitmproxy output (failure test specific)' - path: src/test/regress/proxy.output - - store_artifacts: - name: 'Save results' - path: src/test/regress/results/ - - store_artifacts: - name: 'Save coordinator log' - path: src/test/regress/tmp_check/master/log - - store_artifacts: - name: 'Save worker1 log' - path: src/test/regress/tmp_check/worker.57637/log - - store_artifacts: - name: 'Save worker2 log' - path: src/test/regress/tmp_check/worker.57638/log - - stack_trace: - steps: - - run: - name: 'Print stack traces' - command: | - ./ci/print_stack_trace.sh - when: on_fail - - coverage: - parameters: - flags: - description: 'codecov flags' - type: string - steps: - - codecov/upload: - flags: '<< parameters.flags >>' - - run: - name: 'Create codeclimate coverage' - command: | - lcov --directory . --capture --output-file lcov.info - lcov --remove lcov.info -o lcov.info '/usr/*' - sed "s=^SF:$PWD/=SF:=g" -i lcov.info # relative pats are required by codeclimate - mkdir -p /tmp/codeclimate - # We started getting permissions error. This fixes them and since - # weqre not on a multi-user system so this is safe to do. - git config --global --add safe.directory /home/circleci/project - cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/$CIRCLE_JOB.json lcov.info - - persist_to_workspace: - root: /tmp - paths: - - codeclimate/*.json - -jobs: - build: - description: Build the citus extension - parameters: - pg_major: - description: postgres major version building citus for - type: integer - image: - description: docker image to use for the build - type: string - default: citus/extbuilder - image_tag: - description: tag to use for the docker image - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - steps: - - checkout - - run: - name: 'Configure, Build, and Install' - command: | - ./ci/build-citus.sh - - persist_to_workspace: - root: . - paths: - - build-<< parameters.pg_major >>/* - - install-<>.tar - - check-style: - docker: - - image: 'citus/stylechecker:<< pipeline.parameters.style_checker_tools_version >><< pipeline.parameters.image_suffix >>' - steps: - - checkout - - run: - name: 'Check C Style' - command: citus_indent --check - - run: - name: 'Check Python style' - command: black --check . - - run: - name: 'Check Python import order' - command: isort --check . - - run: - name: 'Check Python lints' - command: flake8 . - - run: - name: 'Fix whitespace' - command: ci/editorconfig.sh && git diff --exit-code - - run: - name: 'Remove useless declarations' - command: ci/remove_useless_declarations.sh && git diff --cached --exit-code - - run: - name: 'Normalize test output' - command: ci/normalize_expected.sh && git diff --exit-code - - run: - name: 'Check for C-style comments in migration files' - command: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code - - run: - name: 'Check for comment--cached ns that start with # character in spec files' - command: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code - - run: - name: 'Check for gitignore entries .for source files' - command: ci/fix_gitignore.sh && git diff --exit-code - - run: - name: 'Check for lengths of changelog entries' - command: ci/disallow_long_changelog_entries.sh - - run: - name: 'Check for banned C API usage' - command: ci/banned.h.sh - - run: - name: 'Check for tests missing in schedules' - command: ci/check_all_tests_are_run.sh - - run: - name: 'Check if all CI scripts are actually run' - command: ci/check_all_ci_scripts_are_run.sh - - run: - name: 'Check if all GUCs are sorted alphabetically' - command: ci/check_gucs_are_alphabetically_sorted.sh - - run: - name: 'Check for missing downgrade scripts' - command: ci/check_migration_files.sh - - check-sql-snapshots: - docker: - - image: 'citus/extbuilder:latest' - steps: - - checkout - - run: - name: 'Check Snapshots' - command: ci/check_sql_snapshots.sh - - test-pg-upgrade: - description: Runs postgres upgrade tests - parameters: - old_pg_major: - description: 'postgres major version to use before the upgrade' - type: integer - new_pg_major: - description: 'postgres major version to upgrade to' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/pgupgradetester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.old_pg_major >> - - install_extension: - pg_major: << parameters.new_pg_major >> - - configure - - enable_core - - run: - name: 'Install and test postgres upgrade' - command: | - gosu circleci \ - make -C src/test/regress \ - check-pg-upgrade \ - old-bindir=/usr/lib/postgresql/<< parameters.old_pg_major >>/bin \ - new-bindir=/usr/lib/postgresql/<< parameters.new_pg_major >>/bin - no_output_timeout: 2m - - run: - name: 'Copy pg_upgrade logs for newData dir' - command: | - mkdir -p /tmp/pg_upgrade_newData_logs - if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then - cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs - fi - when: on_fail - - store_artifacts: - name: 'Save pg_upgrade logs for newData dir' - path: /tmp/pg_upgrade_newData_logs - - save_logs_and_results - - save_regressions - - stack_trace - - coverage: - flags: 'test_<< parameters.old_pg_major >>_<< parameters.new_pg_major >>,upgrade' - - test-pytest: - description: Runs pytest based tests - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/failtester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Run pytest' - command: | - gosu circleci \ - make -C src/test/regress check-pytest - no_output_timeout: 2m - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,pytest' - - - test-arbitrary-configs: - description: Runs tests on arbitrary configs - parallelism: 6 - parameters: - pg_major: - description: 'postgres major version to use' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/failtester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - resource_class: xlarge - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Test arbitrary configs' - command: | - TESTS=$(src/test/regress/citus_tests/print_test_names.py | circleci tests split) - # Our test suite expects comma separated values - TESTS=$(echo $TESTS | tr ' ' ',') - # TESTS will contain subset of configs that will be run on a container and we use multiple containers - # to run the test suite - gosu circleci \ - make -C src/test/regress \ - check-arbitrary-configs parallel=4 CONFIGS=$TESTS - no_output_timeout: 2m - - run: - name: 'Show regressions' - command: | - find src/test/regress/tmp_citus_test/ -name "regression*.diffs" -exec cat {} + - lines=$(find src/test/regress/tmp_citus_test/ -name "regression*.diffs" | wc -l) - if [ $lines -ne 0 ]; then - exit 1 - fi - - when: on_fail - - run: - name: 'Copy logfiles' - command: | - mkdir src/test/regress/tmp_citus_test/logfiles - find src/test/regress/tmp_citus_test/ -name "logfile_*" -exec cp -t src/test/regress/tmp_citus_test/logfiles/ {} + - when: on_fail - - store_artifacts: - name: 'Save logfiles' - path: src/test/regress/tmp_citus_test/logfiles - - save_logs_and_results - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,upgrade' - - test-citus-upgrade: - description: Runs citus upgrade tests - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/citusupgradetester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - configure - - enable_core - - run: - name: 'Install and test citus upgrade' - command: | - # run make check-citus-upgrade for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-old-version=${citus_version} \ - citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - - # run make check-citus-upgrade-mixed for all citus versions - # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of - for citus_version in ${CITUS_VERSIONS}; do \ - gosu circleci \ - make -C src/test/regress \ - check-citus-upgrade-mixed \ - citus-old-version=${citus_version} \ - bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ - citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ - citus-post-tar=/home/circleci/project/install-$PG_MAJOR.tar; \ - done; - no_output_timeout: 2m - - save_logs_and_results - - save_regressions - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,upgrade' - - test-query-generator: - description: Expects that the generated queries that are run on distributed and local tables would have the same results - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/failtester - image_tag: - description: 'docker image tag to use' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/regress check-query-generator - no_output_timeout: 5m - - run: - name: 'Show regressions' - command: | - find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" -exec cat {} + - lines=$(find src/test/regress/citus_tests/query_generator/out/ -name "local_dist.diffs" | wc -l) - if [ $lines -ne 0 ]; then - exit 1 - fi - when: on_fail - - run: - name: 'Copy logfiles' - command: | - mkdir src/test/regress/tmp_citus_test/logfiles - find src/test/regress/tmp_citus_test/ -name "logfile_*" -exec cp -t src/test/regress/tmp_citus_test/logfiles/ {} + - when: on_fail - - store_artifacts: - name: 'Save logfiles' - path: src/test/regress/tmp_citus_test/logfiles - - store_artifacts: - name: 'Save ddls' - path: src/test/regress/citus_tests/query_generator/out/ddls.sql - - store_artifacts: - name: 'Save dmls' - path: src/test/regress/citus_tests/query_generator/out/queries.sql - - store_artifacts: - name: 'Save diffs' - path: src/test/regress/citus_tests/query_generator/out/local_dist.diffs - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,querygen' - - test-citus: - description: Runs the common tests of citus - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - make: - description: 'make target' - type: string - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/regress << parameters.make >> - no_output_timeout: 2m - - save_logs_and_results - - save_regressions - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,<< parameters.make >>' - - tap-test-citus: - description: Runs tap tests for citus - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/exttester - image_tag: - description: 'docker image tag to use' - type: string - suite: - description: 'name of the tap test suite to run' - type: string - make: - description: 'make target' - type: string - default: installcheck - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - checkout - - attach_workspace: - at: . - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Run Test' - command: | - gosu circleci make -C src/test/<< parameters.suite >> << parameters.make >> - no_output_timeout: 2m - - store_artifacts: - name: 'Save tap logs' - path: /home/circleci/project/src/test/<< parameters.suite >>/tmp_check/log - - save_logs_and_results - - stack_trace - - coverage: - flags: 'test_<< parameters.pg_major >>,tap_<< parameters.suite >>_<< parameters.make >>' - - check-merge-to-enterprise: - docker: - - image: citus/extbuilder:<< pipeline.parameters.pg14_version >> - working_directory: /home/circleci/project - steps: - - checkout - - run: - command: | - ci/check_enterprise_merge.sh - - ch_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_ch_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - - tpcc_benchmark: - docker: - - image: buildpack-deps:stretch - working_directory: /home/circleci/project - steps: - - checkout - - azure-cli/install - - azure-cli/login-with-service-principal - - run: - command: | - cd ./src/test/hammerdb - sh run_hammerdb.sh citusbot_tpcc_benchmark_rg - name: install dependencies and run ch_benchmark tests - no_output_timeout: 20m - - test-flakyness: - description: Runs a test multiple times to see if it's flaky - parallelism: 32 - parameters: - pg_major: - description: 'postgres major version' - type: integer - image: - description: 'docker image to use as for the tests' - type: string - default: citus/failtester - image_tag: - description: 'docker image tag to use' - type: string - test: - description: 'the test file path that should be run multiple times' - type: string - default: '' - runs: - description: 'number of times that the test should be run in total' - type: integer - default: 8 - skip: - description: 'A flag to bypass flaky test detection.' - type: boolean - default: false - docker: - - image: '<< parameters.image >>:<< parameters.image_tag >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - resource_class: small - steps: - - checkout - - attach_workspace: - at: . - - run: - name: 'Detect regression tests need to be ran' - command: | - skip=<< parameters.skip >> - if [ "$skip" = true ]; then - echo "Skipping flaky test detection." - circleci-agent step halt - fi - - testForDebugging="<< parameters.test >>" - - if [ -z "$testForDebugging" ]; then - detected_changes=$(git diff origin/main... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true)) - tests=${detected_changes} - else - tests=$testForDebugging; - fi - - if [ -z "$tests" ]; then - echo "No test found." - circleci-agent step halt - else - echo "Detected tests " $tests - fi - - echo export tests=\""$tests"\" >> "$BASH_ENV" - source "$BASH_ENV" - - install_extension: - pg_major: << parameters.pg_major >> - - configure - - enable_core - - run: - name: 'Run minimal tests' - command: | - tests_array=($tests) - for test in "${tests_array[@]}" - do - test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/") - gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat << parameters.runs >> --use-base-schedule --use-whole-schedule-line - done - no_output_timeout: 2m - - save_logs_and_results - - save_regressions - - stack_trace - - upload-coverage: - docker: - - image: 'citus/exttester:<< pipeline.parameters.pg15_version >><< pipeline.parameters.image_suffix >>' - working_directory: /home/circleci/project - steps: - - attach_workspace: - at: . - - run: - name: Upload coverage results to Code Climate - command: | - cc-test-reporter sum-coverage codeclimate/*.json -o total.json - cc-test-reporter upload-coverage -i total.json - -workflows: - version: 2 - flaky_test_debugging: - when: << pipeline.parameters.flaky_test >> - jobs: - - build: - name: build-flaky-15 - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - - - test-flakyness: - name: 'test-15_flaky' - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-flaky-15] - test: '<< pipeline.parameters.flaky_test >>' - runs: << pipeline.parameters.flaky_test_runs_per_job >> - - build_and_test: - when: - not: << pipeline.parameters.flaky_test >> - jobs: - - build: - name: build-14 - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - - build: - name: build-15 - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - - build: - name: build-16 - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - - - check-style - - check-sql-snapshots - - - test-citus: &test-citus-14 - name: 'test-14_check-split' - make: check-split - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - requires: [build-14] - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise' - make: check-enterprise - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise-isolation' - make: check-enterprise-isolation - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise-isolation-logicalrep-1' - make: check-enterprise-isolation-logicalrep-1 - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise-isolation-logicalrep-2' - make: check-enterprise-isolation-logicalrep-2 - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise-isolation-logicalrep-3' - make: check-enterprise-isolation-logicalrep-3 - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-enterprise-failure' - image: citus/failtester - make: check-enterprise-failure - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-multi' - make: check-multi - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-multi-1' - make: check-multi-1 - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-mx' - make: check-multi-mx - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-vanilla' - make: check-vanilla - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-isolation' - make: check-isolation - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-operations' - make: check-operations - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-follower-cluster' - make: check-follower-cluster - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-columnar' - make: check-columnar - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-columnar-isolation' - make: check-columnar-isolation - - test-citus: - <<: *test-citus-14 - name: 'test-14_check-failure' - image: citus/failtester - make: check-failure - - - test-citus: &test-citus-15 - name: 'test-15_check-split' - make: check-split - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise' - make: check-enterprise - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise-isolation' - make: check-enterprise-isolation - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise-isolation-logicalrep-1' - make: check-enterprise-isolation-logicalrep-1 - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise-isolation-logicalrep-2' - make: check-enterprise-isolation-logicalrep-2 - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise-isolation-logicalrep-3' - make: check-enterprise-isolation-logicalrep-3 - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-enterprise-failure' - image: citus/failtester - make: check-enterprise-failure - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-multi' - make: check-multi - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-multi-1' - make: check-multi-1 - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-mx' - make: check-multi-mx - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-vanilla' - make: check-vanilla - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-isolation' - make: check-isolation - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-operations' - make: check-operations - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-follower-cluster' - make: check-follower-cluster - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-columnar' - make: check-columnar - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-columnar-isolation' - make: check-columnar-isolation - - test-citus: - <<: *test-citus-15 - name: 'test-15_check-failure' - image: citus/failtester - make: check-failure - - - test-citus: &test-citus-16 - name: 'test-16_check-split' - make: check-split - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - requires: [build-16] - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise' - make: check-enterprise - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise-isolation' - make: check-enterprise-isolation - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise-isolation-logicalrep-1' - make: check-enterprise-isolation-logicalrep-1 - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise-isolation-logicalrep-2' - make: check-enterprise-isolation-logicalrep-2 - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise-isolation-logicalrep-3' - make: check-enterprise-isolation-logicalrep-3 - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-enterprise-failure' - image: citus/failtester - make: check-enterprise-failure - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-multi' - make: check-multi - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-multi-1' - make: check-multi-1 - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-mx' - make: check-multi-mx - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-vanilla' - make: check-vanilla - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-isolation' - make: check-isolation - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-operations' - make: check-operations - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-follower-cluster' - make: check-follower-cluster - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-columnar' - make: check-columnar - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-columnar-isolation' - make: check-columnar-isolation - - test-citus: - <<: *test-citus-16 - name: 'test-16_check-failure' - image: citus/failtester - make: check-failure - - - test-pytest: - name: 'test-14_pytest' - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - requires: [build-14] - - - test-pytest: - name: 'test-15_pytest' - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - - - test-pytest: - name: 'test-16_pytest' - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - requires: [build-16] - - - tap-test-citus: - name: 'test-15_tap-cdc' - suite: cdc - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - - - tap-test-citus: - name: 'test-16_tap-cdc' - suite: cdc - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - requires: [build-16] - - - test-arbitrary-configs: - name: 'test-14_check-arbitrary-configs' - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - requires: [build-14] - - - test-arbitrary-configs: - name: 'test-15_check-arbitrary-configs' - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - - - test-arbitrary-configs: - name: 'test-16_check-arbitrary-configs' - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - requires: [build-16] - - - test-query-generator: - name: 'test-14_check-query-generator' - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - requires: [build-14] - - - test-query-generator: - name: 'test-15_check-query-generator' - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - - - test-query-generator: - name: 'test-16_check-query-generator' - pg_major: 16 - image_tag: '<< pipeline.parameters.pg16_version >>' - requires: [build-16] - - - test-pg-upgrade: - name: 'test-14-15_check-pg-upgrade' - old_pg_major: 14 - new_pg_major: 15 - image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' - requires: [build-14, build-15] - - - test-pg-upgrade: - name: 'test-15-16_check-pg-upgrade' - old_pg_major: 15 - new_pg_major: 16 - image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' - requires: [build-15, build-16] - - - test-pg-upgrade: - name: 'test-14-16_check-pg-upgrade' - old_pg_major: 14 - new_pg_major: 16 - image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' - requires: [build-14, build-16] - - - test-citus-upgrade: - name: test-14_check-citus-upgrade - pg_major: 14 - image_tag: '<< pipeline.parameters.pg14_version >>' - requires: [build-14] - - - upload-coverage: - requires: - - test-14_check-multi - - test-14_check-multi-1 - - test-14_check-mx - - test-14_check-vanilla - - test-14_check-isolation - - test-14_check-operations - - test-14_check-follower-cluster - - test-14_check-columnar - - test-14_check-columnar-isolation - - test-14_check-failure - - test-14_check-enterprise - - test-14_check-enterprise-isolation - - test-14_check-enterprise-isolation-logicalrep-1 - - test-14_check-enterprise-isolation-logicalrep-2 - - test-14_check-enterprise-isolation-logicalrep-3 - - test-14_check-enterprise-failure - - test-14_check-split - - test-14_check-arbitrary-configs - - test-14_check-query-generator - - test-15_check-multi - - test-15_check-multi-1 - - test-15_check-mx - - test-15_check-vanilla - - test-15_check-isolation - - test-15_check-operations - - test-15_check-follower-cluster - - test-15_check-columnar - - test-15_check-columnar-isolation - - test-15_check-failure - - test-15_check-enterprise - - test-15_check-enterprise-isolation - - test-15_check-enterprise-isolation-logicalrep-1 - - test-15_check-enterprise-isolation-logicalrep-2 - - test-15_check-enterprise-isolation-logicalrep-3 - - test-15_check-enterprise-failure - - test-15_check-split - - test-15_check-arbitrary-configs - - test-15_check-query-generator - - test-16_check-multi - - test-16_check-multi-1 - - test-16_check-mx - - test-16_check-vanilla - - test-16_check-isolation - - test-16_check-operations - - test-16_check-follower-cluster - - test-16_check-columnar - - test-16_check-columnar-isolation - - test-16_check-failure - - test-16_check-enterprise - - test-16_check-enterprise-isolation - - test-16_check-enterprise-isolation-logicalrep-1 - - test-16_check-enterprise-isolation-logicalrep-2 - - test-16_check-enterprise-isolation-logicalrep-3 - - test-16_check-enterprise-failure - - test-16_check-split - - test-16_check-arbitrary-configs - - test-16_check-query-generator - - test-14-15_check-pg-upgrade - - test-15-16_check-pg-upgrade - - test-14-16_check-pg-upgrade - - test-14_check-citus-upgrade - - - ch_benchmark: - requires: [build-14] - filters: - branches: - only: - - /ch_benchmark\/.*/ # match with ch_benchmark/ prefix - - tpcc_benchmark: - requires: [build-14] - filters: - branches: - only: - - /tpcc_benchmark\/.*/ # match with tpcc_benchmark/ prefix - - test-flakyness: - name: 'test-15_flaky' - pg_major: 15 - image_tag: '<< pipeline.parameters.pg15_version >>' - requires: [build-15] - skip: << pipeline.parameters.skip_flaky_tests >> diff --git a/.github/actions/parallelization/action.yml b/.github/actions/parallelization/action.yml new file mode 100644 index 000000000..1f7d00202 --- /dev/null +++ b/.github/actions/parallelization/action.yml @@ -0,0 +1,23 @@ +name: 'Parallelization matrix' +inputs: + count: + required: false + default: 32 +outputs: + json: + value: ${{ steps.generate_matrix.outputs.json }} +runs: + using: "composite" + steps: + - name: Generate parallelization matrix + id: generate_matrix + shell: bash + run: |- + json_array="{\"include\": [" + for ((i = 1; i <= ${{ inputs.count }}; i++)); do + json_array+="{\"id\":\"$i\"}," + done + json_array=${json_array%,} + json_array+=" ]}" + echo "json=$json_array" >> "$GITHUB_OUTPUT" + echo "json=$json_array" diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml new file mode 100644 index 000000000..0f238835d --- /dev/null +++ b/.github/actions/save_logs_and_results/action.yml @@ -0,0 +1,38 @@ +name: save_logs_and_results +inputs: + folder: + required: false + default: "log" +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.1 + name: Upload logs + with: + name: ${{ inputs.folder }} + if-no-files-found: ignore + path: | + src/test/**/proxy.output + src/test/**/results/ + src/test/**/tmp_check/master/log + src/test/**/tmp_check/worker.57638/log + src/test/**/tmp_check/worker.57637/log + src/test/**/*.diffs + src/test/**/out/ddls.sql + src/test/**/out/queries.sql + src/test/**/logfile_* + /tmp/pg_upgrade_newData_logs + - name: Publish regression.diffs + run: |- + diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)" + if ! [ -z "$diffs" ]; then + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo -E "$diffs" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo -E $diffs + fi + shell: bash + - name: Print stack traces + run: "./ci/print_stack_trace.sh" + if: failure() + shell: bash diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml new file mode 100644 index 000000000..96b408e7e --- /dev/null +++ b/.github/actions/setup_extension/action.yml @@ -0,0 +1,35 @@ +name: setup_extension +inputs: + pg_major: + required: false + skip_installation: + required: false + default: false + type: boolean +runs: + using: composite + steps: + - name: Expose $PG_MAJOR to Github Env + run: |- + if [ -z "${{ inputs.pg_major }}" ]; then + echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + else + echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV + fi + shell: bash + - uses: actions/download-artifact@v3.0.1 + with: + name: build-${{ env.PG_MAJOR }} + - name: Install Extension + if: ${{ inputs.skip_installation == 'false' }} + run: tar xfv "install-$PG_MAJOR.tar" --directory / + shell: bash + - name: Configure + run: |- + chown -R circleci . + git config --global --add safe.directory ${GITHUB_WORKSPACE} + gosu circleci ./configure --without-pg-version-check + shell: bash + - name: Enable core dumps + run: ulimit -c unlimited + shell: bash diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml new file mode 100644 index 000000000..0b5f581a6 --- /dev/null +++ b/.github/actions/upload_coverage/action.yml @@ -0,0 +1,27 @@ +name: coverage +inputs: + flags: + required: false + codecov_token: + required: true +runs: + using: composite + steps: + - uses: codecov/codecov-action@v3 + with: + flags: ${{ inputs.flags }} + token: ${{ inputs.codecov_token }} + verbose: true + gcov: true + - name: Create codeclimate coverage + run: |- + lcov --directory . --capture --output-file lcov.info + lcov --remove lcov.info -o lcov.info '/usr/*' + sed "s=^SF:$PWD/=SF:=g" -i lcov.info # relative pats are required by codeclimate + mkdir -p /tmp/codeclimate + cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + path: "/tmp/codeclimate/*.json" + name: codeclimate diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 000000000..491965b6c --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,505 @@ +name: Build & Test +run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +on: + workflow_dispatch: + inputs: + skip_test_flakyness: + required: false + default: false + type: boolean + pull_request: + types: [opened, reopened,synchronize] +jobs: + # Since GHA does not interpolate env varibles in matrix context, we need to + # define them in a separate job and use them in other jobs. + params: + runs-on: ubuntu-latest + name: Initialize parameters + outputs: + build_image_name: "citus/extbuilder" + test_image_name: "citus/exttester" + citusupgrade_image_name: "citus/citusupgradetester" + fail_test_image_name: "citus/failtester" + pgupgrade_image_name: "citus/pgupgradetester" + style_checker_image_name: "citus/stylechecker" + style_checker_tools_version: "0.8.18" + image_suffix: "-v9d71045" + pg14_version: '{ "major": "14", "full": "14.9" }' + pg15_version: '{ "major": "15", "full": "15.4" }' + pg16_version: '{ "major": "16", "full": "16.0" }' + upgrade_pg_versions: "14.9-15.4-16.0" + steps: + # Since GHA jobs needs at least one step we use a noop step here. + - name: Set up parameters + run: echo 'noop' + check-sql-snapshots: + needs: params + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.build_image_name }}:latest + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + ci/check_sql_snapshots.sh + check-style: + needs: params + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }} + steps: + - name: Check Snapshots + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Check C Style + run: citus_indent --check + - name: Check Python style + run: black --check . + - name: Check Python import order + run: isort --check . + - name: Check Python lints + run: flake8 . + - name: Fix whitespace + run: ci/editorconfig.sh && git diff --exit-code + - name: Remove useless declarations + run: ci/remove_useless_declarations.sh && git diff --cached --exit-code + - name: Normalize test output + run: ci/normalize_expected.sh && git diff --exit-code + - name: Check for C-style comments in migration files + run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code + - name: 'Check for comment--cached ns that start with # character in spec files' + run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code + - name: Check for gitignore entries .for source files + run: ci/fix_gitignore.sh && git diff --exit-code + - name: Check for lengths of changelog entries + run: ci/disallow_long_changelog_entries.sh + - name: Check for banned C API usage + run: ci/banned.h.sh + - name: Check for tests missing in schedules + run: ci/check_all_tests_are_run.sh + - name: Check if all CI scripts are actually run + run: ci/check_all_ci_scripts_are_run.sh + - name: Check if all GUCs are sorted alphabetically + run: ci/check_gucs_are_alphabetically_sorted.sh + - name: Check for missing downgrade scripts + run: ci/check_migration_files.sh + build: + needs: params + name: Build for PG${{ fromJson(matrix.pg_version).major }} + strategy: + fail-fast: false + matrix: + image_name: + - ${{ needs.params.outputs.build_image_name }} + image_suffix: + - ${{ needs.params.outputs.image_suffix}} + pg_version: + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Expose $PG_MAJOR to Github Env + run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + shell: bash + - name: Build + run: "./ci/build-citus.sh" + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + test-citus: + name: PG${{ fromJson(matrix.pg_version).major }} - ${{ matrix.make }} + strategy: + fail-fast: false + matrix: + suite: + - regress + image_name: + - ${{ needs.params.outputs.test_image_name }} + pg_version: + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} + make: + - check-split + - check-multi + - check-multi-1 + - check-multi-mx + - check-vanilla + - check-isolation + - check-operations + - check-follower-cluster + - check-columnar + - check-columnar-isolation + - check-enterprise + - check-enterprise-isolation + - check-enterprise-isolation-logicalrep-1 + - check-enterprise-isolation-logicalrep-2 + - check-enterprise-isolation-logicalrep-3 + include: + - make: check-failure + pg_version: ${{ needs.params.outputs.pg14_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg15_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg16_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ needs.params.outputs.pg14_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ needs.params.outputs.pg15_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ needs.params.outputs.pg16_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ needs.params.outputs.pg14_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ needs.params.outputs.pg15_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ needs.params.outputs.pg16_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: installcheck + suite: cdc + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg15_version }} + - make: installcheck + suite: cdc + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg16_version }} + - make: check-query-generator + pg_version: ${{ needs.params.outputs.pg14_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ needs.params.outputs.pg15_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ needs.params.outputs.pg16_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} + runs-on: ubuntu-20.04 + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root --dns=8.8.8.8 + # Due to Github creates a default network for each job, we need to use + # --dns= to have similar DNS settings as our other CI systems or local + # machines. Otherwise, we may see different results. + needs: + - params + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Run Test + run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} + timeout-minutes: 20 + - uses: "./.github/actions/save_logs_and_results" + if: always() + with: + folder: ${{ fromJson(matrix.pg_version).major }}_${{ matrix.make }} + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.PG_MAJOR }}_${{ matrix.suite }}_${{ matrix.make }} + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-arbitrary-configs: + name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }} + runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] + container: + image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root + needs: + - params + - build + strategy: + fail-fast: false + matrix: + image_name: + - ${{ needs.params.outputs.fail_test_image_name }} + pg_version: + - ${{ needs.params.outputs.pg14_version }} + - ${{ needs.params.outputs.pg15_version }} + - ${{ needs.params.outputs.pg16_version }} + parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Test arbitrary configs + run: |- + # we use parallel jobs to split the tests into 6 parts and run them in parallel + # the script below extracts the tests for the current job + N=6 # Total number of jobs (see matrix.parallel) + X=${{ matrix.parallel }} # Current job number + TESTS=$(src/test/regress/citus_tests/print_test_names.py | + tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{ + split("", parts) + for (i = 1; i <= NF; i++) { + parts[i % N] = parts[i % N] $i "," + } + print substr(parts[X], 1, length(parts[X])-1) + }') + echo $TESTS + gosu circleci \ + make -C src/test/regress \ + check-arbitrary-configs parallel=4 CONFIGS=$TESTS + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-pg-upgrade: + name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}" + options: --user root + needs: + - params + - build + strategy: + fail-fast: false + matrix: + include: + - old_pg_major: 14 + new_pg_major: 15 + - old_pg_major: 15 + new_pg_major: 16 + - old_pg_major: 14 + new_pg_major: 16 + env: + old_pg_major: ${{ matrix.old_pg_major }} + new_pg_major: ${{ matrix.new_pg_major }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.old_pg_major }}" + - uses: "./.github/actions/setup_extension" + with: + pg_major: "${{ env.new_pg_major }}" + - name: Install and test postgres upgrade + run: |- + gosu circleci \ + make -C src/test/regress \ + check-pg-upgrade \ + old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ + new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin + - name: Copy pg_upgrade logs for newData dir + run: |- + mkdir -p /tmp/pg_upgrade_newData_logs + if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then + cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs + fi + if: failure() + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + test-citus-upgrade: + name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade + runs-on: ubuntu-20.04 + container: + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}" + options: --user root + needs: + - params + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + with: + skip_installation: true + - name: Install and test citus upgrade + run: |- + # run make check-citus-upgrade for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-old-version=${citus_version} \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + # run make check-citus-upgrade-mixed for all citus versions + # the image has ${CITUS_VERSIONS} set with all verions it contains the binaries of + for citus_version in ${CITUS_VERSIONS}; do \ + gosu circleci \ + make -C src/test/regress \ + check-citus-upgrade-mixed \ + citus-old-version=${citus_version} \ + bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ + citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ + citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ + done; + - uses: "./.github/actions/save_logs_and_results" + if: always() + - uses: "./.github/actions/upload_coverage" + if: always() + with: + flags: ${{ env.pg_major }}_upgrade + codecov_token: ${{ secrets.CODECOV_TOKEN }} + upload-coverage: + if: always() + env: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }} + needs: + - params + - test-citus + - test-arbitrary-configs + - test-citus-upgrade + - test-pg-upgrade + steps: + - uses: actions/download-artifact@v3.0.1 + with: + name: "codeclimate" + path: "codeclimate" + - name: Upload coverage results to Code Climate + run: |- + cc-test-reporter sum-coverage codeclimate/*.json -o total.json + cc-test-reporter upload-coverage -i total.json + ch_benchmark: + name: CH Benchmark + if: startsWith(github.ref, 'refs/heads/ch_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run ch_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_ch_benchmark_rg + tpcc_benchmark: + name: TPCC Benchmark + if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/') + runs-on: ubuntu-20.04 + needs: + - build + steps: + - uses: actions/checkout@v3.5.0 + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: install dependencies and run tpcc_benchmark tests + uses: azure/CLI@v1 + with: + inlineScript: | + cd ./src/test/hammerdb + chmod +x run_hammerdb.sh + run_hammerdb.sh citusbot_tpcc_benchmark_rg + prepare_parallelization_matrix_32: + name: Parallel 32 + if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} + needs: test-flakyness-pre + runs-on: ubuntu-20.04 + outputs: + json: ${{ steps.parallelization.outputs.json }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/parallelization" + id: parallelization + with: + count: 32 + test-flakyness-pre: + name: Detect regression tests need to be ran + if: ${{ !inputs.skip_test_flakyness }}} + runs-on: ubuntu-20.04 + needs: build + outputs: + tests: ${{ steps.detect-regression-tests.outputs.tests }} + steps: + - uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Detect regression tests need to be ran + id: detect-regression-tests + run: |- + detected_changes=$(git diff origin/release-12.1... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true)) + tests=${detected_changes} + if [ -z "$tests" ]; then + echo "No test found." + else + echo "Detected tests " $tests + fi + echo tests="$tests" >> "$GITHUB_OUTPUT" + test-flakyness: + if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} + name: Test flakyness + runs-on: ubuntu-20.04 + container: + image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} + options: --user root + env: + runs: 8 + needs: + - params + - build + - test-flakyness-pre + - prepare_parallelization_matrix_32 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: actions/download-artifact@v3.0.1 + - uses: "./.github/actions/setup_extension" + - name: Run minimal tests + run: |- + tests="${{ needs.test-flakyness-pre.outputs.tests }}" + tests_array=($tests) + for test in "${tests_array[@]}" + do + test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/") + gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line + done + shell: bash + - uses: "./.github/actions/save_logs_and_results" + if: always() diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml new file mode 100644 index 000000000..a666c1cd5 --- /dev/null +++ b/.github/workflows/flaky_test_debugging.yml @@ -0,0 +1,79 @@ +name: Flaky test debugging +run-name: Flaky test debugging - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }}) +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +on: + workflow_dispatch: + inputs: + flaky_test: + required: true + type: string + description: Test to run + flaky_test_runs_per_job: + required: false + default: 8 + type: number + description: Number of times to run the test + flaky_test_parallel_jobs: + required: false + default: 32 + type: number + description: Number of parallel jobs to run +jobs: + build: + name: Build Citus + runs-on: ubuntu-latest + container: + image: ${{ vars.build_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} + options: --user root + steps: + - uses: actions/checkout@v3.5.0 + - name: Configure, Build, and Install + run: | + echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV + ./ci/build-citus.sh + shell: bash + - uses: actions/upload-artifact@v3.1.1 + with: + name: build-${{ env.PG_MAJOR }} + path: |- + ./build-${{ env.PG_MAJOR }}/* + ./install-${{ env.PG_MAJOR }}.tar + prepare_parallelization_matrix: + name: Prepare parallelization matrix + runs-on: ubuntu-latest + outputs: + json: ${{ steps.parallelization.outputs.json }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/parallelization" + id: parallelization + with: + count: ${{ inputs.flaky_test_parallel_jobs }} + test_flakyness: + name: Test flakyness + runs-on: ubuntu-latest + container: + image: ${{ vars.fail_test_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} + options: --user root + needs: + [build, prepare_parallelization_matrix] + env: + test: "${{ inputs.flaky_test }}" + runs: "${{ inputs.flaky_test_runs_per_job }}" + skip: false + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} + steps: + - uses: actions/checkout@v3.5.0 + - uses: "./.github/actions/setup_extension" + - name: Run minimal tests + run: |- + gosu circleci src/test/regress/citus_tests/run_test.py ${{ env.test }} --repeat ${{ env.runs }} --use-base-schedule --use-whole-schedule-line + shell: bash + - uses: "./.github/actions/save_logs_and_results" + if: always() + with: + folder: ${{ matrix.id }} diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 356a590a4..23ed39bba 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -24,14 +24,16 @@ jobs: - name: Get Postgres Versions id: get-postgres-versions run: | - # Postgres versions are stored in .circleci/config.yml file in "build-[pg-version] format. Below command - # extracts the versions and get the unique values. - pg_versions=`grep -Eo 'build-[[:digit:]]{2}' .circleci/config.yml|sed -e "s/^build-//"|sort|uniq|tr '\n' ','| head -c -1` + set -euxo pipefail + # Postgres versions are stored in .github/workflows/build_and_test.yml + # file in json strings with major and full keys. + # Below command extracts the versions and get the unique values. + pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[0-9.]+"' | sed -E 's/"major": "([0-9]+)", "full": "([0-9.]+)"/\1/g' | sort | uniq | tr '\n', ',') pg_versions_array="[ ${pg_versions} ]" echo "Supported PG Versions: ${pg_versions_array}" # Below line is needed to set the output variable to be used in the next job echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT - + shell: bash rpm_build_tests: name: rpm_build_tests needs: get_postgres_versions_from_file @@ -101,7 +103,7 @@ jobs: echo "Postgres version: ${POSTGRES_VERSION}" ## Install required packages to execute packaging tools for rpm based distros - yum install python3-pip python3-devel postgresql-devel -y + yum install python3-pip python3-devel postgresql-devel -y || true python3 -m pip install wheel ./.github/packaging/validate_build_output.sh "rpm" diff --git a/ci/build-citus.sh b/ci/build-citus.sh index 49f92e691..678fd515c 100755 --- a/ci/build-citus.sh +++ b/ci/build-citus.sh @@ -15,9 +15,6 @@ PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} codename=${VERSION#*(} codename=${codename%)*} -# get project from argument -project="${CIRCLE_PROJECT_REPONAME}" - # we'll do everything with absolute paths basedir="$(pwd)" @@ -28,7 +25,7 @@ build_ext() { pg_major="$1" builddir="${basedir}/build-${pg_major}" - echo "Beginning build of ${project} for PostgreSQL ${pg_major}..." >&2 + echo "Beginning build for PostgreSQL ${pg_major}..." >&2 # do everything in a subdirectory to avoid clutter in current directory mkdir -p "${builddir}" && cd "${builddir}" diff --git a/ci/check_all_ci_scripts_are_run.sh b/ci/check_all_ci_scripts_are_run.sh index 0b7abb3e3..12516f793 100755 --- a/ci/check_all_ci_scripts_are_run.sh +++ b/ci/check_all_ci_scripts_are_run.sh @@ -14,8 +14,8 @@ ci_scripts=$( grep -v -E '^(ci_helpers.sh|fix_style.sh)$' ) for script in $ci_scripts; do - if ! grep "\\bci/$script\\b" .circleci/config.yml > /dev/null; then - echo "ERROR: CI script with name \"$script\" is not actually used in .circleci/config.yml" + if ! grep "\\bci/$script\\b" -r .github > /dev/null; then + echo "ERROR: CI script with name \"$script\" is not actually used in .github folder" exit 1 fi if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then diff --git a/ci/check_enterprise_merge.sh b/ci/check_enterprise_merge.sh deleted file mode 100755 index d29ffcad8..000000000 --- a/ci/check_enterprise_merge.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Testing this script locally requires you to set the following environment -# variables: -# CIRCLE_BRANCH, GIT_USERNAME and GIT_TOKEN - -# fail if trying to reference a variable that is not set. -set -u -# exit immediately if a command fails -set -e -# Fail on pipe failures -set -o pipefail - -PR_BRANCH="${CIRCLE_BRANCH}" -ENTERPRISE_REMOTE="https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/citus-enterprise" - -# shellcheck disable=SC1091 -source ci/ci_helpers.sh - -# List executed commands. This is done so debugging this script is easier when -# it fails. It's explicitly done after git remote add so username and password -# are not shown in CI output (even though it's also filtered out by CircleCI) -set -x - -check_compile () { - echo "INFO: checking if merged code can be compiled" - ./configure --without-libcurl - make -j10 -} - -# Clone current git repo (which should be community) to a temporary working -# directory and go there -GIT_DIR_ROOT="$(git rev-parse --show-toplevel)" -TMP_GIT_DIR="$(mktemp --directory -t citus-merge-check.XXXXXXXXX)" -git clone "$GIT_DIR_ROOT" "$TMP_GIT_DIR" -cd "$TMP_GIT_DIR" - -# Fails in CI without this -git config user.email "citus-bot@microsoft.com" -git config user.name "citus bot" - -# Disable "set -x" temporarily, because $ENTERPRISE_REMOTE contains passwords -{ set +x ; } 2> /dev/null -git remote add enterprise "$ENTERPRISE_REMOTE" -set -x - -git remote set-url --push enterprise no-pushing - -# Fetch enterprise-master -git fetch enterprise enterprise-master - - -git checkout "enterprise/enterprise-master" - -if git merge --no-commit "origin/$PR_BRANCH"; then - echo "INFO: community PR branch could be merged into enterprise-master" - # check that we can compile after the merge - if check_compile; then - exit 0 - fi - - echo "WARN: Failed to compile after community PR branch was merged into enterprise" -fi - -# undo partial merge -git merge --abort - -# If we have a conflict on enterprise merge on the master branch, we have a problem. -# Provide an error message to indicate that enterprise merge is needed to fix this check. -if [[ $PR_BRANCH = master ]]; then - echo "ERROR: Master branch has merge conflicts with enterprise-master." - echo "Try re-running this CI job after merging your changes into enterprise-master." - exit 1 -fi - -if ! git fetch enterprise "$PR_BRANCH" ; then - echo "ERROR: enterprise/$PR_BRANCH was not found and community PR branch could not be merged into enterprise-master" - exit 1 -fi - -# Show the top commit of the enterprise PR branch to make debugging easier -git log -n 1 "enterprise/$PR_BRANCH" - -# Check that this branch contains the top commit of the current community PR -# branch. If it does not it means it's not up to date with the current PR, so -# the enterprise branch should be updated. -if ! git merge-base --is-ancestor "origin/$PR_BRANCH" "enterprise/$PR_BRANCH" ; then - echo "ERROR: enterprise/$PR_BRANCH is not up to date with community PR branch" - exit 1 -fi - -# Now check if we can merge the enterprise PR into enterprise-master without -# issues. -git merge --no-commit "enterprise/$PR_BRANCH" -# check that we can compile after the merge -check_compile From b421479d46d318f4f8a8fcd2b3b5411f6a635f21 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Thu, 9 Nov 2023 17:19:28 +0300 Subject: [PATCH 007/155] Add changelog entries for 12.1.1 (#7332) Co-authored-by: Onur Tirtir (cherry picked from commit 92228b279a15fe31db3b8017dfd83b55086df9cf) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02fc91d04..686e78dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +### citus v12.1.1 (November 9, 2023) ### + +* Fixes leaking of memory and memory contexts in Citus foreign key cache + (#7219) + +* Makes sure to disallow creating a replicated distributed table concurrently + (#7236) + ### citus v12.1.0 (September 12, 2023) ### * Adds support for PostgreSQL 16.0 (#7173) From 2c630eca50595c56c79417d58897ffae9fe8b326 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Thu, 9 Nov 2023 17:04:37 +0300 Subject: [PATCH 008/155] Bump Citus version to 12.1.1 --- configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 723b48e26..35c035ea1 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.0. +# Generated by GNU Autoconf 2.69 for Citus 12.1.1. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.0' -PACKAGE_STRING='Citus 12.1.0' +PACKAGE_VERSION='12.1.1' +PACKAGE_STRING='Citus 12.1.1' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.0 to adapt to many kinds of systems. +\`configure' configures Citus 12.1.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.0:";; + short | recursive ) echo "Configuration of Citus 12.1.1:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.0 +Citus configure 12.1.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.0, which was +It was created by Citus $as_me 12.1.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.0, which was +This file was extended by Citus $as_me 12.1.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.0 +Citus config.status 12.1.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 0642cbd75..957d4f5f1 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.0]) +AC_INIT([Citus], [12.1.1]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 67ec1d78b..566cd08e6 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.0 + 12.1.1 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 4c110faf1b9e16aa04f42392a26110c5573bd619 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Thu, 16 Nov 2023 14:12:17 +0300 Subject: [PATCH 009/155] Fix wrong PR links in changelog (#7350) When preparing changelog for 12.1.1 release, I accidentally swapped the PR numbers for the two commits. This commit fixes the changelog to point to the correct PRs. (cherry picked from commit 5efd3f181aa9a88de6d29d4bc74f4adedfe93d0e) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 686e78dd1..8d979c104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ ### citus v12.1.1 (November 9, 2023) ### * Fixes leaking of memory and memory contexts in Citus foreign key cache - (#7219) + (#7236) * Makes sure to disallow creating a replicated distributed table concurrently - (#7236) + (#7219) ### citus v12.1.0 (September 12, 2023) ### From a945971f48f9e9e9c8e714a29949be95473f9aae Mon Sep 17 00:00:00 2001 From: Teja Mupparti Date: Tue, 12 Dec 2023 14:28:43 -0800 Subject: [PATCH 010/155] Fix the incorrect column count after ALTER TABLE, this fixes the bug #7378 (please read the analysis in the bug for more information) (cherry picked from commit 00068e07c53644087e153d8837dedd2b86732c26) --- .github/workflows/build_and_test.yml | 7 +- .../distributed/deparser/ruleutils_14.c | 11 ++- .../distributed/deparser/ruleutils_15.c | 11 ++- .../distributed/deparser/ruleutils_16.c | 11 ++- .../distributed/utils/citus_nodefuncs.c | 12 +++- ...reign_key_to_reference_shard_rebalance.out | 1 + .../expected/multi_alter_table_statements.out | 71 +++++++++++++++++++ ...reign_key_to_reference_shard_rebalance.sql | 1 + .../sql/multi_alter_table_statements.sql | 26 +++++++ 9 files changed, 142 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 491965b6c..f7e125cf1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -469,9 +469,12 @@ jobs: else echo "Detected tests " $tests fi - echo tests="$tests" >> "$GITHUB_OUTPUT" + + echo 'tests<> $GITHUB_OUTPUT + echo "$tests" >> "$GITHUB_OUTPUT" + echo 'EOF' >> $GITHUB_OUTPUT test-flakyness: - if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} + if: false name: Test flakyness runs-on: ubuntu-20.04 container: diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 6ab124537..e765f1103 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -1526,8 +1526,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING - while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) - i++; + for (int col_index = 0; col_index < colinfo->num_cols; col_index++) + { + /* + * In the above processing-loops, "i" advances only if + * the column is not new, check if this is a new column. + */ + if (colinfo->is_new_col[col_index]) + i++; + } Assert(i == colinfo->num_cols); Assert(j == nnewcolumns); #endif diff --git a/src/backend/distributed/deparser/ruleutils_15.c b/src/backend/distributed/deparser/ruleutils_15.c index 755e0f4cd..3755e13f1 100644 --- a/src/backend/distributed/deparser/ruleutils_15.c +++ b/src/backend/distributed/deparser/ruleutils_15.c @@ -1563,8 +1563,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING - while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) - i++; + for (int col_index = 0; col_index < colinfo->num_cols; col_index++) + { + /* + * In the above processing-loops, "i" advances only if + * the column is not new, check if this is a new column. + */ + if (colinfo->is_new_col[col_index]) + i++; + } Assert(i == colinfo->num_cols); Assert(j == nnewcolumns); #endif diff --git a/src/backend/distributed/deparser/ruleutils_16.c b/src/backend/distributed/deparser/ruleutils_16.c index 31e8823b1..f6f0d9766 100644 --- a/src/backend/distributed/deparser/ruleutils_16.c +++ b/src/backend/distributed/deparser/ruleutils_16.c @@ -1580,8 +1580,15 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING - while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) - i++; + for (int col_index = 0; col_index < colinfo->num_cols; col_index++) + { + /* + * In the above processing-loops, "i" advances only if + * the column is not new, check if this is a new column. + */ + if (colinfo->is_new_col[col_index]) + i++; + } Assert(i == colinfo->num_cols); Assert(j == nnewcolumns); #endif diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index aee1ff48a..735b9e15e 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -141,7 +141,17 @@ SetRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSch fauxFunction->funcexpr = (Node *) fauxFuncExpr; /* set the column count to pass ruleutils checks, not used elsewhere */ - fauxFunction->funccolcount = list_length(rte->eref->colnames); + if (rte->relid != 0) + { + Relation rel = RelationIdGetRelation(rte->relid); + fauxFunction->funccolcount = RelationGetNumberOfAttributes(rel); + RelationClose(rel); + } + else + { + fauxFunction->funccolcount = list_length(rte->eref->colnames); + } + fauxFunction->funccolnames = funcColumnNames; fauxFunction->funccoltypes = funcColumnTypes; fauxFunction->funccoltypmods = funcColumnTypeMods; diff --git a/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out b/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out index 2dc329153..bae1895f9 100644 --- a/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out +++ b/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out @@ -210,6 +210,7 @@ select create_distributed_table('partitioned_tbl_with_fkey','x'); create table partition_1_with_fkey partition of partitioned_tbl_with_fkey for values from ('2022-01-01') to ('2022-12-31'); create table partition_2_with_fkey partition of partitioned_tbl_with_fkey for values from ('2023-01-01') to ('2023-12-31'); +create table partition_3_with_fkey partition of partitioned_tbl_with_fkey for values from ('2024-01-01') to ('2024-12-31'); insert into partitioned_tbl_with_fkey (x,y) select s,s%10 from generate_series(1,100) s; ALTER TABLE partitioned_tbl_with_fkey ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (y) REFERENCES ref_table_with_fkey(id); WITH shardid AS (SELECT shardid FROM pg_dist_shard where logicalrelid = 'partitioned_tbl_with_fkey'::regclass ORDER BY shardid LIMIT 1) diff --git a/src/test/regress/expected/multi_alter_table_statements.out b/src/test/regress/expected/multi_alter_table_statements.out index c24927504..461465393 100644 --- a/src/test/regress/expected/multi_alter_table_statements.out +++ b/src/test/regress/expected/multi_alter_table_statements.out @@ -1349,6 +1349,77 @@ SELECT pg_identify_object_as_address(classid, objid, objsubid) from pg_catalog.p (schema,{test_schema_for_sequence_propagation},{}) (1 row) +-- Bug: https://github.com/citusdata/citus/issues/7378 +-- Create a reference table +CREATE TABLE tbl_ref_mats(row_id integer primary key); +INSERT INTO tbl_ref_mats VALUES (1), (2); +SELECT create_reference_table('tbl_ref_mats'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_alter_table_statements.tbl_ref_mats$$) + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Create a distributed table +CREATE TABLE tbl_dist_mats(series_id integer); +INSERT INTO tbl_dist_mats VALUES (1), (1), (2), (2); +SELECT create_distributed_table('tbl_dist_mats', 'series_id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$multi_alter_table_statements.tbl_dist_mats$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Create a view that joins the distributed table with the reference table on the distribution key. +CREATE VIEW vw_citus_views as +SELECT d.series_id FROM tbl_dist_mats d JOIN tbl_ref_mats r ON d.series_id = r.row_id; +-- The view initially works fine +SELECT * FROM vw_citus_views ORDER BY 1; + series_id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +-- Now, alter the table +ALTER TABLE tbl_ref_mats ADD COLUMN category1 varchar(50); +SELECT * FROM vw_citus_views ORDER BY 1; + series_id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +ALTER TABLE tbl_ref_mats ADD COLUMN category2 varchar(50); +SELECT * FROM vw_citus_views ORDER BY 1; + series_id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +ALTER TABLE tbl_ref_mats DROP COLUMN category1; +SELECT * FROM vw_citus_views ORDER BY 1; + series_id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + SET client_min_messages TO WARNING; DROP SCHEMA test_schema_for_sequence_propagation CASCADE; DROP TABLE table_without_sequence; diff --git a/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql b/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql index 7791001e0..0b9826cb7 100644 --- a/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql +++ b/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql @@ -84,6 +84,7 @@ create table partitioned_tbl_with_fkey (x int, y int, t timestamptz default now( select create_distributed_table('partitioned_tbl_with_fkey','x'); create table partition_1_with_fkey partition of partitioned_tbl_with_fkey for values from ('2022-01-01') to ('2022-12-31'); create table partition_2_with_fkey partition of partitioned_tbl_with_fkey for values from ('2023-01-01') to ('2023-12-31'); +create table partition_3_with_fkey partition of partitioned_tbl_with_fkey for values from ('2024-01-01') to ('2024-12-31'); insert into partitioned_tbl_with_fkey (x,y) select s,s%10 from generate_series(1,100) s; ALTER TABLE partitioned_tbl_with_fkey ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (y) REFERENCES ref_table_with_fkey(id); diff --git a/src/test/regress/sql/multi_alter_table_statements.sql b/src/test/regress/sql/multi_alter_table_statements.sql index 10e52cb37..7e2a8f850 100644 --- a/src/test/regress/sql/multi_alter_table_statements.sql +++ b/src/test/regress/sql/multi_alter_table_statements.sql @@ -727,6 +727,32 @@ ALTER TABLE table_without_sequence ADD COLUMN x BIGINT DEFAULT nextval('test_sch SELECT pg_identify_object_as_address(classid, objid, objsubid) from pg_catalog.pg_dist_object WHERE objid IN ('test_schema_for_sequence_propagation.seq_10'::regclass); SELECT pg_identify_object_as_address(classid, objid, objsubid) from pg_catalog.pg_dist_object WHERE objid IN ('test_schema_for_sequence_propagation'::regnamespace); +-- Bug: https://github.com/citusdata/citus/issues/7378 + +-- Create a reference table +CREATE TABLE tbl_ref_mats(row_id integer primary key); +INSERT INTO tbl_ref_mats VALUES (1), (2); +SELECT create_reference_table('tbl_ref_mats'); + +-- Create a distributed table +CREATE TABLE tbl_dist_mats(series_id integer); +INSERT INTO tbl_dist_mats VALUES (1), (1), (2), (2); +SELECT create_distributed_table('tbl_dist_mats', 'series_id'); + +-- Create a view that joins the distributed table with the reference table on the distribution key. +CREATE VIEW vw_citus_views as +SELECT d.series_id FROM tbl_dist_mats d JOIN tbl_ref_mats r ON d.series_id = r.row_id; + +-- The view initially works fine +SELECT * FROM vw_citus_views ORDER BY 1; +-- Now, alter the table +ALTER TABLE tbl_ref_mats ADD COLUMN category1 varchar(50); +SELECT * FROM vw_citus_views ORDER BY 1; +ALTER TABLE tbl_ref_mats ADD COLUMN category2 varchar(50); +SELECT * FROM vw_citus_views ORDER BY 1; +ALTER TABLE tbl_ref_mats DROP COLUMN category1; +SELECT * FROM vw_citus_views ORDER BY 1; + SET client_min_messages TO WARNING; DROP SCHEMA test_schema_for_sequence_propagation CASCADE; DROP TABLE table_without_sequence; From c12a4f76269c11d01a5bb993344f64521826695b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Tue, 13 Feb 2024 16:44:57 +0300 Subject: [PATCH 011/155] Adds Changelog for v12.1.2 (#7499) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d979c104..f7b982af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### citus v12.1.2 (February 12, 2024) ### + +* Fixes the incorrect column count after ALTER TABLE (#7379) + ### citus v12.1.1 (November 9, 2023) ### * Fixes leaking of memory and memory contexts in Citus foreign key cache From e2d18c5472379a4dd1b2f27ed0559fa815d21a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 14 Feb 2024 08:41:15 +0300 Subject: [PATCH 012/155] Bump Citus version to 12.1.2 (#7504) --- configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 35c035ea1..615ca97c5 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.1. +# Generated by GNU Autoconf 2.69 for Citus 12.1.2. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.1' -PACKAGE_STRING='Citus 12.1.1' +PACKAGE_VERSION='12.1.2' +PACKAGE_STRING='Citus 12.1.2' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.1 to adapt to many kinds of systems. +\`configure' configures Citus 12.1.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.1:";; + short | recursive ) echo "Configuration of Citus 12.1.2:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.1 +Citus configure 12.1.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.1, which was +It was created by Citus $as_me 12.1.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.1, which was +This file was extended by Citus $as_me 12.1.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.1 +Citus config.status 12.1.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 957d4f5f1..0a198b007 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.1]) +AC_INIT([Citus], [12.1.2]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 566cd08e6..108eebf36 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.1 + 12.1.2 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 75df19b616d05786bcddb0b6f5770e8b9da03915 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 9 Nov 2023 16:09:39 +0100 Subject: [PATCH 013/155] move pg_version_constants.h to toplevel include (#7335) In preparation of sorting and grouping all includes we wanted to move this file to the toplevel includes for good grouping/sorting. (cherry picked from commit 0dac63afc0da3f8c9019afd677449303537ec13a) --- src/backend/columnar/columnar_compression.c | 2 +- src/backend/columnar/columnar_debug.c | 2 +- src/backend/distributed/commands/call.c | 2 +- .../commands/cascade_table_operation_for_connected_relations.c | 2 +- src/backend/distributed/commands/citus_global_signal.c | 2 +- src/backend/distributed/commands/cluster.c | 2 +- src/backend/distributed/commands/collation.c | 2 +- src/backend/distributed/commands/create_distributed_table.c | 2 +- src/backend/distributed/commands/distribute_object_ops.c | 2 +- src/backend/distributed/commands/foreign_constraint.c | 2 +- src/backend/distributed/commands/function.c | 2 +- src/backend/distributed/commands/index.c | 2 +- src/backend/distributed/commands/multi_copy.c | 2 +- src/backend/distributed/commands/role.c | 2 +- src/backend/distributed/commands/subscription.c | 2 +- src/backend/distributed/commands/table.c | 2 +- src/backend/distributed/commands/trigger.c | 2 +- src/backend/distributed/commands/type.c | 2 +- src/backend/distributed/commands/utility_hook.c | 2 +- src/backend/distributed/commands/vacuum.c | 2 +- .../connection/locally_reserved_shared_connections.c | 2 +- src/backend/distributed/connection/placement_connection.c | 2 +- src/backend/distributed/connection/shared_connection_stats.c | 2 +- src/backend/distributed/deparser/citus_ruleutils.c | 2 +- src/backend/distributed/deparser/deparse_statistics_stmts.c | 2 +- src/backend/distributed/deparser/ruleutils_14.c | 2 +- src/backend/distributed/deparser/ruleutils_15.c | 2 +- src/backend/distributed/deparser/ruleutils_16.c | 2 +- src/backend/distributed/executor/citus_custom_scan.c | 2 +- .../distributed/executor/distributed_intermediate_results.c | 2 +- src/backend/distributed/executor/local_executor.c | 2 +- src/backend/distributed/executor/multi_executor.c | 2 +- src/backend/distributed/executor/query_stats.c | 2 +- src/backend/distributed/metadata/dependency.c | 2 +- src/backend/distributed/metadata/distobject.c | 2 +- src/backend/distributed/metadata/metadata_cache.c | 2 +- src/backend/distributed/metadata/metadata_utility.c | 2 +- .../distributed/metadata/pg_get_object_address_13_14_15.c | 2 +- src/backend/distributed/operations/delete_protocol.c | 2 +- src/backend/distributed/operations/modify_multiple_shards.c | 2 +- src/backend/distributed/operations/node_protocol.c | 2 +- src/backend/distributed/operations/shard_rebalancer.c | 2 +- src/backend/distributed/planner/combine_query_planner.c | 2 +- src/backend/distributed/planner/cte_inline.c | 2 +- src/backend/distributed/planner/distributed_planner.c | 2 +- src/backend/distributed/planner/extended_op_node_utils.c | 2 +- src/backend/distributed/planner/fast_path_router_planner.c | 2 +- src/backend/distributed/planner/function_call_delegation.c | 2 +- src/backend/distributed/planner/insert_select_planner.c | 2 +- .../distributed/planner/local_distributed_join_planner.c | 2 +- src/backend/distributed/planner/local_plan_cache.c | 2 +- src/backend/distributed/planner/merge_planner.c | 2 +- src/backend/distributed/planner/multi_explain.c | 2 +- src/backend/distributed/planner/multi_join_order.c | 2 +- src/backend/distributed/planner/multi_logical_optimizer.c | 2 +- src/backend/distributed/planner/multi_logical_planner.c | 2 +- src/backend/distributed/planner/multi_physical_planner.c | 2 +- src/backend/distributed/planner/multi_router_planner.c | 2 +- src/backend/distributed/planner/query_colocation_checker.c | 2 +- src/backend/distributed/planner/query_pushdown_planning.c | 2 +- src/backend/distributed/planner/recursive_planning.c | 2 +- .../distributed/planner/relation_restriction_equivalence.c | 2 +- src/backend/distributed/planner/shard_pruning.c | 2 +- src/backend/distributed/replication/multi_logical_replication.c | 2 +- src/backend/distributed/test/fake_am.c | 2 +- src/backend/distributed/test/fake_fdw.c | 2 +- src/backend/distributed/test/prune_shard_list.c | 2 +- src/backend/distributed/transaction/backend_data.c | 2 +- src/backend/distributed/transaction/relation_access_tracking.c | 2 +- src/backend/distributed/transaction/transaction_recovery.c | 2 +- src/backend/distributed/utils/citus_nodefuncs.c | 2 +- src/backend/distributed/utils/citus_outfuncs.c | 2 +- src/backend/distributed/utils/citus_safe_lib.c | 2 +- src/backend/distributed/utils/enable_ssl.c | 2 +- src/backend/distributed/utils/foreign_key_relationship.c | 2 +- src/backend/distributed/utils/log_utils.c | 2 +- src/backend/distributed/utils/maintenanced.c | 2 +- src/backend/distributed/utils/multi_partitioning_utils.c | 2 +- src/backend/distributed/utils/task_execution_utils.c | 2 +- src/include/columnar/columnar_version_compat.h | 2 +- src/include/distributed/commands/utility_hook.h | 2 +- src/include/distributed/distributed_planner.h | 2 +- src/include/distributed/hash_helpers.h | 2 +- src/include/distributed/multi_physical_planner.h | 2 +- src/include/distributed/recursive_planning.h | 2 +- src/include/distributed/relation_utils.h | 2 +- src/include/pg_version_compat.h | 2 +- src/include/{distributed => }/pg_version_constants.h | 0 88 files changed, 87 insertions(+), 87 deletions(-) rename src/include/{distributed => }/pg_version_constants.h (100%) diff --git a/src/backend/columnar/columnar_compression.c b/src/backend/columnar/columnar_compression.c index 98a175b06..50cdfb01b 100644 --- a/src/backend/columnar/columnar_compression.c +++ b/src/backend/columnar/columnar_compression.c @@ -18,7 +18,7 @@ #include "lib/stringinfo.h" #include "columnar/columnar_compression.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #if HAVE_CITUS_LIBLZ4 #include diff --git a/src/backend/columnar/columnar_debug.c b/src/backend/columnar/columnar_debug.c index cbb0d554f..c60919513 100644 --- a/src/backend/columnar/columnar_debug.c +++ b/src/backend/columnar/columnar_debug.c @@ -15,7 +15,7 @@ #include "access/table.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" #include "storage/fd.h" #include "storage/smgr.h" diff --git a/src/backend/distributed/commands/call.c b/src/backend/distributed/commands/call.c index b2f0bfca1..12a1d93b8 100644 --- a/src/backend/distributed/commands/call.c +++ b/src/backend/distributed/commands/call.c @@ -13,7 +13,7 @@ #include "postgres.h" #include "funcapi.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/pg_proc.h" #include "commands/defrem.h" diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index 1102a3a51..9b22fb161 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/xact.h" #include "catalog/pg_constraint.h" diff --git a/src/backend/distributed/commands/citus_global_signal.c b/src/backend/distributed/commands/citus_global_signal.c index 8183d6673..b1f4cf187 100644 --- a/src/backend/distributed/commands/citus_global_signal.c +++ b/src/backend/distributed/commands/citus_global_signal.c @@ -11,7 +11,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/metadata_cache.h" diff --git a/src/backend/distributed/commands/cluster.c b/src/backend/distributed/commands/cluster.c index 92fcb3ec6..cdae6fc08 100644 --- a/src/backend/distributed/commands/cluster.c +++ b/src/backend/distributed/commands/cluster.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "commands/defrem.h" diff --git a/src/backend/distributed/commands/collation.c b/src/backend/distributed/commands/collation.c index 023197e15..521ce4b3d 100644 --- a/src/backend/distributed/commands/collation.c +++ b/src/backend/distributed/commands/collation.c @@ -27,7 +27,7 @@ #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_create_or_replace.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/worker_manager.h" #include "parser/parse_type.h" #include "utils/builtins.h" diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 1e89c6b93..768e20b73 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -11,7 +11,7 @@ #include "postgres.h" #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/commands/utility_hook.h" #include "access/genam.h" diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index e31fda7b0..233831f8e 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -14,7 +14,7 @@ #include "distributed/commands.h" #include "distributed/deparser.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/version_compat.h" #include "distributed/commands/utility_hook.h" diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 7c2d50f44..709287c56 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/htup_details.h" #include "access/sysattr.h" diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 01911677d..9108d8bd2 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -21,7 +21,7 @@ #include "miscadmin.h" #include "funcapi.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 8271cc4f4..bed729992 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index a684d06cc..a5c7a47f4 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -52,7 +52,7 @@ #include /* for htons */ #include -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/htup_details.h" #include "access/htup.h" diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 63ede986d..9f819e950 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -12,7 +12,7 @@ #include "pg_version_compat.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/heapam.h" #include "access/htup_details.h" diff --git a/src/backend/distributed/commands/subscription.c b/src/backend/distributed/commands/subscription.c index 59603b559..52519b680 100644 --- a/src/backend/distributed/commands/subscription.c +++ b/src/backend/distributed/commands/subscription.c @@ -17,7 +17,7 @@ #include "commands/defrem.h" #include "distributed/commands.h" #include "distributed/connection_management.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/version_compat.h" #include "libpq-fe.h" #include "nodes/parsenodes.h" diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 500c6f3f2..e8404d38c 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -9,7 +9,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" diff --git a/src/backend/distributed/commands/trigger.c b/src/backend/distributed/commands/trigger.c index 7577dfd31..0ec8287f5 100644 --- a/src/backend/distributed/commands/trigger.c +++ b/src/backend/distributed/commands/trigger.c @@ -9,7 +9,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/table.h" diff --git a/src/backend/distributed/commands/type.c b/src/backend/distributed/commands/type.c index 02e5f0dee..ccb7bf528 100644 --- a/src/backend/distributed/commands/type.c +++ b/src/backend/distributed/commands/type.c @@ -43,7 +43,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 10e424623..a4a718733 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -25,7 +25,7 @@ *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "postgres.h" #include "miscadmin.h" diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index ee03aeae1..ee1594e1d 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "commands/defrem.h" #include "commands/vacuum.h" diff --git a/src/backend/distributed/connection/locally_reserved_shared_connections.c b/src/backend/distributed/connection/locally_reserved_shared_connections.c index e3f7cb628..d3138c734 100644 --- a/src/backend/distributed/connection/locally_reserved_shared_connections.c +++ b/src/backend/distributed/connection/locally_reserved_shared_connections.c @@ -33,7 +33,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c index cc7962e37..3924e5a05 100644 --- a/src/backend/distributed/connection/placement_connection.c +++ b/src/backend/distributed/connection/placement_connection.c @@ -11,7 +11,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/hash.h" #include "distributed/colocation_utils.h" diff --git a/src/backend/distributed/connection/shared_connection_stats.c b/src/backend/distributed/connection/shared_connection_stats.c index fcd396fe4..104caed07 100644 --- a/src/backend/distributed/connection/shared_connection_stats.c +++ b/src/backend/distributed/connection/shared_connection_stats.c @@ -13,7 +13,7 @@ #include "postgres.h" #include "pgstat.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "libpq-fe.h" diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 220ea3ec7..1456f2fb5 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -10,7 +10,7 @@ #include "postgres.h" #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 4a165ec72..599738dc5 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -12,7 +12,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/namespace.h" #include "distributed/citus_ruleutils.h" diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index e765f1103..88948cff5 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -14,7 +14,7 @@ * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "pg_config.h" diff --git a/src/backend/distributed/deparser/ruleutils_15.c b/src/backend/distributed/deparser/ruleutils_15.c index 3755e13f1..018468d0b 100644 --- a/src/backend/distributed/deparser/ruleutils_15.c +++ b/src/backend/distributed/deparser/ruleutils_15.c @@ -14,7 +14,7 @@ * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "pg_config.h" diff --git a/src/backend/distributed/deparser/ruleutils_16.c b/src/backend/distributed/deparser/ruleutils_16.c index f6f0d9766..7f2a41d75 100644 --- a/src/backend/distributed/deparser/ruleutils_16.c +++ b/src/backend/distributed/deparser/ruleutils_16.c @@ -14,7 +14,7 @@ * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "pg_config.h" diff --git a/src/backend/distributed/executor/citus_custom_scan.c b/src/backend/distributed/executor/citus_custom_scan.c index a2a2ff6cb..3403e27ca 100644 --- a/src/backend/distributed/executor/citus_custom_scan.c +++ b/src/backend/distributed/executor/citus_custom_scan.c @@ -9,7 +9,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" diff --git a/src/backend/distributed/executor/distributed_intermediate_results.c b/src/backend/distributed/executor/distributed_intermediate_results.c index c10303e18..cc351a1fc 100644 --- a/src/backend/distributed/executor/distributed_intermediate_results.c +++ b/src/backend/distributed/executor/distributed_intermediate_results.c @@ -8,7 +8,7 @@ *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include #include diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index 5661403b9..cac073b3a 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -78,7 +78,7 @@ #include "postgres.h" #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/commands/utility_hook.h" diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 662eaaf97..306698251 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" diff --git a/src/backend/distributed/executor/query_stats.c b/src/backend/distributed/executor/query_stats.c index 1ac70489c..b59777d45 100644 --- a/src/backend/distributed/executor/query_stats.c +++ b/src/backend/distributed/executor/query_stats.c @@ -15,7 +15,7 @@ #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/hash.h" #include "catalog/pg_authid.h" diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index f970cecd1..989e957af 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -11,7 +11,7 @@ #include "postgres.h" #include "distributed/commands.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/heapam.h" diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index c6a8b0a22..f0176e867 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 55d0f11c5..a2e2d87c7 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -8,7 +8,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "pg_version_compat.h" #include "stdint.h" diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index ae0f6589a..8668c0ca1 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -17,7 +17,7 @@ #include "libpq-fe.h" #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" diff --git a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c index 54f764fc1..a7f40e2ad 100644 --- a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c +++ b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c @@ -24,7 +24,7 @@ #include "distributed/citus_safe_lib.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/version_compat.h" #include "nodes/value.h" #include "utils/array.h" diff --git a/src/backend/distributed/operations/delete_protocol.c b/src/backend/distributed/operations/delete_protocol.c index abed39272..54cb568be 100644 --- a/src/backend/distributed/operations/delete_protocol.c +++ b/src/backend/distributed/operations/delete_protocol.c @@ -15,7 +15,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/operations/modify_multiple_shards.c b/src/backend/distributed/operations/modify_multiple_shards.c index 8def1b26e..8d596a10b 100644 --- a/src/backend/distributed/operations/modify_multiple_shards.c +++ b/src/backend/distributed/operations/modify_multiple_shards.c @@ -14,7 +14,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "funcapi.h" #include "libpq-fe.h" diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index a3f7092d1..eeaf34321 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index e3ee4aa4d..d339ac56a 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -17,7 +17,7 @@ #include -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/htup_details.h" #include "access/genam.h" diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index e61ff8daf..6a171dac1 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -11,7 +11,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/pg_type.h" #include "distributed/citus_ruleutils.h" diff --git a/src/backend/distributed/planner/cte_inline.c b/src/backend/distributed/planner/cte_inline.c index ce258916d..9a1bbab96 100644 --- a/src/backend/distributed/planner/cte_inline.c +++ b/src/backend/distributed/planner/cte_inline.c @@ -13,7 +13,7 @@ */ #include "postgres.h" #include "pg_version_compat.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/cte_inline.h" #include "nodes/nodeFuncs.h" diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 65278d1ea..ed644c8a7 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -9,7 +9,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "funcapi.h" diff --git a/src/backend/distributed/planner/extended_op_node_utils.c b/src/backend/distributed/planner/extended_op_node_utils.c index 0a2a8b834..bb87b6949 100644 --- a/src/backend/distributed/planner/extended_op_node_utils.c +++ b/src/backend/distributed/planner/extended_op_node_utils.c @@ -9,7 +9,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/extended_op_node_utils.h" #include "distributed/listutils.h" diff --git a/src/backend/distributed/planner/fast_path_router_planner.c b/src/backend/distributed/planner/fast_path_router_planner.c index 933ee7425..6037da534 100644 --- a/src/backend/distributed/planner/fast_path_router_planner.c +++ b/src/backend/distributed/planner/fast_path_router_planner.c @@ -34,7 +34,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/distributed_planner.h" #include "distributed/insert_select_planner.h" diff --git a/src/backend/distributed/planner/function_call_delegation.c b/src/backend/distributed/planner/function_call_delegation.c index 2f8da29c0..aeea3b714 100644 --- a/src/backend/distributed/planner/function_call_delegation.c +++ b/src/backend/distributed/planner/function_call_delegation.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index 1b7f468f8..dd4bee90f 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/pg_class.h" #include "catalog/pg_type.h" diff --git a/src/backend/distributed/planner/local_distributed_join_planner.c b/src/backend/distributed/planner/local_distributed_join_planner.c index d93921966..1867a790c 100644 --- a/src/backend/distributed/planner/local_distributed_join_planner.c +++ b/src/backend/distributed/planner/local_distributed_join_planner.c @@ -71,7 +71,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "funcapi.h" diff --git a/src/backend/distributed/planner/local_plan_cache.c b/src/backend/distributed/planner/local_plan_cache.c index 946d9fc46..1ac8e24a3 100644 --- a/src/backend/distributed/planner/local_plan_cache.c +++ b/src/backend/distributed/planner/local_plan_cache.c @@ -9,7 +9,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 3cadea23a..5c593d153 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -29,7 +29,7 @@ #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_node_metadata.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_colocation_checker.h" #include "distributed/repartition_executor.h" diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 94d125f41..bf9a1871e 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -11,7 +11,7 @@ #include "libpq-fe.h" #include "miscadmin.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/htup_details.h" #include "access/xact.h" diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index 7714a1e08..0eede6b9b 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 455f050a0..9001d724d 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index 0969e0c7c..d6897d17b 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -14,7 +14,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/heapam.h" #include "access/nbtree.h" diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 21befa6f2..aa2c2b5b4 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include #include diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 0d7a0de78..60aa81e73 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index 77baab197..fd1df1be9 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -21,7 +21,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/relation.h" #include "distributed/multi_logical_planner.h" diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 3bad73459..8ccc35c82 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -21,7 +21,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_ruleutils.h" diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index c2426cf5f..d16280662 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -48,7 +48,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "funcapi.h" diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 368ba2026..4b51a537d 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -10,7 +10,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/distributed_planner.h" diff --git a/src/backend/distributed/planner/shard_pruning.c b/src/backend/distributed/planner/shard_pruning.c index 5375a70fa..ef244ea66 100644 --- a/src/backend/distributed/planner/shard_pruning.c +++ b/src/backend/distributed/planner/shard_pruning.c @@ -66,7 +66,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "fmgr.h" diff --git a/src/backend/distributed/replication/multi_logical_replication.c b/src/backend/distributed/replication/multi_logical_replication.c index c8d0325b6..30491f42c 100644 --- a/src/backend/distributed/replication/multi_logical_replication.c +++ b/src/backend/distributed/replication/multi_logical_replication.c @@ -15,7 +15,7 @@ #include "pgstat.h" #include "libpq-fe.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" diff --git a/src/backend/distributed/test/fake_am.c b/src/backend/distributed/test/fake_am.c index 8a723e4c4..4b11d7871 100644 --- a/src/backend/distributed/test/fake_am.c +++ b/src/backend/distributed/test/fake_am.c @@ -19,7 +19,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "pg_version_compat.h" diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index 4784248c0..f53242f7f 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index a9f5e4a88..023a759cb 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index 3e2ea5ca1..c1981b77a 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" #include "unistd.h" diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index b0af4e476..3ad61ac79 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -15,7 +15,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index b46419dc2..a833f5a46 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -14,7 +14,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "miscadmin.h" #include "libpq-fe.h" diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index 735b9e15e..e74f598d8 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -10,7 +10,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "catalog/pg_type.h" #include "distributed/citus_nodes.h" diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index b4062751a..9d593492d 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -18,7 +18,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/utils/citus_safe_lib.c b/src/backend/distributed/utils/citus_safe_lib.c index 82fa8f6f2..cbd06fc50 100644 --- a/src/backend/distributed/utils/citus_safe_lib.c +++ b/src/backend/distributed/utils/citus_safe_lib.c @@ -14,7 +14,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "safe_lib.h" diff --git a/src/backend/distributed/utils/enable_ssl.c b/src/backend/distributed/utils/enable_ssl.c index cac32f74c..35b1e0f1a 100644 --- a/src/backend/distributed/utils/enable_ssl.c +++ b/src/backend/distributed/utils/enable_ssl.c @@ -18,7 +18,7 @@ * it otherwise we get warnings about redefining this value. This needs to be * done before including libpq.h. */ -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/connection_management.h" #include "distributed/memutils.h" diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index d30c767df..d69d9044d 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" diff --git a/src/backend/distributed/utils/log_utils.c b/src/backend/distributed/utils/log_utils.c index 59a090a16..7d808591b 100644 --- a/src/backend/distributed/utils/log_utils.c +++ b/src/backend/distributed/utils/log_utils.c @@ -9,7 +9,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "utils/guc.h" #include "distributed/log_utils.h" diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index 5f49de20a..db5365a81 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -16,7 +16,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index 924ba4c54..404d792f9 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -6,7 +6,7 @@ */ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "access/genam.h" #include "access/heapam.h" diff --git a/src/backend/distributed/utils/task_execution_utils.c b/src/backend/distributed/utils/task_execution_utils.c index 50652b6bd..7251514b5 100644 --- a/src/backend/distributed/utils/task_execution_utils.c +++ b/src/backend/distributed/utils/task_execution_utils.c @@ -6,7 +6,7 @@ #include #include -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "common/hashfn.h" diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 0e0ae3112..d9b29cdb0 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -12,7 +12,7 @@ #ifndef COLUMNAR_COMPAT_H #define COLUMNAR_COMPAT_H -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_15 #define ExecARDeleteTriggers_compat(a, b, c, d, e, f) \ diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index f02f83fe3..3a42cf55e 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -10,7 +10,7 @@ #ifndef MULTI_UTILITY_H #define MULTI_UTILITY_H -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "postgres.h" diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index d46fbf2e6..bc8f5bc94 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -12,7 +12,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "nodes/plannodes.h" diff --git a/src/include/distributed/hash_helpers.h b/src/include/distributed/hash_helpers.h index 2b16d110c..168879b4d 100644 --- a/src/include/distributed/hash_helpers.h +++ b/src/include/distributed/hash_helpers.h @@ -11,7 +11,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "utils/hsearch.h" diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index b7acc0574..89d144a92 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -16,7 +16,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "c.h" diff --git a/src/include/distributed/recursive_planning.h b/src/include/distributed/recursive_planning.h index a883047f6..87df7fba2 100644 --- a/src/include/distributed/recursive_planning.h +++ b/src/include/distributed/recursive_planning.h @@ -10,7 +10,7 @@ #ifndef RECURSIVE_PLANNING_H #define RECURSIVE_PLANNING_H -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/relation_restriction_equivalence.h" diff --git a/src/include/distributed/relation_utils.h b/src/include/distributed/relation_utils.h index acf84a9da..d3a5ab105 100644 --- a/src/include/distributed/relation_utils.h +++ b/src/include/distributed/relation_utils.h @@ -13,7 +13,7 @@ #include "postgres.h" -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "parser/parse_relation.h" #endif diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 1bdbae580..4e874e2ee 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -11,7 +11,7 @@ #ifndef PG_VERSION_COMPAT_H #define PG_VERSION_COMPAT_H -#include "distributed/pg_version_constants.h" +#include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_16 diff --git a/src/include/distributed/pg_version_constants.h b/src/include/pg_version_constants.h similarity index 100% rename from src/include/distributed/pg_version_constants.h rename to src/include/pg_version_constants.h From f1b1d7579c6c485417b7098e070a014b4dd60878 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 23 Nov 2023 18:19:54 +0100 Subject: [PATCH 014/155] Sort includes (#7326) CHERRY-PICK NOTES: This cherry-pick only includes the scripts, not the actual changes. These are done in a follow up commit to ease further backporting. This change adds a script to programatically group all includes in a specific order. The script was used as a one time invocation to group and sort all includes throught our formatted code. The grouping is as follows: - System includes (eg. `#include<...>`) - Postgres.h (eg. `#include "postgres.h"`) - Toplevel imports from postgres, not contained in a directory (eg. `#include "miscadmin.h"`) - General postgres includes (eg . `#include "nodes/..."`) - Toplevel citus includes, not contained in a directory (eg. `#include "citus_verion.h"`) - Columnar includes (eg. `#include "columnar/..."`) - Distributed includes (eg. `#include "distributed/..."`) Because it is quite hard to understand the difference between toplevel citus includes and toplevel postgres includes it hardcodes the list of toplevel citus includes. In the same manner it assumes anything not prefixed with `columnar/` or `distributed/` as a postgres include. The sorting/grouping is enforced by CI. Since we do so with our own script there are not changes required in our uncrustify configuration. (cherry picked from commit 0620c8f9a6f37bacf9f2e709ccca3c27f6dfbaad) --- .github/workflows/build_and_test.yml | 2 + ci/README.md | 15 +++ ci/fix_style.sh | 1 + ci/include_grouping.py | 157 +++++++++++++++++++++++++++ ci/sort_and_group_includes.sh | 12 ++ 5 files changed, 187 insertions(+) create mode 100755 ci/include_grouping.py create mode 100755 ci/sort_and_group_includes.sh diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f7e125cf1..840814f7f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -71,6 +71,8 @@ jobs: run: ci/editorconfig.sh && git diff --exit-code - name: Remove useless declarations run: ci/remove_useless_declarations.sh && git diff --cached --exit-code + - name: Sort and group includes + run: ci/sort_and_group_includes.sh && git diff --exit-code - name: Normalize test output run: ci/normalize_expected.sh && git diff --exit-code - name: Check for C-style comments in migration files diff --git a/ci/README.md b/ci/README.md index 37ef94f4f..b8dad35ac 100644 --- a/ci/README.md +++ b/ci/README.md @@ -385,3 +385,18 @@ definitions are in alphabetical order. ## `print_stack_trace.sh` This script prints stack traces for failed tests, if they left core files. + +## `sort_and_group_includes.sh` + +This script checks and fixes issues with include grouping and sorting in C files. + +Includes are grouped in the following groups: + - System includes (eg. `#include `) + - Postgres.h include (eg. `#include "postgres.h"`) + - Toplevel postgres includes (includes not in a directory eg. `#include "miscadmin.h`) + - Postgres includes in a directory (eg. `#include "catalog/pg_type.h"`) + - Toplevel citus includes (includes not in a directory eg. `#include "pg_version_constants.h"`) + - Columnar includes (eg. `#include "columnar/columnar.h"`) + - Distributed includes (eg. `#include "distributed/maintenanced.h"`) + +Within every group the include lines are sorted alphabetically. diff --git a/ci/fix_style.sh b/ci/fix_style.sh index 3d6e7ae83..bb78d5f50 100755 --- a/ci/fix_style.sh +++ b/ci/fix_style.sh @@ -19,3 +19,4 @@ ci/disallow_long_changelog_entries.sh ci/normalize_expected.sh ci/fix_gitignore.sh ci/print_stack_trace.sh +ci/sort_and_group_includes.sh diff --git a/ci/include_grouping.py b/ci/include_grouping.py new file mode 100755 index 000000000..4b1370d61 --- /dev/null +++ b/ci/include_grouping.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +easy command line to run against all citus-style checked files: + +$ git ls-files \ + | git check-attr --stdin citus-style \ + | grep 'citus-style: set' \ + | awk '{print $1}' \ + | cut -d':' -f1 \ + | xargs -n1 ./ci/include_grouping.py +""" + +import collections +import os +import sys + + +def main(args): + if len(args) < 2: + print("Usage: include_grouping.py ") + return + + file = args[1] + if not os.path.isfile(file): + sys.exit(f"File '{file}' does not exist") + + with open(file, "r") as in_file: + with open(file + ".tmp", "w") as out_file: + includes = [] + skipped_lines = [] + + # This calls print_sorted_includes on a set of consecutive #include lines. + # This implicitly keeps separation of any #include lines that are contained in + # an #ifdef, because it will order the #include lines inside and after the + # #ifdef completely separately. + for line in in_file: + # if a line starts with #include we don't want to print it yet, instead we + # want to collect all consecutive #include lines + if line.startswith("#include"): + includes.append(line) + skipped_lines = [] + continue + + # if we have collected any #include lines, we want to print them sorted + # before printing the current line. However, if the current line is empty + # we want to perform a lookahead to see if the next line is an #include. + # To maintain any separation between #include lines and their subsequent + # lines we keep track of all lines we have skipped inbetween. + if len(includes) > 0: + if len(line.strip()) == 0: + skipped_lines.append(line) + continue + + # we have includes that need to be grouped before printing the current + # line. + print_sorted_includes(includes, file=out_file) + includes = [] + + # print any skipped lines + print("".join(skipped_lines), end="", file=out_file) + skipped_lines = [] + + print(line, end="", file=out_file) + + # move out_file to file + os.rename(file + ".tmp", file) + + +def print_sorted_includes(includes, file=sys.stdout): + default_group_key = 1 + groups = collections.defaultdict(set) + + # define the groups that we separate correctly. The matchers are tested in the order + # of their priority field. The first matcher that matches the include is used to + # assign the include to a group. + # The groups are printed in the order of their group_key. + matchers = [ + { + "name": "system includes", + "matcher": lambda x: x.startswith("<"), + "group_key": -2, + "priority": 0, + }, + { + "name": "toplevel postgres includes", + "matcher": lambda x: "/" not in x, + "group_key": 0, + "priority": 9, + }, + { + "name": "postgres.h", + "matcher": lambda x: x.strip() in ['"postgres.h"'], + "group_key": -1, + "priority": -1, + }, + { + "name": "toplevel citus inlcudes", + "matcher": lambda x: x.strip() + in [ + '"citus_version.h"', + '"pg_version_compat.h"', + '"pg_version_constants.h"', + ], + "group_key": 3, + "priority": 0, + }, + { + "name": "columnar includes", + "matcher": lambda x: x.startswith('"columnar/'), + "group_key": 4, + "priority": 1, + }, + { + "name": "distributed includes", + "matcher": lambda x: x.startswith('"distributed/'), + "group_key": 5, + "priority": 1, + }, + ] + matchers.sort(key=lambda x: x["priority"]) + + # throughout our codebase we have some includes where either postgres or citus + # includes are wrongfully included with the syntax for system includes. Before we + # try to match those we will change the <> to "" to make them match our system. This + # will also rewrite the include to the correct syntax. + common_system_include_error_prefixes = [" 0: + print(file=file) + includes = group[1] + print("".join(sorted(includes)), end="", file=file) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/ci/sort_and_group_includes.sh b/ci/sort_and_group_includes.sh new file mode 100755 index 000000000..1c3a91458 --- /dev/null +++ b/ci/sort_and_group_includes.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail +# shellcheck disable=SC1091 +source ci/ci_helpers.sh + +git ls-files \ + | git check-attr --stdin citus-style \ + | grep 'citus-style: set' \ + | awk '{print $1}' \ + | cut -d':' -f1 \ + | xargs -n1 ./ci/include_grouping.py From 48c62095ffe2f6895401c677c774d9dfc4d9d83b Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 16 Apr 2024 17:46:15 +0200 Subject: [PATCH 015/155] Actually sort includes after cherry-pick --- src/backend/columnar/columnar.c | 6 +- src/backend/columnar/columnar_compression.c | 5 +- src/backend/columnar/columnar_customscan.c | 8 +- src/backend/columnar/columnar_debug.c | 6 +- src/backend/columnar/columnar_metadata.c | 30 ++-- src/backend/columnar/columnar_reader.c | 7 +- src/backend/columnar/columnar_storage.c | 2 +- src/backend/columnar/columnar_tableam.c | 23 +-- src/backend/columnar/columnar_writer.c | 5 +- src/backend/columnar/write_state_management.c | 20 ++- src/backend/distributed/cdc/cdc_decoder.c | 3 +- .../distributed/cdc/cdc_decoder_utils.c | 11 +- .../distributed/cdc/cdc_decoder_utils.h | 6 +- src/backend/distributed/clock/causal_clock.c | 29 ++-- .../distributed/commands/alter_table.c | 16 +- src/backend/distributed/commands/begin.c | 4 +- src/backend/distributed/commands/call.c | 29 ++-- ..._table_operation_for_connected_relations.c | 16 +- .../citus_add_local_table_to_metadata.c | 20 +-- .../commands/citus_global_signal.c | 6 +- src/backend/distributed/commands/cluster.c | 6 +- src/backend/distributed/commands/collation.c | 34 ++-- src/backend/distributed/commands/common.c | 2 +- .../commands/create_distributed_table.c | 81 +++++----- src/backend/distributed/commands/database.c | 3 +- .../distributed/commands/dependencies.c | 8 +- .../commands/distribute_object_ops.c | 7 +- .../commands/drop_distributed_table.c | 9 +- src/backend/distributed/commands/extension.c | 23 +-- .../distributed/commands/foreign_constraint.c | 29 ++-- .../commands/foreign_data_wrapper.c | 13 +- .../distributed/commands/foreign_server.c | 14 +- src/backend/distributed/commands/function.c | 58 +++---- src/backend/distributed/commands/grant.c | 7 +- src/backend/distributed/commands/index.c | 45 +++--- .../distributed/commands/local_multi_copy.c | 25 +-- src/backend/distributed/commands/multi_copy.c | 48 +++--- src/backend/distributed/commands/owned.c | 37 ++--- src/backend/distributed/commands/policy.c | 16 +- .../distributed/commands/publication.c | 18 ++- src/backend/distributed/commands/rename.c | 5 +- src/backend/distributed/commands/role.c | 44 +++--- src/backend/distributed/commands/schema.c | 34 ++-- .../commands/schema_based_sharding.c | 12 +- src/backend/distributed/commands/sequence.c | 11 +- src/backend/distributed/commands/statistics.c | 20 +-- .../distributed/commands/subscription.c | 20 +-- src/backend/distributed/commands/table.c | 49 +++--- .../distributed/commands/text_search.c | 3 +- src/backend/distributed/commands/trigger.c | 14 +- src/backend/distributed/commands/truncate.c | 14 +- src/backend/distributed/commands/type.c | 26 +-- .../distributed/commands/utility_hook.c | 38 ++--- src/backend/distributed/commands/vacuum.c | 15 +- .../distributed/commands/variableset.c | 16 +- src/backend/distributed/commands/view.c | 26 +-- .../connection/connection_configuration.c | 8 +- .../connection/connection_management.c | 28 ++-- .../locally_reserved_shared_connections.c | 9 +- .../connection/placement_connection.c | 13 +- .../distributed/connection/remote_commands.c | 18 +-- .../connection/shared_connection_stats.c | 16 +- .../connection/worker_log_messages.c | 3 +- .../distributed/deparser/citus_grantutils.c | 4 +- .../distributed/deparser/citus_ruleutils.c | 38 ++--- .../deparser/deparse_collation_stmts.c | 2 +- .../deparser/deparse_database_stmts.c | 7 +- .../deparser/deparse_extension_stmts.c | 5 +- .../deparse_foreign_data_wrapper_stmts.c | 7 +- .../deparser/deparse_foreign_server_stmts.c | 7 +- .../deparser/deparse_function_stmts.c | 11 +- .../deparser/deparse_owned_stmts.c | 7 +- .../deparser/deparse_publication_stmts.c | 11 +- .../distributed/deparser/deparse_role_stmts.c | 11 +- .../deparser/deparse_schema_stmts.c | 7 +- .../deparser/deparse_sequence_stmts.c | 5 +- .../deparser/deparse_statistics_stmts.c | 9 +- .../deparser/deparse_table_stmts.c | 10 +- .../distributed/deparser/deparse_view_stmts.c | 9 +- .../distributed/deparser/objectaddress.c | 7 +- .../deparser/qualify_aggregate_stmts.c | 3 +- .../deparser/qualify_function_stmt.c | 5 +- .../deparser/qualify_publication_stmt.c | 5 +- .../distributed/deparser/qualify_role_stmt.c | 3 +- .../deparser/qualify_sequence_stmt.c | 5 +- .../deparser/qualify_statistics_stmt.c | 9 +- .../distributed/deparser/qualify_table_stmt.c | 1 + .../distributed/deparser/qualify_type_stmt.c | 9 +- .../distributed/deparser/qualify_view_stmt.c | 5 +- .../distributed/executor/adaptive_executor.c | 39 ++--- .../distributed/executor/citus_custom_scan.c | 27 ++-- .../directed_acyclic_graph_execution.c | 3 +- .../executor/distributed_execution_locks.c | 2 +- .../distributed_intermediate_results.c | 16 +- .../executor/executor_util_params.c | 4 +- .../executor/executor_util_tasks.c | 1 + .../executor/executor_util_tuples.c | 4 +- .../executor/insert_select_executor.c | 53 ++++--- .../executor/intermediate_results.c | 34 ++-- .../distributed/executor/local_executor.c | 20 +-- .../distributed/executor/merge_executor.c | 9 +- .../distributed/executor/multi_executor.c | 44 +++--- .../executor/multi_server_executor.c | 14 +- .../partitioned_intermediate_results.c | 14 +- .../distributed/executor/placement_access.c | 2 +- .../distributed/executor/query_stats.c | 26 +-- .../executor/repartition_executor.c | 1 + .../executor/repartition_join_execution.c | 8 +- .../distributed/executor/subplan_execution.c | 5 +- src/backend/distributed/executor/transmit.c | 18 ++- .../distributed/executor/tuple_destination.c | 8 +- src/backend/distributed/metadata/dependency.c | 15 +- src/backend/distributed/metadata/distobject.c | 29 ++-- .../distributed/metadata/metadata_cache.c | 75 +++++---- .../distributed/metadata/metadata_sync.c | 84 +++++----- .../distributed/metadata/metadata_utility.c | 70 ++++---- .../distributed/metadata/node_metadata.c | 39 ++--- .../metadata/pg_get_object_address_13_14_15.c | 19 ++- .../operations/citus_create_restore_point.c | 12 +- .../citus_split_shard_by_split_points.c | 12 +- .../distributed/operations/citus_tools.c | 12 +- .../distributed/operations/create_shards.c | 42 ++--- .../distributed/operations/delete_protocol.c | 49 +++--- .../distributed/operations/health_check.c | 3 +- .../distributed/operations/isolate_shards.c | 24 +-- .../operations/modify_multiple_shards.c | 48 +++--- .../distributed/operations/node_protocol.c | 33 ++-- .../distributed/operations/partitioning.c | 6 +- .../replicate_none_dist_table_shard.c | 2 + .../distributed/operations/shard_cleaner.c | 12 +- .../distributed/operations/shard_rebalancer.c | 56 +++---- .../distributed/operations/shard_split.c | 48 +++--- .../distributed/operations/shard_transfer.c | 37 ++--- .../distributed/operations/stage_protocol.c | 30 ++-- .../worker_copy_table_to_node_udf.c | 1 + .../operations/worker_node_manager.c | 16 +- .../operations/worker_shard_copy.c | 19 ++- .../operations/worker_split_copy_udf.c | 9 +- .../worker_split_shard_release_dsm_udf.c | 1 + ...worker_split_shard_replication_setup_udf.c | 31 ++-- .../planner/combine_query_planner.c | 17 +- src/backend/distributed/planner/cte_inline.c | 8 +- .../distributed/planner/deparse_shard_query.c | 24 +-- .../distributed/planner/distributed_planner.c | 39 ++--- .../planner/extended_op_node_utils.c | 10 +- .../planner/fast_path_router_planner.c | 21 +-- .../planner/function_call_delegation.c | 48 +++--- .../planner/insert_select_planner.c | 43 ++--- .../planner/intermediate_result_pruning.c | 5 +- .../planner/local_distributed_join_planner.c | 66 ++++---- .../distributed/planner/local_plan_cache.c | 11 +- .../distributed/planner/merge_planner.c | 8 +- .../distributed/planner/multi_explain.c | 64 ++++---- .../distributed/planner/multi_join_order.c | 30 ++-- .../planner/multi_logical_optimizer.c | 39 ++--- .../planner/multi_logical_planner.c | 39 ++--- .../planner/multi_physical_planner.c | 67 ++++---- .../planner/multi_router_planner.c | 84 +++++----- .../planner/query_colocation_checker.c | 24 +-- .../planner/query_pushdown_planning.c | 15 +- .../distributed/planner/recursive_planning.c | 61 ++++--- .../relation_restriction_equivalence.c | 24 +-- .../distributed/planner/shard_pruning.c | 30 ++-- .../distributed/planner/tdigest_extension.c | 7 +- .../distributed/progress/multi_progress.c | 6 +- .../distributed/relay/relay_event_utility.c | 28 ++-- .../replication/multi_logical_replication.c | 66 ++++---- .../shardsplit/shardsplit_decoder.c | 18 ++- .../shardsplit_logical_replication.c | 19 ++- .../shardsplit/shardsplit_shared_memory.c | 12 +- src/backend/distributed/shared_library_init.c | 149 +++++++++--------- .../distributed/test/backend_counter.c | 1 + .../distributed/test/citus_depended_object.c | 9 +- .../distributed/test/citus_stat_tenants.c | 4 +- .../distributed/test/colocation_utils.c | 2 + src/backend/distributed/test/create_shards.c | 8 +- .../distributed/test/deparse_function_query.c | 3 +- .../distributed/test/deparse_shard_query.c | 16 +- src/backend/distributed/test/dependency.c | 1 + .../test/distributed_deadlock_detection.c | 8 +- .../test/distributed_intermediate_results.c | 5 +- .../distributed/test/distribution_metadata.c | 34 ++-- src/backend/distributed/test/fake_am.c | 9 +- src/backend/distributed/test/fake_fdw.c | 10 +- .../test/foreign_key_relationship_query.c | 6 +- src/backend/distributed/test/global_pid.c | 1 + src/backend/distributed/test/hide_shards.c | 1 + .../distributed/test/intermediate_results.c | 1 + .../test/make_external_connection.c | 15 +- src/backend/distributed/test/metadata_sync.c | 12 +- .../distributed/test/partitioning_utils.c | 8 +- src/backend/distributed/test/progress_utils.c | 10 +- .../distributed/test/prune_shard_list.c | 25 +-- .../test/relation_access_tracking.c | 1 + .../test/run_from_same_connection.c | 15 +- .../distributed/test/sequential_execution.c | 1 + .../distributed/test/shard_rebalancer.c | 25 +-- .../test/shared_connection_counters.c | 10 +- src/backend/distributed/test/xact_stats.c | 1 + .../distributed/transaction/backend_data.c | 32 ++-- .../transaction/citus_dist_stat_activity.c | 3 +- .../distributed_deadlock_detection.c | 8 +- .../distributed/transaction/lock_graph.c | 9 +- .../transaction/relation_access_tracking.c | 13 +- .../transaction/remote_transaction.c | 6 +- .../transaction/transaction_management.c | 28 ++-- .../transaction/transaction_recovery.c | 31 ++-- .../transaction/worker_transaction.c | 20 +-- src/backend/distributed/utils/acquire_lock.c | 4 +- .../distributed/utils/aggregate_utils.c | 10 +- src/backend/distributed/utils/array_type.c | 7 +- .../distributed/utils/background_jobs.c | 6 +- src/backend/distributed/utils/cancel_utils.c | 2 + src/backend/distributed/utils/citus_clauses.c | 12 +- .../distributed/utils/citus_copyfuncs.c | 4 +- .../distributed/utils/citus_depended_object.c | 20 +-- .../distributed/utils/citus_nodefuncs.c | 7 +- .../distributed/utils/citus_safe_lib.c | 9 +- .../distributed/utils/citus_stat_tenants.c | 26 +-- src/backend/distributed/utils/citus_version.c | 3 +- .../distributed/utils/colocation_utils.c | 18 ++- src/backend/distributed/utils/directory.c | 1 + .../distributed/utils/distribution_column.c | 12 +- .../utils/distribution_column_map.c | 3 +- src/backend/distributed/utils/enable_ssl.c | 12 +- src/backend/distributed/utils/errormessage.c | 5 +- .../utils/foreign_key_relationship.c | 20 +-- src/backend/distributed/utils/function.c | 4 +- .../distributed/utils/function_utils.c | 5 +- src/backend/distributed/utils/hash_helpers.c | 3 +- src/backend/distributed/utils/jsonbutils.c | 12 +- src/backend/distributed/utils/listutils.c | 8 +- src/backend/distributed/utils/log_utils.c | 13 +- src/backend/distributed/utils/maintenanced.c | 46 +++--- .../utils/multi_partitioning_utils.c | 28 ++-- .../distributed/utils/namespace_utils.c | 3 +- src/backend/distributed/utils/param_utils.c | 13 +- src/backend/distributed/utils/priority.c | 6 +- src/backend/distributed/utils/query_utils.c | 7 +- .../distributed/utils/reference_table_utils.c | 20 +-- .../utils/replication_origin_session_utils.c | 10 +- src/backend/distributed/utils/resource_lock.c | 46 +++--- src/backend/distributed/utils/role.c | 6 +- src/backend/distributed/utils/shard_utils.c | 2 + .../distributed/utils/shardinterval_utils.c | 12 +- .../distributed/utils/statistics_collection.c | 17 +- .../distributed/utils/task_execution_utils.c | 25 +-- .../utils/tenant_schema_metadata.c | 7 +- src/backend/distributed/utils/tuplestore.c | 3 +- src/backend/distributed/utils/type_utils.c | 3 +- .../worker/task_tracker_protocol.c | 1 + .../worker/worker_create_or_replace.c | 7 +- .../worker/worker_data_fetch_protocol.c | 30 ++-- .../distributed/worker/worker_drop_protocol.c | 21 +-- .../worker/worker_partition_protocol.c | 1 + .../worker/worker_shard_visibility.c | 18 ++- .../worker/worker_sql_task_protocol.c | 6 +- .../worker/worker_truncate_trigger_protocol.c | 10 +- src/include/columnar/columnar.h | 4 +- src/include/columnar/columnar_tableam.h | 12 +- src/include/distributed/backend_data.h | 3 +- src/include/distributed/citus_custom_scan.h | 5 +- .../distributed/citus_depended_object.h | 3 +- src/include/distributed/citus_nodefuncs.h | 3 +- src/include/distributed/citus_ruleutils.h | 3 +- src/include/distributed/colocation_utils.h | 3 +- .../distributed/combine_query_planner.h | 3 +- src/include/distributed/commands.h | 5 +- src/include/distributed/commands/multi_copy.h | 7 +- .../distributed/commands/utility_hook.h | 6 +- .../distributed/connection_management.h | 8 +- .../distributed/coordinator_protocol.h | 8 +- src/include/distributed/deparse_shard_query.h | 1 + src/include/distributed/deparser.h | 4 +- .../distributed_deadlock_detection.h | 3 +- .../distributed/distributed_execution_locks.h | 1 + src/include/distributed/distributed_planner.h | 5 +- src/include/distributed/enterprise.h | 1 + src/include/distributed/errormessage.h | 4 +- src/include/distributed/executor_util.h | 3 +- .../distributed/foreign_key_relationship.h | 6 +- src/include/distributed/hash_helpers.h | 4 +- .../distributed/insert_select_planner.h | 5 +- .../distributed/intermediate_results.h | 3 +- src/include/distributed/listutils.h | 4 +- .../local_distributed_join_planner.h | 1 + src/include/distributed/lock_graph.h | 4 +- src/include/distributed/merge_planner.h | 1 + src/include/distributed/metadata/dependency.h | 3 +- src/include/distributed/metadata_cache.h | 4 +- src/include/distributed/metadata_sync.h | 3 +- src/include/distributed/metadata_utility.h | 5 +- src/include/distributed/multi_executor.h | 2 +- src/include/distributed/multi_explain.h | 3 +- .../distributed/multi_logical_planner.h | 9 +- .../distributed/multi_logical_replication.h | 1 + .../distributed/multi_partitioning_utils.h | 3 +- .../distributed/multi_physical_planner.h | 19 +-- src/include/distributed/multi_progress.h | 1 + .../distributed/multi_router_planner.h | 5 +- src/include/distributed/placement_access.h | 2 + .../distributed/query_colocation_checker.h | 3 +- .../distributed/query_pushdown_planning.h | 4 +- src/include/distributed/query_utils.h | 1 + src/include/distributed/recursive_planning.h | 8 +- src/include/distributed/relay_utility.h | 1 + src/include/distributed/remote_transaction.h | 3 +- .../replication_origin_session_utils.h | 2 + src/include/distributed/resource_lock.h | 4 +- src/include/distributed/shard_pruning.h | 3 +- src/include/distributed/shard_rebalancer.h | 2 + src/include/distributed/shard_transfer.h | 3 +- src/include/distributed/shardinterval_utils.h | 5 +- .../distributed/transaction_management.h | 1 - src/include/distributed/tuple_destination.h | 3 +- .../distributed/utils/citus_stat_tenants.h | 3 +- src/include/distributed/utils/directory.h | 1 + src/include/distributed/utils/function.h | 1 + src/include/distributed/version_compat.h | 12 +- src/include/distributed/worker_manager.h | 2 +- src/include/distributed/worker_protocol.h | 4 +- src/include/distributed/worker_transaction.h | 3 +- 322 files changed, 2625 insertions(+), 2240 deletions(-) diff --git a/src/backend/columnar/columnar.c b/src/backend/columnar/columnar.c index 85ec06d00..4914bbc3a 100644 --- a/src/backend/columnar/columnar.c +++ b/src/backend/columnar/columnar.c @@ -11,16 +11,18 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - #include #include +#include "postgres.h" + #include "miscadmin.h" + #include "utils/guc.h" #include "utils/rel.h" #include "citus_version.h" + #include "columnar/columnar.h" #include "columnar/columnar_tableam.h" diff --git a/src/backend/columnar/columnar_compression.c b/src/backend/columnar/columnar_compression.c index 50cdfb01b..2ff35da98 100644 --- a/src/backend/columnar/columnar_compression.c +++ b/src/backend/columnar/columnar_compression.c @@ -13,13 +13,14 @@ */ #include "postgres.h" -#include "citus_version.h" #include "common/pg_lzcompress.h" #include "lib/stringinfo.h" -#include "columnar/columnar_compression.h" +#include "citus_version.h" #include "pg_version_constants.h" +#include "columnar/columnar_compression.h" + #if HAVE_CITUS_LIBLZ4 #include #endif diff --git a/src/backend/columnar/columnar_customscan.c b/src/backend/columnar/columnar_customscan.c index 4ea96a121..ebf815d20 100644 --- a/src/backend/columnar/columnar_customscan.c +++ b/src/backend/columnar/columnar_customscan.c @@ -10,18 +10,17 @@ *------------------------------------------------------------------------- */ -#include "citus_version.h" +#include #include "postgres.h" -#include +#include "miscadmin.h" #include "access/amapi.h" #include "access/skey.h" #include "catalog/pg_am.h" #include "catalog/pg_statistic.h" #include "commands/defrem.h" -#include "miscadmin.h" #include "nodes/extensible.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -33,6 +32,8 @@ #include "optimizer/paths.h" #include "optimizer/plancat.h" #include "optimizer/restrictinfo.h" + +#include "citus_version.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "parser/parse_relation.h" #include "parser/parsetree.h" @@ -48,6 +49,7 @@ #include "columnar/columnar_customscan.h" #include "columnar/columnar_metadata.h" #include "columnar/columnar_tableam.h" + #include "distributed/listutils.h" /* diff --git a/src/backend/columnar/columnar_debug.c b/src/backend/columnar/columnar_debug.c index c60919513..bf12108a9 100644 --- a/src/backend/columnar/columnar_debug.c +++ b/src/backend/columnar/columnar_debug.c @@ -11,12 +11,12 @@ #include "postgres.h" #include "funcapi.h" +#include "miscadmin.h" + #include "access/nbtree.h" #include "access/table.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" -#include "pg_version_constants.h" -#include "miscadmin.h" #include "storage/fd.h" #include "storage/smgr.h" #include "utils/guc.h" @@ -25,6 +25,8 @@ #include "utils/tuplestore.h" #include "pg_version_compat.h" +#include "pg_version_constants.h" + #include "columnar/columnar.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_version_compat.h" diff --git a/src/backend/columnar/columnar_metadata.c b/src/backend/columnar/columnar_metadata.c index e7a6bfa95..b756a7607 100644 --- a/src/backend/columnar/columnar_metadata.c +++ b/src/backend/columnar/columnar_metadata.c @@ -19,46 +19,50 @@ */ +#include + #include "postgres.h" +#include "miscadmin.h" #include "safe_lib.h" -#include "citus_version.h" -#include "columnar/columnar.h" -#include "columnar/columnar_storage.h" -#include "columnar/columnar_version_compat.h" -#include "distributed/listutils.h" - -#include #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "access/xact.h" #include "catalog/indexing.h" -#include "catalog/pg_namespace.h" -#include "catalog/pg_collation.h" -#include "catalog/pg_type.h" #include "catalog/namespace.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" -#include "miscadmin.h" -#include "nodes/execnodes.h" #include "lib/stringinfo.h" +#include "nodes/execnodes.h" + +#include "citus_version.h" + +#include "columnar/columnar.h" +#include "columnar/columnar_storage.h" +#include "columnar/columnar_version_compat.h" + +#include "distributed/listutils.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "parser/parse_relation.h" #endif #include "port.h" + #include "storage/fd.h" #include "storage/lmgr.h" #include "storage/procarray.h" #include "storage/smgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" -#include "utils/memutils.h" #include "utils/lsyscache.h" +#include "utils/memutils.h" #include "utils/rel.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "storage/relfilelocator.h" diff --git a/src/backend/columnar/columnar_reader.c b/src/backend/columnar/columnar_reader.c index 526dd03cb..7ef0d15d7 100644 --- a/src/backend/columnar/columnar_reader.c +++ b/src/backend/columnar/columnar_reader.c @@ -22,16 +22,15 @@ #include "access/xact.h" #include "catalog/pg_am.h" #include "commands/defrem.h" -#include "distributed/listutils.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" -#include "optimizer/optimizer.h" #include "optimizer/clauses.h" +#include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "storage/fd.h" #include "utils/guc.h" -#include "utils/memutils.h" #include "utils/lsyscache.h" +#include "utils/memutils.h" #include "utils/rel.h" #include "columnar/columnar.h" @@ -39,6 +38,8 @@ #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" +#include "distributed/listutils.h" + #define UNEXPECTED_STRIPE_READ_ERR_MSG \ "attempted to read an unexpected stripe while reading columnar " \ "table %s, stripe with id=" UINT64_FORMAT " is not flushed" diff --git a/src/backend/columnar/columnar_storage.c b/src/backend/columnar/columnar_storage.c index 21aa7ab9c..0ae6ccca3 100644 --- a/src/backend/columnar/columnar_storage.c +++ b/src/backend/columnar/columnar_storage.c @@ -36,11 +36,11 @@ #include "postgres.h" +#include "miscadmin.h" #include "safe_lib.h" #include "access/generic_xlog.h" #include "catalog/storage.h" -#include "miscadmin.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index dade931df..40486d08f 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -1,42 +1,38 @@ -#include "citus_version.h" +#include #include "postgres.h" -#include - #include "miscadmin.h" +#include "pgstat.h" +#include "safe_lib.h" +#include "access/detoast.h" #include "access/genam.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/rewriteheap.h" #include "access/tableam.h" #include "access/tsmapi.h" -#include "access/detoast.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" +#include "catalog/pg_extension.h" #include "catalog/pg_publication.h" #include "catalog/pg_trigger.h" -#include "catalog/pg_extension.h" #include "catalog/storage.h" #include "catalog/storage_xlog.h" #include "commands/defrem.h" +#include "commands/extension.h" #include "commands/progress.h" #include "commands/vacuum.h" -#include "commands/extension.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "optimizer/plancat.h" -#include "pg_version_compat.h" -#include "pgstat.h" -#include "safe_lib.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" -#include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/predicate.h" #include "storage/procarray.h" @@ -44,17 +40,22 @@ #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" +#include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" #include "utils/rel.h" #include "utils/relcache.h" -#include "utils/lsyscache.h" #include "utils/syscache.h" + +#include "citus_version.h" +#include "pg_version_compat.h" + #include "columnar/columnar.h" #include "columnar/columnar_customscan.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" + #include "distributed/listutils.h" /* diff --git a/src/backend/columnar/columnar_writer.c b/src/backend/columnar/columnar_writer.c index 3b510ce74..52d4e17aa 100644 --- a/src/backend/columnar/columnar_writer.c +++ b/src/backend/columnar/columnar_writer.c @@ -16,18 +16,19 @@ #include "postgres.h" +#include "miscadmin.h" #include "safe_lib.h" #include "access/heapam.h" #include "access/nbtree.h" #include "catalog/pg_am.h" -#include "miscadmin.h" -#include "pg_version_compat.h" #include "storage/fd.h" #include "storage/smgr.h" #include "utils/guc.h" #include "utils/memutils.h" #include "utils/rel.h" + +#include "pg_version_compat.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "storage/relfilelocator.h" #include "utils/relfilenumbermap.h" diff --git a/src/backend/columnar/write_state_management.c b/src/backend/columnar/write_state_management.c index 27d902e61..7f35c5dd1 100644 --- a/src/backend/columnar/write_state_management.c +++ b/src/backend/columnar/write_state_management.c @@ -1,21 +1,17 @@ -#include "citus_version.h" - -#include "postgres.h" -#include "columnar/columnar.h" - - #include +#include "postgres.h" + #include "miscadmin.h" +#include "pgstat.h" #include "access/genam.h" #include "access/heapam.h" +#include "access/heaptoast.h" #include "access/multixact.h" #include "access/rewriteheap.h" #include "access/tsmapi.h" -#include "access/heaptoast.h" -#include "common/hashfn.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/index.h" @@ -26,14 +22,12 @@ #include "catalog/storage_xlog.h" #include "commands/progress.h" #include "commands/vacuum.h" +#include "common/hashfn.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "optimizer/plancat.h" -#include "pg_version_compat.h" -#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" -#include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/predicate.h" #include "storage/procarray.h" @@ -44,6 +38,10 @@ #include "utils/rel.h" #include "utils/syscache.h" +#include "citus_version.h" +#include "pg_version_compat.h" + +#include "columnar/columnar.h" #include "columnar/columnar_customscan.h" #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" diff --git a/src/backend/distributed/cdc/cdc_decoder.c b/src/backend/distributed/cdc/cdc_decoder.c index 2beb27772..cf9f4963b 100644 --- a/src/backend/distributed/cdc/cdc_decoder.c +++ b/src/backend/distributed/cdc/cdc_decoder.c @@ -8,8 +8,9 @@ *------------------------------------------------------------------------- */ -#include "cdc_decoder_utils.h" #include "postgres.h" + +#include "cdc_decoder_utils.h" #include "fmgr.h" #include "access/genam.h" diff --git a/src/backend/distributed/cdc/cdc_decoder_utils.c b/src/backend/distributed/cdc/cdc_decoder_utils.c index f5b23aa12..b571d18b9 100644 --- a/src/backend/distributed/cdc/cdc_decoder_utils.c +++ b/src/backend/distributed/cdc/cdc_decoder_utils.c @@ -8,18 +8,21 @@ *------------------------------------------------------------------------- */ #include "postgres.h" -#include "commands/extension.h" + +#include "cdc_decoder_utils.h" #include "fmgr.h" #include "miscadmin.h" + #include "access/genam.h" #include "access/heapam.h" +#include "catalog/pg_namespace.h" +#include "commands/extension.h" #include "common/hashfn.h" #include "common/string.h" #include "utils/fmgroids.h" -#include "utils/typcache.h" #include "utils/lsyscache.h" -#include "catalog/pg_namespace.h" -#include "cdc_decoder_utils.h" +#include "utils/typcache.h" + #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/relay_utility.h" diff --git a/src/backend/distributed/cdc/cdc_decoder_utils.h b/src/backend/distributed/cdc/cdc_decoder_utils.h index 46d1e4ae5..8b9cb298b 100644 --- a/src/backend/distributed/cdc/cdc_decoder_utils.h +++ b/src/backend/distributed/cdc/cdc_decoder_utils.h @@ -12,9 +12,11 @@ #define CITUS_CDC_DECODER_H #include "postgres.h" -#include "fmgr.h" -#include "replication/logical.h" + #include "c.h" +#include "fmgr.h" + +#include "replication/logical.h" #define InvalidRepOriginId 0 #define INVALID_SHARD_ID 0 diff --git a/src/backend/distributed/clock/causal_clock.c b/src/backend/distributed/clock/causal_clock.c index 74c87bad4..3d64757e3 100644 --- a/src/backend/distributed/clock/causal_clock.c +++ b/src/backend/distributed/clock/causal_clock.c @@ -11,36 +11,37 @@ #include #include "postgres.h" -#include "miscadmin.h" + #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" +#include "miscadmin.h" -#include "utils/builtins.h" -#include "utils/datum.h" -#include "utils/numeric.h" -#include "utils/typcache.h" -#include "nodes/pg_list.h" #include "catalog/namespace.h" #include "commands/extension.h" #include "commands/sequence.h" #include "executor/spi.h" +#include "nodes/pg_list.h" #include "postmaster/postmaster.h" #include "storage/ipc.h" #include "storage/lwlock.h" +#include "storage/s_lock.h" #include "storage/shmem.h" #include "storage/spin.h" -#include "storage/s_lock.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/numeric.h" +#include "utils/typcache.h" #include "distributed/causal_clock.h" -#include "distributed/listutils.h" -#include "distributed/lock_graph.h" -#include "distributed/local_executor.h" -#include "distributed/metadata_cache.h" -#include "distributed/remote_commands.h" -#include "distributed/placement_connection.h" -#include "distributed/coordinator_protocol.h" #include "distributed/citus_safe_lib.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/lock_graph.h" +#include "distributed/metadata_cache.h" +#include "distributed/placement_connection.h" +#include "distributed/remote_commands.h" #define SAVE_AND_PERSIST(c) \ do { \ diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index 8c2736a28..a81f23ad6 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -34,9 +34,16 @@ #include "catalog/pg_am.h" #include "catalog/pg_depend.h" #include "catalog/pg_rewrite_d.h" +#include "commands/defrem.h" +#include "executor/spi.h" +#include "nodes/pg_list.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + #include "columnar/columnar.h" #include "columnar/columnar_tableam.h" -#include "commands/defrem.h" + #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" @@ -57,16 +64,11 @@ #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/replication_origin_session_utils.h" -#include "distributed/shared_library_init.h" #include "distributed/shard_utils.h" +#include "distributed/shared_library_init.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" -#include "executor/spi.h" -#include "nodes/pg_list.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" /* Table Conversion Types */ diff --git a/src/backend/distributed/commands/begin.c b/src/backend/distributed/commands/begin.c index 3ff28ac20..b19b04484 100644 --- a/src/backend/distributed/commands/begin.c +++ b/src/backend/distributed/commands/begin.c @@ -9,12 +9,14 @@ */ #include "postgres.h" + #include "c.h" +#include "nodes/parsenodes.h" + #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/transaction_management.h" -#include "nodes/parsenodes.h" /* diff --git a/src/backend/distributed/commands/call.c b/src/backend/distributed/commands/call.c index 12a1d93b8..9e54513c6 100644 --- a/src/backend/distributed/commands/call.c +++ b/src/backend/distributed/commands/call.c @@ -11,12 +11,23 @@ */ #include "postgres.h" -#include "funcapi.h" -#include "pg_version_constants.h" +#include "funcapi.h" +#include "miscadmin.h" #include "catalog/pg_proc.h" #include "commands/defrem.h" +#include "nodes/nodeFuncs.h" +#include "nodes/parsenodes.h" +#include "nodes/primnodes.h" +#include "optimizer/clauses.h" +#include "tcop/dest.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + +#include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" @@ -26,27 +37,17 @@ #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/function_call_delegation.h" -#include "distributed/metadata_utility.h" #include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" -#include "distributed/adaptive_executor.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" -#include "distributed/reference_table_utils.h" #include "distributed/shard_pruning.h" #include "distributed/tuple_destination.h" #include "distributed/version_compat.h" -#include "distributed/worker_manager.h" #include "distributed/worker_log_messages.h" -#include "optimizer/clauses.h" -#include "nodes/nodeFuncs.h" -#include "nodes/parsenodes.h" -#include "nodes/primnodes.h" -#include "miscadmin.h" -#include "tcop/dest.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" +#include "distributed/worker_manager.h" /* global variable tracking whether we are in a delegated procedure call */ diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index 9b22fb161..c88367462 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -12,12 +12,19 @@ #include "postgres.h" -#include "pg_version_constants.h" +#include "miscadmin.h" #include "access/xact.h" #include "catalog/pg_constraint.h" -#include "distributed/commands/utility_hook.h" +#include "executor/spi.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + #include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" @@ -26,11 +33,6 @@ #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_protocol.h" -#include "executor/spi.h" -#include "miscadmin.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList); diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index c713ce099..d95cdd353 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -18,6 +18,7 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "access/genam.h" @@ -25,29 +26,30 @@ #include "catalog/pg_constraint.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" -#include "distributed/coordinator_protocol.h" +#include "foreign/foreign.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/ruleutils.h" +#include "utils/syscache.h" + #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/commands/utility_hook.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata/dependency.h" +#include "distributed/coordinator_protocol.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/namespace_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/ruleutils.h" -#include "utils/syscache.h" -#include "foreign/foreign.h" /* diff --git a/src/backend/distributed/commands/citus_global_signal.c b/src/backend/distributed/commands/citus_global_signal.c index b1f4cf187..23df2d0c1 100644 --- a/src/backend/distributed/commands/citus_global_signal.c +++ b/src/backend/distributed/commands/citus_global_signal.c @@ -11,14 +11,16 @@ #include "postgres.h" +#include "signal.h" + +#include "lib/stringinfo.h" + #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/worker_manager.h" -#include "lib/stringinfo.h" -#include "signal.h" static bool CitusSignalBackend(uint64 globalPID, uint64 timeout, int sig); diff --git a/src/backend/distributed/commands/cluster.c b/src/backend/distributed/commands/cluster.c index cdae6fc08..7a1dac302 100644 --- a/src/backend/distributed/commands/cluster.c +++ b/src/backend/distributed/commands/cluster.c @@ -10,11 +10,11 @@ #include "postgres.h" -#include "pg_version_constants.h" - +#include "catalog/namespace.h" #include "commands/defrem.h" -#include "catalog/namespace.h" +#include "pg_version_constants.h" + #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/listutils.h" diff --git a/src/backend/distributed/commands/collation.c b/src/backend/distributed/commands/collation.c index 521ce4b3d..5ce3d1436 100644 --- a/src/backend/distributed/commands/collation.c +++ b/src/backend/distributed/commands/collation.c @@ -10,30 +10,32 @@ */ #include "postgres.h" -#include "pg_version_compat.h" +#include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/pg_collation.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_executor.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/worker_create_or_replace.h" -#include "pg_version_constants.h" -#include "distributed/worker_manager.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" -#include "miscadmin.h" + +#include "pg_version_compat.h" +#include "pg_version_constants.h" + +#include "distributed/citus_safe_lib.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/worker_create_or_replace.h" +#include "distributed/worker_manager.h" static char * CreateCollationDDLInternal(Oid collationId, Oid *collowner, diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index 797981d47..347a99e8a 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -23,9 +23,9 @@ #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" -#include "distributed/metadata_sync.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 768e20b73..9f3975a1e 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -9,10 +9,8 @@ */ #include "postgres.h" -#include "miscadmin.h" -#include "pg_version_constants.h" -#include "distributed/commands/utility_hook.h" +#include "miscadmin.h" #include "access/genam.h" #include "access/hash.h" @@ -37,42 +35,6 @@ #include "commands/sequence.h" #include "commands/tablecmds.h" #include "commands/trigger.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/distributed_execution_locks.h" -#include "distributed/distribution_column.h" -#include "distributed/listutils.h" -#include "distributed/local_executor.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/pg_dist_colocation.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/remote_commands.h" -#include "distributed/replicate_none_dist_table_shard.h" -#include "distributed/resource_lock.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/shard_split.h" -#include "distributed/shard_transfer.h" -#include "distributed/shared_library_init.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_shard_visibility.h" -#include "distributed/worker_transaction.h" -#include "distributed/utils/distribution_column_map.h" -#include "distributed/version_compat.h" #include "executor/executor.h" #include "executor/spi.h" #include "nodes/execnodes.h" @@ -88,12 +50,51 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" -#include "utils/inval.h" + +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/distributed_execution_locks.h" +#include "distributed/distribution_column.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/pg_dist_colocation.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/remote_commands.h" +#include "distributed/replicate_none_dist_table_shard.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" +#include "distributed/shard_split.h" +#include "distributed/shard_transfer.h" +#include "distributed/shared_library_init.h" +#include "distributed/utils/distribution_column_map.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_shard_visibility.h" +#include "distributed/worker_transaction.h" /* common params that apply to all Citus table types */ diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 78061aa63..a668c9ba6 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -11,12 +11,13 @@ #include "postgres.h" +#include "miscadmin.h" + #include "access/htup_details.h" #include "access/xact.h" #include "catalog/objectaddress.h" #include "catalog/pg_database.h" #include "commands/dbcommands.h" -#include "miscadmin.h" #include "nodes/parsenodes.h" #include "utils/syscache.h" diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 977efb145..735fb16a9 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -10,9 +10,14 @@ #include "postgres.h" +#include "miscadmin.h" + #include "catalog/dependency.h" #include "catalog/objectaddress.h" #include "commands/extension.h" +#include "storage/lmgr.h" +#include "utils/lsyscache.h" + #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" @@ -25,9 +30,6 @@ #include "distributed/remote_commands.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "miscadmin.h" -#include "storage/lmgr.h" -#include "utils/lsyscache.h" static void EnsureDependenciesCanBeDistributed(const ObjectAddress *relationAddress); diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 233831f8e..16f6507df 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -12,11 +12,12 @@ #include "postgres.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" #include "pg_version_constants.h" -#include "distributed/version_compat.h" + +#include "distributed/commands.h" #include "distributed/commands/utility_hook.h" +#include "distributed/deparser.h" +#include "distributed/version_compat.h" static DistributeObjectOps NoDistributeOps = { .deparse = NULL, diff --git a/src/backend/distributed/commands/drop_distributed_table.c b/src/backend/distributed/commands/drop_distributed_table.c index 26579cd60..c3d488b09 100644 --- a/src/backend/distributed/commands/drop_distributed_table.c +++ b/src/backend/distributed/commands/drop_distributed_table.c @@ -9,20 +9,21 @@ */ #include "postgres.h" + #include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" #include "distributed/colocation_utils.h" -#include "distributed/commands/utility_hook.h" #include "distributed/commands.h" -#include "distributed/metadata_utility.h" +#include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_transaction.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" /* local function forward declarations */ diff --git a/src/backend/distributed/commands/extension.c b/src/backend/distributed/commands/extension.c index 5bddf1ede..36267ff66 100644 --- a/src/backend/distributed/commands/extension.c +++ b/src/backend/distributed/commands/extension.c @@ -12,32 +12,35 @@ #include "access/genam.h" #include "access/xact.h" -#include "citus_version.h" #include "catalog/dependency.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension_d.h" -#include "columnar/columnar.h" #include "catalog/pg_foreign_data_wrapper.h" #include "commands/defrem.h" #include "commands/extension.h" +#include "foreign/foreign.h" +#include "nodes/makefuncs.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "citus_version.h" + +#include "columnar/columnar.h" + #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_sync.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/transaction_management.h" -#include "foreign/foreign.h" -#include "nodes/makefuncs.h" -#include "utils/lsyscache.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/syscache.h" /* Local functions forward declarations for helper functions */ diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 709287c56..c1f2b83b6 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -12,27 +12,15 @@ #include "postgres.h" -#include "pg_version_constants.h" +#include "miscadmin.h" +#include "access/genam.h" #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" -#include "access/genam.h" #include "catalog/pg_type.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/commands/sequence.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/multi_join_order.h" -#include "distributed/namespace_utils.h" -#include "distributed/reference_table_utils.h" -#include "distributed/utils/array_type.h" -#include "distributed/version_compat.h" -#include "miscadmin.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" @@ -42,6 +30,19 @@ #include "utils/ruleutils.h" #include "utils/syscache.h" +#include "pg_version_constants.h" + +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/sequence.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/multi_join_order.h" +#include "distributed/namespace_utils.h" +#include "distributed/reference_table_utils.h" +#include "distributed/utils/array_type.h" +#include "distributed/version_compat.h" + #define BehaviorIsRestrictOrNoAction(x) \ ((x) == FKCONSTR_ACTION_NOACTION || (x) == FKCONSTR_ACTION_RESTRICT) diff --git a/src/backend/distributed/commands/foreign_data_wrapper.c b/src/backend/distributed/commands/foreign_data_wrapper.c index c9a08c41a..a181e63a7 100644 --- a/src/backend/distributed/commands/foreign_data_wrapper.c +++ b/src/backend/distributed/commands/foreign_data_wrapper.c @@ -11,17 +11,18 @@ #include "postgres.h" #include "catalog/pg_foreign_data_wrapper.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/distobject.h" #include "foreign/foreign.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "utils/syscache.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" + static bool NameListHasFDWOwnedByDistributedExtension(List *FDWNames); static ObjectAddress GetObjectAddressByFDWName(char *FDWName, bool missing_ok); diff --git a/src/backend/distributed/commands/foreign_server.c b/src/backend/distributed/commands/foreign_server.c index 7d19f9336..d2e575564 100644 --- a/src/backend/distributed/commands/foreign_server.c +++ b/src/backend/distributed/commands/foreign_server.c @@ -9,11 +9,18 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/pg_foreign_server.h" -#include "distributed/commands/utility_hook.h" +#include "foreign/foreign.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "nodes/primnodes.h" +#include "utils/builtins.h" + #include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" @@ -21,11 +28,6 @@ #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" -#include "foreign/foreign.h" -#include "nodes/makefuncs.h" -#include "nodes/parsenodes.h" -#include "nodes/primnodes.h" -#include "utils/builtins.h" static char * GetForeignServerAlterOwnerCommand(Oid serverId); static Node * RecreateForeignServerStmt(Oid serverId); diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 9108d8bd2..3a029b47f 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -18,43 +18,19 @@ */ #include "postgres.h" -#include "miscadmin.h" -#include "funcapi.h" -#include "pg_version_constants.h" +#include "funcapi.h" +#include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" -#include "catalog/pg_aggregate.h" #include "catalog/dependency.h" #include "catalog/namespace.h" +#include "catalog/pg_aggregate.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/extension.h" -#include "distributed/citus_depended_object.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/maintenanced.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata/dependency.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata/pg_dist_object.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_executor.h" -#include "distributed/namespace_utils.h" -#include "distributed/pg_dist_node.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/version_compat.h" -#include "distributed/worker_create_or_replace.h" -#include "distributed/worker_transaction.h" #include "nodes/makefuncs.h" #include "parser/parse_coerce.h" #include "parser/parse_type.h" @@ -63,8 +39,34 @@ #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" -#include "utils/syscache.h" #include "utils/regproc.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + +#include "distributed/citus_depended_object.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/maintenanced.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata/pg_dist_object.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/namespace_utils.h" +#include "distributed/pg_dist_node.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/version_compat.h" +#include "distributed/worker_create_or_replace.h" +#include "distributed/worker_transaction.h" #define DISABLE_LOCAL_CHECK_FUNCTION_BODIES "SET LOCAL check_function_bodies TO off;" #define RESET_CHECK_FUNCTION_BODIES "RESET check_function_bodies;" diff --git a/src/backend/distributed/commands/grant.c b/src/backend/distributed/commands/grant.c index c7861060a..c4278cee1 100644 --- a/src/backend/distributed/commands/grant.c +++ b/src/backend/distributed/commands/grant.c @@ -10,15 +10,16 @@ #include "postgres.h" +#include "lib/stringinfo.h" +#include "nodes/parsenodes.h" +#include "utils/lsyscache.h" + #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/version_compat.h" -#include "lib/stringinfo.h" -#include "nodes/parsenodes.h" -#include "utils/lsyscache.h" /* Local functions forward declarations for helper functions */ diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index bed729992..5874f1780 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -10,7 +10,6 @@ #include "postgres.h" -#include "pg_version_constants.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" @@ -18,32 +17,16 @@ #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" + +#include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "catalog/pg_namespace.h" #endif +#include "miscadmin.h" + #include "commands/defrem.h" #include "commands/tablecmds.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/deparser.h" -#include "distributed/distributed_planner.h" -#include "distributed/listutils.h" -#include "distributed/local_executor.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/namespace_utils.h" -#include "distributed/resource_lock.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/relation_utils.h" -#include "distributed/version_compat.h" -#include "distributed/worker_manager.h" #include "lib/stringinfo.h" -#include "miscadmin.h" #include "nodes/parsenodes.h" #include "parser/parse_utilcmd.h" #include "storage/lmgr.h" @@ -53,6 +36,26 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/deparser.h" +#include "distributed/distributed_planner.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/namespace_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/relation_utils.h" +#include "distributed/resource_lock.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" + /* Local functions forward declarations for helper functions */ static void ErrorIfCreateIndexHasTooManyColumns(IndexStmt *createIndexStatement); diff --git a/src/backend/distributed/commands/local_multi_copy.c b/src/backend/distributed/commands/local_multi_copy.c index 7dbf0ae36..13ff88353 100644 --- a/src/backend/distributed/commands/local_multi_copy.c +++ b/src/backend/distributed/commands/local_multi_copy.c @@ -19,24 +19,27 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "commands/copy.h" -#include "catalog/namespace.h" -#include "parser/parse_relation.h" -#include "utils/lsyscache.h" -#include "nodes/makefuncs.h" -#include "safe_lib.h" #include /* for htons */ -#include "distributed/transmit.h" +#include "postgres.h" + +#include "safe_lib.h" + +#include "catalog/namespace.h" +#include "commands/copy.h" +#include "nodes/makefuncs.h" +#include "parser/parse_relation.h" +#include "utils/lsyscache.h" + #include "distributed/commands/multi_copy.h" #include "distributed/intermediate_results.h" -#include "distributed/multi_partitioning_utils.h" #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" -#include "distributed/shard_utils.h" -#include "distributed/version_compat.h" +#include "distributed/multi_partitioning_utils.h" #include "distributed/replication_origin_session_utils.h" +#include "distributed/shard_utils.h" +#include "distributed/transmit.h" +#include "distributed/version_compat.h" /* managed via GUC, default is 512 kB */ int LocalCopyFlushThresholdByte = 512 * 1024; diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index a5c7a47f4..aaff24a10 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -43,19 +43,18 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "libpq-fe.h" -#include "miscadmin.h" -#include "pgstat.h" - #include /* for htons */ #include /* for htons */ #include -#include "pg_version_constants.h" +#include "postgres.h" + +#include "libpq-fe.h" +#include "miscadmin.h" +#include "pgstat.h" -#include "access/htup_details.h" #include "access/htup.h" +#include "access/htup_details.h" #include "access/sdir.h" #include "access/sysattr.h" #include "access/xact.h" @@ -65,41 +64,30 @@ #include "commands/copy.h" #include "commands/defrem.h" #include "commands/progress.h" + +#include "pg_version_constants.h" + #include "distributed/citus_safe_lib.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" +#include "distributed/locally_reserved_shared_connections.h" #include "distributed/log_utils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" -#include "distributed/multi_executor.h" -#include "distributed/listutils.h" -#include "distributed/locally_reserved_shared_connections.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "distributed/relation_utils.h" #endif -#include "distributed/remote_commands.h" -#include "distributed/remote_transaction.h" -#include "distributed/replication_origin_session_utils.h" -#include "distributed/resource_lock.h" -#include "distributed/shard_pruning.h" -#include "distributed/shared_connection_stats.h" -#include "distributed/version_compat.h" -#include "distributed/worker_protocol.h" -#include "distributed/local_multi_copy.h" -#include "distributed/hash_helpers.h" -#include "distributed/transmit.h" #include "executor/executor.h" #include "foreign/foreign.h" - #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "nodes/makefuncs.h" @@ -110,9 +98,21 @@ #include "tsearch/ts_locale.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/memutils.h" #include "utils/rel.h" #include "utils/syscache.h" -#include "utils/memutils.h" + +#include "distributed/hash_helpers.h" +#include "distributed/local_multi_copy.h" +#include "distributed/remote_commands.h" +#include "distributed/remote_transaction.h" +#include "distributed/replication_origin_session_utils.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_pruning.h" +#include "distributed/shared_connection_stats.h" +#include "distributed/transmit.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" /* constant used in binary protocol */ diff --git a/src/backend/distributed/commands/owned.c b/src/backend/distributed/commands/owned.c index c8f6a4bbe..3b4b043f8 100644 --- a/src/backend/distributed/commands/owned.c +++ b/src/backend/distributed/commands/owned.c @@ -10,33 +10,20 @@ #include "postgres.h" +#include "miscadmin.h" + +#include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" -#include "access/genam.h" #include "access/table.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "catalog/objectaddress.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_db_role_setting.h" #include "catalog/pg_type.h" -#include "catalog/objectaddress.h" #include "commands/dbcommands.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/distobject.h" -#include "distributed/multi_executor.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/version_compat.h" -#include "distributed/worker_transaction.h" -#include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" @@ -44,8 +31,22 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/rel.h" -#include "utils/varlena.h" #include "utils/syscache.h" +#include "utils/varlena.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/version_compat.h" +#include "distributed/worker_transaction.h" /* * PreprocessDropOwnedStmt finds the distributed role out of the ones diff --git a/src/backend/distributed/commands/policy.c b/src/backend/distributed/commands/policy.c index 0d66e150e..a2a926b66 100644 --- a/src/backend/distributed/commands/policy.c +++ b/src/backend/distributed/commands/policy.c @@ -10,15 +10,10 @@ */ #include "postgres.h" +#include "miscadmin.h" + #include "catalog/namespace.h" #include "commands/policy.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "miscadmin.h" #include "nodes/makefuncs.h" #include "parser/parse_clause.h" #include "parser/parse_relation.h" @@ -27,6 +22,13 @@ #include "utils/builtins.h" #include "utils/ruleutils.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" + static const char * unparse_policy_command(const char aclchar); static RowSecurityPolicy * GetPolicyByName(Oid relationId, const char *policyName); diff --git a/src/backend/distributed/commands/publication.c b/src/backend/distributed/commands/publication.c index 581f7f874..67d33832c 100644 --- a/src/backend/distributed/commands/publication.c +++ b/src/backend/distributed/commands/publication.c @@ -9,18 +9,11 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_rel.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/distobject.h" -#include "distributed/reference_table_utils.h" -#include "distributed/worker_create_or_replace.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" @@ -29,6 +22,15 @@ #include "pg_version_compat.h" +#include "distributed/commands.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/reference_table_utils.h" +#include "distributed/worker_create_or_replace.h" + static CreatePublicationStmt * BuildCreatePublicationStmt(Oid publicationId); #if (PG_VERSION_NUM >= PG_VERSION_15) diff --git a/src/backend/distributed/commands/rename.c b/src/backend/distributed/commands/rename.c index 5e313d68c..362fc57bb 100644 --- a/src/backend/distributed/commands/rename.c +++ b/src/backend/distributed/commands/rename.c @@ -12,11 +12,12 @@ #include "catalog/index.h" #include "catalog/namespace.h" +#include "nodes/parsenodes.h" +#include "utils/lsyscache.h" + #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/metadata_cache.h" -#include "nodes/parsenodes.h" -#include "utils/lsyscache.h" /* diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 9f819e950..85af6e368 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -10,37 +10,20 @@ #include "postgres.h" -#include "pg_version_compat.h" - -#include "pg_version_constants.h" +#include "miscadmin.h" +#include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" -#include "access/genam.h" #include "access/table.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "catalog/objectaddress.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_db_role_setting.h" #include "catalog/pg_type.h" -#include "catalog/objectaddress.h" #include "commands/dbcommands.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/distobject.h" -#include "distributed/multi_executor.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/version_compat.h" -#include "distributed/worker_transaction.h" -#include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" @@ -48,11 +31,28 @@ #include "utils/acl.h" #include "utils/builtins.h" #include "utils/fmgroids.h" -#include "utils/guc_tables.h" #include "utils/guc.h" +#include "utils/guc_tables.h" #include "utils/rel.h" -#include "utils/varlena.h" #include "utils/syscache.h" +#include "utils/varlena.h" + +#include "pg_version_compat.h" +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/version_compat.h" +#include "distributed/worker_transaction.h" static const char * ExtractEncryptedPassword(Oid roleOid); static const char * CreateAlterRoleIfExistsCommand(AlterRoleStmt *stmt); diff --git a/src/backend/distributed/commands/schema.c b/src/backend/distributed/commands/schema.c index d48a73647..7f79897fa 100644 --- a/src/backend/distributed/commands/schema.c +++ b/src/backend/distributed/commands/schema.c @@ -19,28 +19,28 @@ #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_namespace.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include -#include "distributed/commands/utility_hook.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata_cache.h" -#include -#include "distributed/multi_executor.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/resource_lock.h" -#include -#include -#include "distributed/tenant_schema_metadata.h" -#include "distributed/version_compat.h" #include "nodes/parsenodes.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/relcache.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/connection_management.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/tenant_schema_metadata.h" +#include "distributed/version_compat.h" + static List * GetObjectAddressBySchemaName(char *schemaName, bool missing_ok); static List * FilterDistributedSchemas(List *schemas); diff --git a/src/backend/distributed/commands/schema_based_sharding.c b/src/backend/distributed/commands/schema_based_sharding.c index 65d2b8127..7cde96982 100644 --- a/src/backend/distributed/commands/schema_based_sharding.c +++ b/src/backend/distributed/commands/schema_based_sharding.c @@ -7,27 +7,29 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "access/genam.h" #include "catalog/catalog.h" #include "catalog/pg_namespace_d.h" #include "commands/extension.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + #include "distributed/argutils.h" #include "distributed/backend_data.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/listutils.h" -#include "distributed/metadata_sync.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/shard_transfer.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_shard_visibility.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" /* return value of CreateCitusMoveSchemaParams() */ diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index 9ff586c8c..4d838a882 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -16,6 +16,12 @@ #include "catalog/namespace.h" #include "commands/defrem.h" #include "commands/extension.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "rewrite/rewriteHandler.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" + #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/commands/utility_hook.h" @@ -24,12 +30,7 @@ #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" -#include "nodes/makefuncs.h" #include "distributed/worker_create_or_replace.h" -#include "nodes/parsenodes.h" -#include "rewrite/rewriteHandler.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" /* Local functions forward declarations for helper functions */ static bool OptionsSpecifyOwnedBy(List *optionList, Oid *ownedByTableId); diff --git a/src/backend/distributed/commands/statistics.c b/src/backend/distributed/commands/statistics.c index dae72ada9..5fac767fd 100644 --- a/src/backend/distributed/commands/statistics.c +++ b/src/backend/distributed/commands/statistics.c @@ -19,6 +19,8 @@ #include "postgres.h" +#include "miscadmin.h" + #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" @@ -26,8 +28,16 @@ #include "catalog/pg_namespace.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_type.h" -#include "distributed/commands/utility_hook.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/fmgrprotos.h" +#include "utils/lsyscache.h" +#include "utils/relcache.h" +#include "utils/ruleutils.h" +#include "utils/syscache.h" + #include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/listutils.h" @@ -37,14 +47,6 @@ #include "distributed/relation_access_tracking.h" #include "distributed/resource_lock.h" #include "distributed/worker_transaction.h" -#include "miscadmin.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/fmgrprotos.h" -#include "utils/lsyscache.h" -#include "utils/relcache.h" -#include "utils/ruleutils.h" -#include "utils/syscache.h" #define DEFAULT_STATISTICS_TARGET -1 #define ALTER_INDEX_COLUMN_SET_STATS_COMMAND \ diff --git a/src/backend/distributed/commands/subscription.c b/src/backend/distributed/commands/subscription.c index 52519b680..f5f80d17a 100644 --- a/src/backend/distributed/commands/subscription.c +++ b/src/backend/distributed/commands/subscription.c @@ -8,21 +8,23 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "safe_lib.h" - #include -#include "commands/defrem.h" -#include "distributed/commands.h" -#include "distributed/connection_management.h" -#include "pg_version_constants.h" -#include "distributed/version_compat.h" +#include "postgres.h" + #include "libpq-fe.h" +#include "safe_lib.h" + +#include "commands/defrem.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" +#include "pg_version_constants.h" + +#include "distributed/commands.h" +#include "distributed/connection_management.h" +#include "distributed/version_compat.h" + static char * GenerateConninfoWithAuth(char *conninfo); diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index e8404d38c..074a789ed 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -9,7 +9,7 @@ */ #include "postgres.h" -#include "pg_version_constants.h" + #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" @@ -20,28 +20,6 @@ #include "catalog/pg_depend.h" #include "catalog/pg_type.h" #include "commands/tablecmds.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/deparser.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/distribution_column.h" -#include "distributed/foreign_key_relationship.h" -#include "distributed/local_executor.h" -#include "distributed/listutils.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/resource_lock.h" -#include "distributed/version_compat.h" -#include "distributed/worker_shard_visibility.h" -#include "distributed/tenant_schema_metadata.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" @@ -53,6 +31,31 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/deparser.h" +#include "distributed/distribution_column.h" +#include "distributed/foreign_key_relationship.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/resource_lock.h" +#include "distributed/tenant_schema_metadata.h" +#include "distributed/version_compat.h" +#include "distributed/worker_shard_visibility.h" + /* controlled via GUC, should be accessed via GetEnableLocalReferenceForeignKeys() */ bool EnableLocalReferenceForeignKeys = true; diff --git a/src/backend/distributed/commands/text_search.c b/src/backend/distributed/commands/text_search.c index 54dfdae85..4a386e321 100644 --- a/src/backend/distributed/commands/text_search.c +++ b/src/backend/distributed/commands/text_search.c @@ -10,6 +10,8 @@ #include "postgres.h" +#include "fmgr.h" + #include "access/genam.h" #include "access/xact.h" #include "catalog/namespace.h" @@ -22,7 +24,6 @@ #include "commands/comment.h" #include "commands/defrem.h" #include "commands/extension.h" -#include "fmgr.h" #include "nodes/makefuncs.h" #include "tsearch/ts_cache.h" #include "tsearch/ts_public.h" diff --git a/src/backend/distributed/commands/trigger.c b/src/backend/distributed/commands/trigger.c index 0ec8287f5..74cb6259f 100644 --- a/src/backend/distributed/commands/trigger.c +++ b/src/backend/distributed/commands/trigger.c @@ -9,7 +9,6 @@ *------------------------------------------------------------------------- */ #include "postgres.h" -#include "pg_version_constants.h" #include "access/genam.h" #include "access/table.h" @@ -18,6 +17,14 @@ #include "catalog/pg_trigger.h" #include "commands/extension.h" #include "commands/trigger.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/fmgrprotos.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" @@ -29,11 +36,6 @@ #include "distributed/namespace_utils.h" #include "distributed/shard_utils.h" #include "distributed/worker_protocol.h" -#include "utils/builtins.h" -#include "utils/fmgrprotos.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" /* appropriate lock modes for the owner relation according to postgres */ diff --git a/src/backend/distributed/commands/truncate.c b/src/backend/distributed/commands/truncate.c index 4de518a06..0eb43f529 100644 --- a/src/backend/distributed/commands/truncate.c +++ b/src/backend/distributed/commands/truncate.c @@ -9,12 +9,19 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "commands/tablecmds.h" #include "commands/trigger.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/regproc.h" +#include "utils/rel.h" + #include "distributed/adaptive_executor.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" @@ -31,13 +38,8 @@ #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" -#include "distributed/worker_transaction.h" #include "distributed/worker_shard_visibility.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/regproc.h" -#include "utils/rel.h" +#include "distributed/worker_transaction.h" /* Local functions forward declarations for unsupported command checks */ diff --git a/src/backend/distributed/commands/type.c b/src/backend/distributed/commands/type.c index ccb7bf528..b1e573638 100644 --- a/src/backend/distributed/commands/type.c +++ b/src/backend/distributed/commands/type.c @@ -43,7 +43,7 @@ #include "postgres.h" -#include "pg_version_constants.h" +#include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" @@ -52,6 +52,18 @@ #include "catalog/pg_enum.h" #include "catalog/pg_type.h" #include "commands/extension.h" +#include "nodes/makefuncs.h" +#include "parser/parse_type.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/regproc.h" +#include "utils/syscache.h" +#include "utils/typcache.h" + +#include "pg_version_constants.h" + #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" @@ -64,20 +76,10 @@ #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/transaction_management.h" -#include "distributed/worker_create_or_replace.h" #include "distributed/version_compat.h" +#include "distributed/worker_create_or_replace.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "miscadmin.h" -#include "nodes/makefuncs.h" -#include "parser/parse_type.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/regproc.h" -#include "utils/syscache.h" -#include "utils/typcache.h" #define AlterEnumIsRename(stmt) (stmt->oldVal != NULL) #define AlterEnumIsAddValue(stmt) (stmt->oldVal == NULL) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index a4a718733..e1bbd5972 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -25,9 +25,8 @@ *------------------------------------------------------------------------- */ -#include "pg_version_constants.h" - #include "postgres.h" + #include "miscadmin.h" #include "access/attnum.h" @@ -35,11 +34,25 @@ #include "access/htup_details.h" #include "catalog/catalog.h" #include "catalog/dependency.h" -#include "citus_version.h" #include "commands/dbcommands.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/tablecmds.h" +#include "foreign/foreign.h" +#include "lib/stringinfo.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "nodes/pg_list.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "citus_version.h" +#include "pg_version_constants.h" + #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_depended_object.h" @@ -48,19 +61,19 @@ #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" /* IWYU pragma: keep */ #include "distributed/coordinator_protocol.h" -#include "distributed/deparser.h" #include "distributed/deparse_shard_query.h" +#include "distributed/deparser.h" #include "distributed/executor_util.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/maintenanced.h" -#include "distributed/multi_logical_replication.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/metadata_sync.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" +#include "distributed/multi_logical_replication.h" +#include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" @@ -69,17 +82,6 @@ #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" -#include "foreign/foreign.h" -#include "lib/stringinfo.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" -#include "nodes/makefuncs.h" -#include "tcop/utility.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/inval.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" bool EnableDDLPropagation = true; /* ddl propagation is enabled */ diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index ee1594e1d..6250d92b4 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -10,10 +10,16 @@ #include "postgres.h" -#include "pg_version_constants.h" - +#include "access/xact.h" #include "commands/defrem.h" #include "commands/vacuum.h" +#include "postmaster/bgworker_internals.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" + +#include "pg_version_constants.h" + #include "distributed/adaptive_executor.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" @@ -24,11 +30,6 @@ #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "postmaster/bgworker_internals.h" -#include "access/xact.h" #define VACUUM_PARALLEL_NOTSET -2 diff --git a/src/backend/distributed/commands/variableset.c b/src/backend/distributed/commands/variableset.c index 277f5b63f..2a3bc2f67 100644 --- a/src/backend/distributed/commands/variableset.c +++ b/src/backend/distributed/commands/variableset.c @@ -9,21 +9,23 @@ */ #include "postgres.h" + #include "c.h" #include "common/string.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/metadata_cache.h" -#include "distributed/resource_lock.h" -#include "distributed/transaction_management.h" -#include "distributed/version_compat.h" +#include "lib/ilist.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" -#include "lib/ilist.h" #include "utils/varlena.h" + +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/transaction_management.h" +#include "distributed/version_compat.h" /* * ShouldPropagateSetCommand determines whether a SET or RESET command should be diff --git a/src/backend/distributed/commands/view.c b/src/backend/distributed/commands/view.c index 7c4816144..0c39be4ca 100644 --- a/src/backend/distributed/commands/view.c +++ b/src/backend/distributed/commands/view.c @@ -9,23 +9,12 @@ */ #include "postgres.h" + #include "fmgr.h" #include "access/genam.h" #include "catalog/objectaddress.h" #include "commands/extension.h" -#include "distributed/commands.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/deparser.h" -#include "distributed/errormessage.h" -#include "distributed/listutils.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/multi_executor.h" -#include "distributed/namespace_utils.h" -#include "distributed/worker_transaction.h" #include "executor/spi.h" #include "nodes/nodes.h" #include "nodes/pg_list.h" @@ -35,6 +24,19 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/deparser.h" +#include "distributed/errormessage.h" +#include "distributed/listutils.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/namespace_utils.h" +#include "distributed/worker_transaction.h" + /* * GUC controls some restrictions for local objects. For example, * if it is disabled, a local view with no distributed relation dependency diff --git a/src/backend/distributed/connection/connection_configuration.c b/src/backend/distributed/connection/connection_configuration.c index bf61f7fac..c6a34a9d7 100644 --- a/src/backend/distributed/connection/connection_configuration.c +++ b/src/backend/distributed/connection/connection_configuration.c @@ -12,6 +12,10 @@ #include "access/transam.h" #include "access/xact.h" +#include "mb/pg_wchar.h" +#include "postmaster/postmaster.h" +#include "utils/builtins.h" + #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" @@ -19,10 +23,6 @@ #include "distributed/metadata_cache.h" #include "distributed/worker_manager.h" -#include "postmaster/postmaster.h" -#include "mb/pg_wchar.h" -#include "utils/builtins.h" - /* stores the string representation of our node connection GUC */ #ifdef USE_SSL char *NodeConninfo = "sslmode=require"; diff --git a/src/backend/distributed/connection/connection_management.c b/src/backend/distributed/connection/connection_management.c index 9439b38c5..64ec1904f 100644 --- a/src/backend/distributed/connection/connection_management.c +++ b/src/backend/distributed/connection/connection_management.c @@ -9,39 +9,39 @@ */ #include "postgres.h" -#include "pgstat.h" #include "libpq-fe.h" - #include "miscadmin.h" - +#include "pg_config.h" +#include "pgstat.h" #include "safe_lib.h" -#include "postmaster/postmaster.h" + #include "access/hash.h" #include "commands/dbcommands.h" +#include "mb/pg_wchar.h" +#include "portability/instr_time.h" +#include "postmaster/postmaster.h" +#include "storage/ipc.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" + #include "distributed/backend_data.h" +#include "distributed/cancel_utils.h" #include "distributed/connection_management.h" -#include "distributed/errormessage.h" #include "distributed/error_codes.h" +#include "distributed/errormessage.h" +#include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/memutils.h" #include "distributed/metadata_cache.h" -#include "distributed/hash_helpers.h" #include "distributed/placement_connection.h" +#include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" #include "distributed/shared_connection_stats.h" -#include "distributed/cancel_utils.h" -#include "distributed/remote_commands.h" #include "distributed/time_constants.h" #include "distributed/version_compat.h" #include "distributed/worker_log_messages.h" -#include "mb/pg_wchar.h" -#include "pg_config.h" -#include "portability/instr_time.h" -#include "storage/ipc.h" -#include "utils/hsearch.h" -#include "utils/memutils.h" int NodeConnectionTimeout = 30000; diff --git a/src/backend/distributed/connection/locally_reserved_shared_connections.c b/src/backend/distributed/connection/locally_reserved_shared_connections.c index d3138c734..69b03c116 100644 --- a/src/backend/distributed/connection/locally_reserved_shared_connections.c +++ b/src/backend/distributed/connection/locally_reserved_shared_connections.c @@ -33,12 +33,15 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "miscadmin.h" #include "access/hash.h" #include "commands/dbcommands.h" +#include "common/hashfn.h" +#include "utils/builtins.h" + +#include "pg_version_constants.h" + #include "distributed/listutils.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/metadata_cache.h" @@ -47,8 +50,6 @@ #include "distributed/shared_connection_stats.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" -#include "utils/builtins.h" -#include "common/hashfn.h" #define RESERVED_CONNECTION_COLUMNS 4 diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c index 3924e5a05..10c99bd80 100644 --- a/src/backend/distributed/connection/placement_connection.c +++ b/src/backend/distributed/connection/placement_connection.c @@ -11,23 +11,24 @@ #include "postgres.h" +#include "access/hash.h" +#include "common/hashfn.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" + #include "pg_version_constants.h" -#include "access/hash.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" -#include "distributed/distributed_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" -#include "utils/hsearch.h" -#include "common/hashfn.h" -#include "utils/memutils.h" /* diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index 15dd985ec..f694ff390 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -9,23 +9,23 @@ */ #include "postgres.h" -#include "pgstat.h" #include "libpq-fe.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "lib/stringinfo.h" +#include "storage/latch.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/palloc.h" + +#include "distributed/cancel_utils.h" #include "distributed/connection_management.h" #include "distributed/errormessage.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/remote_commands.h" -#include "distributed/errormessage.h" -#include "distributed/cancel_utils.h" -#include "lib/stringinfo.h" -#include "miscadmin.h" -#include "storage/latch.h" -#include "utils/builtins.h" -#include "utils/fmgrprotos.h" -#include "utils/palloc.h" /* diff --git a/src/backend/distributed/connection/shared_connection_stats.c b/src/backend/distributed/connection/shared_connection_stats.c index 104caed07..26598b465 100644 --- a/src/backend/distributed/connection/shared_connection_stats.c +++ b/src/backend/distributed/connection/shared_connection_stats.c @@ -11,18 +11,21 @@ */ #include "postgres.h" -#include "pgstat.h" - -#include "pg_version_constants.h" #include "libpq-fe.h" - #include "miscadmin.h" +#include "pgstat.h" #include "access/hash.h" #include "access/htup_details.h" #include "catalog/pg_authid.h" #include "commands/dbcommands.h" +#include "common/hashfn.h" +#include "storage/ipc.h" +#include "utils/builtins.h" + +#include "pg_version_constants.h" + #include "distributed/backend_data.h" #include "distributed/cancel_utils.h" #include "distributed/connection_management.h" @@ -32,12 +35,9 @@ #include "distributed/multi_executor.h" #include "distributed/placement_connection.h" #include "distributed/shared_connection_stats.h" -#include "distributed/worker_manager.h" #include "distributed/time_constants.h" #include "distributed/tuplestore.h" -#include "utils/builtins.h" -#include "common/hashfn.h" -#include "storage/ipc.h" +#include "distributed/worker_manager.h" #define REMOTE_CONNECTION_STATS_COLUMNS 4 diff --git a/src/backend/distributed/connection/worker_log_messages.c b/src/backend/distributed/connection/worker_log_messages.c index 9c240620e..9b64b81a6 100644 --- a/src/backend/distributed/connection/worker_log_messages.c +++ b/src/backend/distributed/connection/worker_log_messages.c @@ -10,12 +10,13 @@ #include "postgres.h" +#include "utils/elog.h" + #include "distributed/connection_management.h" #include "distributed/error_codes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/worker_log_messages.h" -#include "utils/elog.h" /* diff --git a/src/backend/distributed/deparser/citus_grantutils.c b/src/backend/distributed/deparser/citus_grantutils.c index 8e0dadff2..c944013f6 100644 --- a/src/backend/distributed/deparser/citus_grantutils.c +++ b/src/backend/distributed/deparser/citus_grantutils.c @@ -1,8 +1,10 @@ #include "postgres.h" + #include "lib/stringinfo.h" #include "nodes/parsenodes.h" -#include "distributed/deparser.h" + #include "distributed/citus_ruleutils.h" +#include "distributed/deparser.h" /* * Append the 'WITH GRANT OPTION' clause to the given buffer if the given diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 1456f2fb5..f99462058 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -7,13 +7,12 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "miscadmin.h" - -#include "pg_version_constants.h" - #include +#include "postgres.h" + +#include "miscadmin.h" + #include "access/attnum.h" #include "access/genam.h" #include "access/heapam.h" @@ -39,21 +38,11 @@ #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata_utility.h" -#include "distributed/namespace_utils.h" -#include "distributed/relay_utility.h" -#include "distributed/version_compat.h" -#include "distributed/worker_protocol.h" +#include "commands/sequence.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" -#include "nodes/nodes.h" #include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "parser/parse_utilcmd.h" @@ -71,7 +60,20 @@ #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" -#include "commands/sequence.h" + +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/namespace_utils.h" +#include "distributed/relay_utility.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" static void deparse_index_columns(StringInfo buffer, List *indexParameterList, diff --git a/src/backend/distributed/deparser/deparse_collation_stmts.c b/src/backend/distributed/deparser/deparse_collation_stmts.c index 44f7f9098..3a568d2ad 100644 --- a/src/backend/distributed/deparser/deparse_collation_stmts.c +++ b/src/backend/distributed/deparser/deparse_collation_stmts.c @@ -17,8 +17,8 @@ #include "nodes/value.h" #include "utils/builtins.h" -#include "distributed/deparser.h" #include "distributed/citus_ruleutils.h" +#include "distributed/deparser.h" static void AppendDropCollationStmt(StringInfo buf, DropStmt *stmt); static void AppendRenameCollationStmt(StringInfo buf, RenameStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 8a24f738a..596d10713 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -11,16 +11,15 @@ #include "postgres.h" -#include "pg_version_compat.h" - #include "catalog/namespace.h" +#include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" -#include "distributed/deparser.h" +#include "pg_version_compat.h" + #include "distributed/citus_ruleutils.h" -#include "commands/defrem.h" #include "distributed/deparser.h" #include "distributed/log_utils.h" diff --git a/src/backend/distributed/deparser/deparse_extension_stmts.c b/src/backend/distributed/deparser/deparse_extension_stmts.c index bb6b15dbd..92d54602f 100644 --- a/src/backend/distributed/deparser/deparse_extension_stmts.c +++ b/src/backend/distributed/deparser/deparse_extension_stmts.c @@ -14,13 +14,14 @@ #include "catalog/namespace.h" #include "commands/defrem.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "utils/builtins.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + /* Local functions forward declarations for helper functions */ static void AppendCreateExtensionStmt(StringInfo buf, CreateExtensionStmt *stmt); static void AppendCreateExtensionStmtOptions(StringInfo buf, List *options); diff --git a/src/backend/distributed/deparser/deparse_foreign_data_wrapper_stmts.c b/src/backend/distributed/deparser/deparse_foreign_data_wrapper_stmts.c index 3f755c905..fab1cc7ab 100644 --- a/src/backend/distributed/deparser/deparse_foreign_data_wrapper_stmts.c +++ b/src/backend/distributed/deparser/deparse_foreign_data_wrapper_stmts.c @@ -10,13 +10,14 @@ #include "postgres.h" #include "commands/defrem.h" +#include "lib/stringinfo.h" +#include "nodes/nodes.h" +#include "utils/builtins.h" + #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" -#include "lib/stringinfo.h" -#include "nodes/nodes.h" -#include "utils/builtins.h" static void AppendGrantOnFDWStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnFDWNames(StringInfo buf, GrantStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_foreign_server_stmts.c b/src/backend/distributed/deparser/deparse_foreign_server_stmts.c index 403569b94..9c708a771 100644 --- a/src/backend/distributed/deparser/deparse_foreign_server_stmts.c +++ b/src/backend/distributed/deparser/deparse_foreign_server_stmts.c @@ -10,13 +10,14 @@ #include "postgres.h" #include "commands/defrem.h" +#include "lib/stringinfo.h" +#include "nodes/nodes.h" +#include "utils/builtins.h" + #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" -#include "lib/stringinfo.h" -#include "nodes/nodes.h" -#include "utils/builtins.h" static void AppendCreateForeignServerStmt(StringInfo buf, CreateForeignServerStmt *stmt); static void AppendAlterForeignServerStmt(StringInfo buf, AlterForeignServerStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_function_stmts.c b/src/backend/distributed/deparser/deparse_function_stmts.c index c1d7d3128..fbd1d25e6 100644 --- a/src/backend/distributed/deparser/deparse_function_stmts.c +++ b/src/backend/distributed/deparser/deparse_function_stmts.c @@ -22,10 +22,6 @@ #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/defrem.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/version_compat.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" @@ -38,8 +34,13 @@ #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" -#include "utils/syscache.h" #include "utils/regproc.h" +#include "utils/syscache.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/deparser.h" +#include "distributed/version_compat.h" /* forward declaration for deparse functions */ diff --git a/src/backend/distributed/deparser/deparse_owned_stmts.c b/src/backend/distributed/deparser/deparse_owned_stmts.c index 888071165..af7fa0968 100644 --- a/src/backend/distributed/deparser/deparse_owned_stmts.c +++ b/src/backend/distributed/deparser/deparse_owned_stmts.c @@ -11,13 +11,14 @@ #include "postgres.h" +#include "lib/stringinfo.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" -#include "lib/stringinfo.h" -#include "nodes/parsenodes.h" -#include "utils/builtins.h" static void AppendDropOwnedStmt(StringInfo buf, DropOwnedStmt *stmt); static void AppendRoleList(StringInfo buf, List *roleList); diff --git a/src/backend/distributed/deparser/deparse_publication_stmts.c b/src/backend/distributed/deparser/deparse_publication_stmts.c index e22333146..8e3118171 100644 --- a/src/backend/distributed/deparser/deparse_publication_stmts.c +++ b/src/backend/distributed/deparser/deparse_publication_stmts.c @@ -13,20 +13,21 @@ #include "access/relation.h" #include "catalog/namespace.h" #include "commands/defrem.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" -#include "distributed/namespace_utils.h" #include "lib/stringinfo.h" +#include "nodes/value.h" #include "parser/parse_clause.h" #include "parser/parse_collate.h" #include "parser/parse_node.h" #include "parser/parse_relation.h" -#include "nodes/value.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/namespace_utils.h" + static void AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, bool whereClauseNeedsTransform, diff --git a/src/backend/distributed/deparser/deparse_role_stmts.c b/src/backend/distributed/deparser/deparse_role_stmts.c index 4d41f8ec4..0194d0d67 100644 --- a/src/backend/distributed/deparser/deparse_role_stmts.c +++ b/src/backend/distributed/deparser/deparse_role_stmts.c @@ -13,16 +13,17 @@ #include "postgres.h" -#include "pg_version_compat.h" - #include "commands/defrem.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" +#include "pg_version_compat.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + static void AppendAlterRoleStmt(StringInfo buf, AlterRoleStmt *stmt); static void AppendAlterRoleSetStmt(StringInfo buf, AlterRoleSetStmt *stmt); static void AppendCreateRoleStmt(StringInfo buf, CreateRoleStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_schema_stmts.c b/src/backend/distributed/deparser/deparse_schema_stmts.c index 10317b899..0a9c49801 100644 --- a/src/backend/distributed/deparser/deparse_schema_stmts.c +++ b/src/backend/distributed/deparser/deparse_schema_stmts.c @@ -12,13 +12,14 @@ */ #include "postgres.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "utils/builtins.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + static void AppendCreateSchemaStmt(StringInfo buf, CreateSchemaStmt *stmt); static void AppendDropSchemaStmt(StringInfo buf, DropStmt *stmt); static void AppendGrantOnSchemaStmt(StringInfo buf, GrantStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_sequence_stmts.c b/src/backend/distributed/deparser/deparse_sequence_stmts.c index 98488c160..9e5fab2c8 100644 --- a/src/backend/distributed/deparser/deparse_sequence_stmts.c +++ b/src/backend/distributed/deparser/deparse_sequence_stmts.c @@ -14,12 +14,13 @@ #include "postgres.h" #include "catalog/namespace.h" -#include "distributed/deparser.h" -#include "distributed/version_compat.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "distributed/deparser.h" +#include "distributed/version_compat.h" + /* forward declaration for deparse functions */ static void AppendDropSequenceStmt(StringInfo buf, DropStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 599738dc5..99b9d1c2d 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -12,16 +12,17 @@ */ #include "postgres.h" +#include "catalog/namespace.h" +#include "lib/stringinfo.h" +#include "nodes/nodes.h" +#include "utils/builtins.h" + #include "pg_version_constants.h" -#include "catalog/namespace.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" -#include "lib/stringinfo.h" -#include "nodes/nodes.h" -#include "utils/builtins.h" static void AppendCreateStatisticsStmt(StringInfo buf, CreateStatsStmt *stmt); static void AppendDropStatisticsStmt(StringInfo buf, List *nameList, bool ifExists); diff --git a/src/backend/distributed/deparser/deparse_table_stmts.c b/src/backend/distributed/deparser/deparse_table_stmts.c index ff96d7fc3..e976b0e2f 100644 --- a/src/backend/distributed/deparser/deparse_table_stmts.c +++ b/src/backend/distributed/deparser/deparse_table_stmts.c @@ -13,20 +13,20 @@ #include "catalog/heap.h" #include "commands/defrem.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/version_compat.h" +#include "commands/tablecmds.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "parser/parse_expr.h" -#include "parser/parse_type.h" #include "parser/parse_relation.h" +#include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" +#include "distributed/commands.h" +#include "distributed/deparser.h" #include "distributed/namespace_utils.h" -#include "commands/tablecmds.h" +#include "distributed/version_compat.h" static void AppendAlterTableSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterTableStmt(StringInfo buf, AlterTableStmt *stmt); diff --git a/src/backend/distributed/deparser/deparse_view_stmts.c b/src/backend/distributed/deparser/deparse_view_stmts.c index 39c4ccb63..5592aec9d 100644 --- a/src/backend/distributed/deparser/deparse_view_stmts.c +++ b/src/backend/distributed/deparser/deparse_view_stmts.c @@ -13,15 +13,16 @@ #include "catalog/namespace.h" #include "commands/defrem.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + static void AppendDropViewStmt(StringInfo buf, DropStmt *stmt); static void AppendViewNameList(StringInfo buf, List *objects); static void AppendAlterViewStmt(StringInfo buf, AlterTableStmt *stmt); diff --git a/src/backend/distributed/deparser/objectaddress.c b/src/backend/distributed/deparser/objectaddress.c index d835a3b1a..6718c22cf 100644 --- a/src/backend/distributed/deparser/objectaddress.c +++ b/src/backend/distributed/deparser/objectaddress.c @@ -12,11 +12,12 @@ #include "postgres.h" -#include "commands/extension.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" #include "catalog/objectaddress.h" #include "catalog/pg_extension_d.h" +#include "commands/extension.h" + +#include "distributed/commands.h" +#include "distributed/deparser.h" /* diff --git a/src/backend/distributed/deparser/qualify_aggregate_stmts.c b/src/backend/distributed/deparser/qualify_aggregate_stmts.c index 9debc244a..e5d7210f3 100644 --- a/src/backend/distributed/deparser/qualify_aggregate_stmts.c +++ b/src/backend/distributed/deparser/qualify_aggregate_stmts.c @@ -15,10 +15,11 @@ #include "postgres.h" #include "catalog/namespace.h" -#include "distributed/deparser.h" #include "nodes/makefuncs.h" #include "utils/lsyscache.h" +#include "distributed/deparser.h" + void QualifyDefineAggregateStmt(Node *node) { diff --git a/src/backend/distributed/deparser/qualify_function_stmt.c b/src/backend/distributed/deparser/qualify_function_stmt.c index fbd6c17a0..184ff92bf 100644 --- a/src/backend/distributed/deparser/qualify_function_stmt.c +++ b/src/backend/distributed/deparser/qualify_function_stmt.c @@ -21,12 +21,13 @@ #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_proc.h" -#include "distributed/deparser.h" -#include "distributed/version_compat.h" #include "parser/parse_func.h" #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/deparser.h" +#include "distributed/version_compat.h" + /* forward declaration for qualify functions */ static void QualifyFunction(ObjectWithArgs *func, ObjectType type); static void QualifyFunctionSchemaName(ObjectWithArgs *func, ObjectType type); diff --git a/src/backend/distributed/deparser/qualify_publication_stmt.c b/src/backend/distributed/deparser/qualify_publication_stmt.c index 3231fe363..73ffe3a35 100644 --- a/src/backend/distributed/deparser/qualify_publication_stmt.c +++ b/src/backend/distributed/deparser/qualify_publication_stmt.c @@ -12,12 +12,13 @@ #include "postgres.h" #include "catalog/namespace.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "nodes/nodes.h" #include "utils/guc.h" #include "utils/lsyscache.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + #if (PG_VERSION_NUM >= PG_VERSION_15) static void QualifyPublicationObjects(List *publicationObjects); #else diff --git a/src/backend/distributed/deparser/qualify_role_stmt.c b/src/backend/distributed/deparser/qualify_role_stmt.c index 93a958ea9..cffb7ac4c 100644 --- a/src/backend/distributed/deparser/qualify_role_stmt.c +++ b/src/backend/distributed/deparser/qualify_role_stmt.c @@ -17,10 +17,11 @@ #include "postgres.h" -#include "distributed/deparser.h" #include "nodes/nodes.h" #include "utils/guc.h" +#include "distributed/deparser.h" + static void QualifyVarSetCurrent(VariableSetStmt *setStmt); diff --git a/src/backend/distributed/deparser/qualify_sequence_stmt.c b/src/backend/distributed/deparser/qualify_sequence_stmt.c index 384e0c953..1a0ecc831 100644 --- a/src/backend/distributed/deparser/qualify_sequence_stmt.c +++ b/src/backend/distributed/deparser/qualify_sequence_stmt.c @@ -17,12 +17,13 @@ #include "postgres.h" +#include "parser/parse_func.h" +#include "utils/lsyscache.h" + #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/version_compat.h" -#include "parser/parse_func.h" -#include "utils/lsyscache.h" /* diff --git a/src/backend/distributed/deparser/qualify_statistics_stmt.c b/src/backend/distributed/deparser/qualify_statistics_stmt.c index ce9443930..ba8e8a764 100644 --- a/src/backend/distributed/deparser/qualify_statistics_stmt.c +++ b/src/backend/distributed/deparser/qualify_statistics_stmt.c @@ -16,15 +16,16 @@ #include "catalog/namespace.h" #include "catalog/pg_statistic_ext.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "nodes/parsenodes.h" #include "nodes/value.h" -#include "utils/syscache.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" +#include "utils/syscache.h" + +#include "distributed/commands.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" static Oid GetStatsNamespaceOid(Oid statsOid); diff --git a/src/backend/distributed/deparser/qualify_table_stmt.c b/src/backend/distributed/deparser/qualify_table_stmt.c index 9667c4c79..e760ff388 100644 --- a/src/backend/distributed/deparser/qualify_table_stmt.c +++ b/src/backend/distributed/deparser/qualify_table_stmt.c @@ -23,6 +23,7 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" + #include "distributed/deparser.h" void diff --git a/src/backend/distributed/deparser/qualify_type_stmt.c b/src/backend/distributed/deparser/qualify_type_stmt.c index 487e6fc97..91052b576 100644 --- a/src/backend/distributed/deparser/qualify_type_stmt.c +++ b/src/backend/distributed/deparser/qualify_type_stmt.c @@ -23,13 +23,14 @@ #include "catalog/namespace.h" #include "catalog/objectaddress.h" #include "catalog/pg_type.h" +#include "nodes/makefuncs.h" +#include "parser/parse_type.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" -#include "nodes/makefuncs.h" -#include "parser/parse_type.h" -#include "utils/syscache.h" -#include "utils/lsyscache.h" /* * GetTypeNamespaceNameByNameList resolved the schema name of a type by its namelist. diff --git a/src/backend/distributed/deparser/qualify_view_stmt.c b/src/backend/distributed/deparser/qualify_view_stmt.c index 1f450d50a..af3fb280a 100644 --- a/src/backend/distributed/deparser/qualify_view_stmt.c +++ b/src/backend/distributed/deparser/qualify_view_stmt.c @@ -12,12 +12,13 @@ #include "postgres.h" #include "catalog/namespace.h" -#include "distributed/deparser.h" -#include "distributed/listutils.h" #include "nodes/nodes.h" #include "utils/guc.h" #include "utils/lsyscache.h" +#include "distributed/deparser.h" +#include "distributed/listutils.h" + static void QualifyViewRangeVar(RangeVar *view); /* diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c index 61a52e7c4..b5566985a 100644 --- a/src/backend/distributed/executor/adaptive_executor.c +++ b/src/backend/distributed/executor/adaptive_executor.c @@ -118,32 +118,43 @@ *------------------------------------------------------------------------- */ +#include +#include +#include + #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" -#include -#include -#include - +#include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" -#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "commands/schemacmds.h" +#include "lib/ilist.h" +#include "portability/instr_time.h" +#include "storage/fd.h" +#include "storage/latch.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/syscache.h" +#include "utils/timestamp.h" + #include "distributed/adaptive_executor.h" +#include "distributed/backend_data.h" #include "distributed/cancel_utils.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_safe_lib.h" -#include "distributed/connection_management.h" #include "distributed/commands/multi_copy.h" +#include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" -#include "distributed/executor_util.h" -#include "distributed/shared_connection_stats.h" #include "distributed/distributed_execution_locks.h" +#include "distributed/executor_util.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" @@ -161,21 +172,11 @@ #include "distributed/resource_lock.h" #include "distributed/shared_connection_stats.h" #include "distributed/subplan_execution.h" -#include "distributed/transaction_management.h" #include "distributed/transaction_identifier.h" +#include "distributed/transaction_management.h" #include "distributed/tuple_destination.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "distributed/backend_data.h" -#include "lib/ilist.h" -#include "portability/instr_time.h" -#include "storage/fd.h" -#include "storage/latch.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/syscache.h" -#include "utils/timestamp.h" #define SLOW_START_DISABLED 0 diff --git a/src/backend/distributed/executor/citus_custom_scan.c b/src/backend/distributed/executor/citus_custom_scan.c index 3403e27ca..34a2f3d90 100644 --- a/src/backend/distributed/executor/citus_custom_scan.c +++ b/src/backend/distributed/executor/citus_custom_scan.c @@ -9,19 +9,30 @@ */ #include "postgres.h" -#include "pg_version_constants.h" - #include "miscadmin.h" #include "commands/copy.h" +#include "executor/executor.h" +#include "nodes/makefuncs.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +#include "pg_version_constants.h" + #include "distributed/backend_data.h" #include "distributed/citus_clauses.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_execution_locks.h" +#include "distributed/function_call_delegation.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" @@ -30,23 +41,13 @@ #include "distributed/merge_executor.h" #include "distributed/merge_planner.h" #include "distributed/multi_executor.h" -#include "distributed/multi_server_executor.h" #include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" #include "distributed/query_stats.h" #include "distributed/shard_utils.h" #include "distributed/subplan_execution.h" #include "distributed/worker_log_messages.h" #include "distributed/worker_protocol.h" -#include "distributed/colocation_utils.h" -#include "distributed/function_call_delegation.h" -#include "executor/executor.h" -#include "nodes/makefuncs.h" -#include "optimizer/optimizer.h" -#include "optimizer/clauses.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/rel.h" -#include "utils/datum.h" extern AllowedDistributionColumn AllowedDistributionColumnValue; diff --git a/src/backend/distributed/executor/directed_acyclic_graph_execution.c b/src/backend/distributed/executor/directed_acyclic_graph_execution.c index e0d4c9b81..15b0272dd 100644 --- a/src/backend/distributed/executor/directed_acyclic_graph_execution.c +++ b/src/backend/distributed/executor/directed_acyclic_graph_execution.c @@ -8,11 +8,12 @@ */ #include "postgres.h" + #include "access/hash.h" -#include "distributed/hash_helpers.h" #include "distributed/adaptive_executor.h" #include "distributed/directed_acyclic_graph_execution.h" +#include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" diff --git a/src/backend/distributed/executor/distributed_execution_locks.c b/src/backend/distributed/executor/distributed_execution_locks.c index f7d2fd49d..4424accb7 100644 --- a/src/backend/distributed/executor/distributed_execution_locks.c +++ b/src/backend/distributed/executor/distributed_execution_locks.c @@ -8,10 +8,10 @@ * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ +#include "distributed/coordinator_protocol.h" #include "distributed/distributed_execution_locks.h" #include "distributed/executor_util.h" #include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" diff --git a/src/backend/distributed/executor/distributed_intermediate_results.c b/src/backend/distributed/executor/distributed_intermediate_results.c index cc351a1fc..c5ac27fb6 100644 --- a/src/backend/distributed/executor/distributed_intermediate_results.c +++ b/src/backend/distributed/executor/distributed_intermediate_results.c @@ -8,12 +8,11 @@ *------------------------------------------------------------------------- */ -#include "pg_version_constants.h" - #include #include #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" #include "port.h" @@ -21,21 +20,24 @@ #include "access/htup_details.h" #include "access/tupdesc.h" #include "catalog/pg_type.h" +#include "tcop/pquery.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" + +#include "pg_version_constants.h" + #include "distributed/deparse_shard_query.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" -#include "distributed/metadata_utility.h" #include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/transaction_management.h" #include "distributed/tuple_destination.h" #include "distributed/tuplestore.h" #include "distributed/worker_protocol.h" -#include "tcop/pquery.h" -#include "tcop/tcopprot.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" /* diff --git a/src/backend/distributed/executor/executor_util_params.c b/src/backend/distributed/executor/executor_util_params.c index 6b5139bff..975654f22 100644 --- a/src/backend/distributed/executor/executor_util_params.c +++ b/src/backend/distributed/executor/executor_util_params.c @@ -8,12 +8,14 @@ */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" -#include "distributed/executor_util.h" #include "utils/lsyscache.h" +#include "distributed/executor_util.h" + /* * ExtractParametersForRemoteExecution extracts parameter types and values from diff --git a/src/backend/distributed/executor/executor_util_tasks.c b/src/backend/distributed/executor/executor_util_tasks.c index abf721196..cef005042 100644 --- a/src/backend/distributed/executor/executor_util_tasks.c +++ b/src/backend/distributed/executor/executor_util_tasks.c @@ -8,6 +8,7 @@ */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" diff --git a/src/backend/distributed/executor/executor_util_tuples.c b/src/backend/distributed/executor/executor_util_tuples.c index c5fde9f90..68f699956 100644 --- a/src/backend/distributed/executor/executor_util_tuples.c +++ b/src/backend/distributed/executor/executor_util_tuples.c @@ -8,12 +8,14 @@ */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" -#include "distributed/executor_util.h" #include "utils/lsyscache.h" +#include "distributed/executor_util.h" + /* * TupleDescGetAttBinaryInMetadata - Build an AttInMetadata structure based on diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index 4a15289e6..f5fbb3f78 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -9,34 +9,9 @@ */ #include "postgres.h" + #include "miscadmin.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/adaptive_executor.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/distributed_execution_locks.h" -#include "distributed/insert_select_executor.h" -#include "distributed/insert_select_planner.h" -#include "distributed/intermediate_results.h" -#include "distributed/local_executor.h" -#include "distributed/merge_planner.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_router_planner.h" -#include "distributed/local_executor.h" -#include "distributed/distributed_planner.h" -#include "distributed/recursive_planning.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/repartition_executor.h" -#include "distributed/resource_lock.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/subplan_execution.h" -#include "distributed/transaction_management.h" -#include "distributed/version_compat.h" #include "executor/executor.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" @@ -53,6 +28,32 @@ #include "utils/rel.h" #include "utils/snapmgr.h" +#include "distributed/adaptive_executor.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/distributed_execution_locks.h" +#include "distributed/distributed_planner.h" +#include "distributed/insert_select_executor.h" +#include "distributed/insert_select_planner.h" +#include "distributed/intermediate_results.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/merge_planner.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/recursive_planning.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/repartition_executor.h" +#include "distributed/resource_lock.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/subplan_execution.h" +#include "distributed/transaction_management.h" +#include "distributed/version_compat.h" + /* Config variables managed via guc.c */ bool EnableRepartitionedInsertSelect = true; diff --git a/src/backend/distributed/executor/intermediate_results.c b/src/backend/distributed/executor/intermediate_results.c index d17e65217..0e18d4416 100644 --- a/src/backend/distributed/executor/intermediate_results.c +++ b/src/backend/distributed/executor/intermediate_results.c @@ -11,6 +11,7 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" @@ -19,22 +20,6 @@ #include "catalog/pg_enum.h" #include "catalog/pg_type.h" #include "commands/copy.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/connection_management.h" -#include "distributed/error_codes.h" -#include "distributed/intermediate_results.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/remote_commands.h" -#include "distributed/transmit.h" -#include "distributed/transaction_identifier.h" -#include "distributed/tuplestore.h" -#include "distributed/utils/array_type.h" -#include "distributed/utils/directory.h" -#include "distributed/version_compat.h" -#include "distributed/worker_protocol.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" @@ -45,6 +30,23 @@ #include "utils/memutils.h" #include "utils/syscache.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/connection_management.h" +#include "distributed/error_codes.h" +#include "distributed/intermediate_results.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/remote_commands.h" +#include "distributed/transaction_identifier.h" +#include "distributed/transmit.h" +#include "distributed/tuplestore.h" +#include "distributed/utils/array_type.h" +#include "distributed/utils/directory.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" + static List *CreatedResultsDirectories = NIL; diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index cac073b3a..5af115357 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -76,36 +76,38 @@ * via coordinator cannot happen via the local execution. */ #include "postgres.h" + #include "miscadmin.h" +#include "executor/tstoreReceiver.h" +#include "executor/tuptable.h" +#include "nodes/params.h" +#include "optimizer/optimizer.h" +#include "utils/snapmgr.h" + #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" -#include "distributed/commands/utility_hook.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" -#include "distributed/query_utils.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" +#include "distributed/executor_util.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_plan_cache.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/executor_util.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_server_executor.h" +#include "distributed/query_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" /* to access LogRemoteCommands */ #include "distributed/transaction_management.h" #include "distributed/utils/citus_stat_tenants.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "executor/tstoreReceiver.h" -#include "executor/tuptable.h" -#include "optimizer/optimizer.h" -#include "nodes/params.h" -#include "utils/snapmgr.h" /* controlled via a GUC */ bool EnableLocalExecution = true; diff --git a/src/backend/distributed/executor/merge_executor.c b/src/backend/distributed/executor/merge_executor.c index bcacbcd1e..969b03faf 100644 --- a/src/backend/distributed/executor/merge_executor.c +++ b/src/backend/distributed/executor/merge_executor.c @@ -9,8 +9,13 @@ */ #include "postgres.h" + #include "miscadmin.h" +#include "nodes/execnodes.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" + #include "distributed/distributed_execution_locks.h" #include "distributed/insert_select_executor.h" #include "distributed/intermediate_results.h" @@ -23,10 +28,6 @@ #include "distributed/repartition_executor.h" #include "distributed/subplan_execution.h" -#include "nodes/execnodes.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" - static void ExecuteSourceAtWorkerAndRepartition(CitusScanState *scanState); static void ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState); static HTAB * ExecuteMergeSourcePlanIntoColocatedIntermediateResults(Oid targetRelationId, diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 306698251..45a791af4 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -10,50 +10,50 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "miscadmin.h" #include "access/xact.h" #include "catalog/dependency.h" -#include "catalog/pg_class.h" #include "catalog/namespace.h" +#include "catalog/pg_class.h" +#include "commands/copy.h" +#include "executor/execdebug.h" +#include "nodes/execnodes.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "parser/parse_oper.h" +#include "parser/parsetree.h" +#include "storage/lmgr.h" +#include "tcop/dest.h" +#include "tcop/pquery.h" +#include "tcop/utility.h" +#include "utils/fmgrprotos.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" + +#include "pg_version_constants.h" + #include "distributed/backend_data.h" #include "distributed/citus_custom_scan.h" +#include "distributed/combine_query_planner.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" -#include "distributed/coordinator_protocol.h" #include "distributed/multi_executor.h" -#include "distributed/combine_query_planner.h" -#include "distributed/distributed_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" -#include "distributed/worker_shard_visibility.h" #include "distributed/worker_protocol.h" -#include "distributed/function_call_delegation.h" -#include "executor/execdebug.h" -#include "commands/copy.h" -#include "nodes/execnodes.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "parser/parsetree.h" -#include "parser/parse_oper.h" -#include "storage/lmgr.h" -#include "tcop/dest.h" -#include "tcop/pquery.h" -#include "tcop/utility.h" -#include "utils/fmgrprotos.h" -#include "utils/snapmgr.h" -#include "utils/memutils.h" +#include "distributed/worker_shard_visibility.h" /* diff --git a/src/backend/distributed/executor/multi_server_executor.c b/src/backend/distributed/executor/multi_server_executor.c index ac144c350..209019833 100644 --- a/src/backend/distributed/executor/multi_server_executor.c +++ b/src/backend/distributed/executor/multi_server_executor.c @@ -14,22 +14,24 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "miscadmin.h" - #include +#include "postgres.h" + +#include "miscadmin.h" + +#include "utils/lsyscache.h" + +#include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" -#include "distributed/multi_server_executor.h" #include "distributed/multi_router_planner.h" -#include "distributed/coordinator_protocol.h" +#include "distributed/multi_server_executor.h" #include "distributed/subplan_execution.h" #include "distributed/tuple_destination.h" #include "distributed/worker_protocol.h" -#include "utils/lsyscache.h" int RemoteTaskCheckInterval = 10; /* per cycle sleep interval in millisecs */ int TaskExecutorType = MULTI_EXECUTOR_ADAPTIVE; /* distributed executor type */ diff --git a/src/backend/distributed/executor/partitioned_intermediate_results.c b/src/backend/distributed/executor/partitioned_intermediate_results.c index 752552343..3ec73a456 100644 --- a/src/backend/distributed/executor/partitioned_intermediate_results.c +++ b/src/backend/distributed/executor/partitioned_intermediate_results.c @@ -11,6 +11,7 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" @@ -20,9 +21,15 @@ #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" +#include "nodes/makefuncs.h" +#include "nodes/primnodes.h" +#include "tcop/pquery.h" +#include "tcop/tcopprot.h" +#include "utils/typcache.h" + #include "distributed/intermediate_results.h" -#include "distributed/metadata_utility.h" #include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/pg_dist_shard.h" #include "distributed/remote_commands.h" @@ -31,11 +38,6 @@ #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "nodes/makefuncs.h" -#include "nodes/primnodes.h" -#include "tcop/pquery.h" -#include "tcop/tcopprot.h" -#include "utils/typcache.h" /* diff --git a/src/backend/distributed/executor/placement_access.c b/src/backend/distributed/executor/placement_access.c index df5143a54..a8573de7c 100644 --- a/src/backend/distributed/executor/placement_access.c +++ b/src/backend/distributed/executor/placement_access.c @@ -8,9 +8,9 @@ * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ -#include "distributed/placement_access.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" +#include "distributed/placement_access.h" static List * BuildPlacementSelectList(int32 groupId, List *relationShardList); static List * BuildPlacementDDLList(int32 groupId, List *relationShardList); diff --git a/src/backend/distributed/executor/query_stats.c b/src/backend/distributed/executor/query_stats.c index b59777d45..f37a99bbf 100644 --- a/src/backend/distributed/executor/query_stats.c +++ b/src/backend/distributed/executor/query_stats.c @@ -9,32 +9,32 @@ *------------------------------------------------------------------------- */ +#include + #include "postgres.h" -#include "safe_lib.h" - +#include "funcapi.h" #include "miscadmin.h" - -#include "pg_version_constants.h" +#include "safe_lib.h" #include "access/hash.h" #include "catalog/pg_authid.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/spin.h" +#include "tcop/utility.h" +#include "utils/builtins.h" + +#include "pg_version_constants.h" + #include "distributed/citus_safe_lib.h" #include "distributed/function_utils.h" #include "distributed/hash_helpers.h" #include "distributed/multi_executor.h" #include "distributed/multi_server_executor.h" -#include "distributed/version_compat.h" #include "distributed/query_stats.h" #include "distributed/tuplestore.h" -#include "funcapi.h" -#include "storage/ipc.h" -#include "storage/fd.h" -#include "storage/spin.h" -#include "tcop/utility.h" -#include "utils/builtins.h" - -#include +#include "distributed/version_compat.h" #define CITUS_STATS_DUMP_FILE "pg_stat/citus_query_stats.stat" #define CITUS_STAT_STATEMENTS_COLS 6 diff --git a/src/backend/distributed/executor/repartition_executor.c b/src/backend/distributed/executor/repartition_executor.c index af4f0ac7e..6e4dd3df4 100644 --- a/src/backend/distributed/executor/repartition_executor.c +++ b/src/backend/distributed/executor/repartition_executor.c @@ -10,6 +10,7 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "nodes/makefuncs.h" diff --git a/src/backend/distributed/executor/repartition_join_execution.c b/src/backend/distributed/executor/repartition_join_execution.c index 29d994e59..8dce12390 100644 --- a/src/backend/distributed/executor/repartition_join_execution.c +++ b/src/backend/distributed/executor/repartition_join_execution.c @@ -24,20 +24,22 @@ */ #include "postgres.h" -#include "access/hash.h" + #include "miscadmin.h" + +#include "access/hash.h" #include "utils/builtins.h" -#include "distributed/hash_helpers.h" #include "distributed/adaptive_executor.h" #include "distributed/directed_acyclic_graph_execution.h" +#include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" -#include "distributed/task_execution_utils.h" #include "distributed/repartition_join_execution.h" +#include "distributed/task_execution_utils.h" #include "distributed/transaction_management.h" #include "distributed/transmit.h" #include "distributed/worker_manager.h" diff --git a/src/backend/distributed/executor/subplan_execution.c b/src/backend/distributed/executor/subplan_execution.c index 3651d7f52..4e81bb486 100644 --- a/src/backend/distributed/executor/subplan_execution.c +++ b/src/backend/distributed/executor/subplan_execution.c @@ -10,6 +10,9 @@ #include "postgres.h" +#include "executor/executor.h" +#include "utils/datetime.h" + #include "distributed/intermediate_result_pruning.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" @@ -19,8 +22,6 @@ #include "distributed/subplan_execution.h" #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" -#include "executor/executor.h" -#include "utils/datetime.h" #define SECOND_TO_MILLI_SECOND 1000 #define MICRO_TO_MILLI_SECOND 0.001 diff --git a/src/backend/distributed/executor/transmit.c b/src/backend/distributed/executor/transmit.c index 24cbbb550..a10ae4fbf 100644 --- a/src/backend/distributed/executor/transmit.c +++ b/src/backend/distributed/executor/transmit.c @@ -7,24 +7,26 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "miscadmin.h" -#include "pgstat.h" - #include #include #include +#include "postgres.h" + +#include "miscadmin.h" +#include "pgstat.h" + #include "commands/defrem.h" +#include "libpq/libpq.h" +#include "libpq/pqformat.h" +#include "storage/fd.h" + #include "distributed/listutils.h" #include "distributed/relay_utility.h" #include "distributed/transmit.h" #include "distributed/utils/directory.h" -#include "distributed/worker_protocol.h" #include "distributed/version_compat.h" -#include "libpq/libpq.h" -#include "libpq/pqformat.h" -#include "storage/fd.h" +#include "distributed/worker_protocol.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/executor/tuple_destination.c b/src/backend/distributed/executor/tuple_destination.c index 42dbf001e..3c44d21c0 100644 --- a/src/backend/distributed/executor/tuple_destination.c +++ b/src/backend/distributed/executor/tuple_destination.c @@ -1,13 +1,15 @@ +#include +#include + #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" -#include -#include - #include "access/htup_details.h" + #include "distributed/multi_server_executor.h" #include "distributed/subplan_execution.h" #include "distributed/tuple_destination.h" diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index 989e957af..6932c453c 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -10,8 +10,7 @@ #include "postgres.h" -#include "distributed/commands.h" -#include "pg_version_constants.h" +#include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" @@ -36,6 +35,13 @@ #include "catalog/pg_type.h" #include "commands/extension.h" #include "common/hashfn.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + #include "distributed/citus_depended_object.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" @@ -46,11 +52,6 @@ #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/version_compat.h" -#include "miscadmin.h" -#include "utils/fmgroids.h" -#include "utils/hsearch.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" /* * ObjectAddressCollector keeps track of collected ObjectAddresses. This can be used diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index f0176e867..3f5824979 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -10,8 +10,6 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "miscadmin.h" #include "access/genam.h" @@ -26,19 +24,7 @@ #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" -#include "citus_version.h" #include "commands/extension.h" -#include "distributed/listutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata/pg_dist_object.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/version_compat.h" -#include "distributed/worker_transaction.h" #include "executor/spi.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" @@ -49,6 +35,21 @@ #include "utils/regproc.h" #include "utils/rel.h" +#include "citus_version.h" +#include "pg_version_constants.h" + +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/listutils.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata/pg_dist_object.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/version_compat.h" +#include "distributed/worker_transaction.h" + static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index a2e2d87c7..b15586818 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -8,20 +8,17 @@ */ #include "postgres.h" -#include "pg_version_constants.h" -#include "pg_version_compat.h" -#include "stdint.h" -#include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" +#include "stdint.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" -#include "access/xact.h" #include "access/sysattr.h" +#include "access/xact.h" #include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/pg_am.h" @@ -30,38 +27,10 @@ #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "catalog/pg_type.h" -#include "citus_version.h" #include "commands/dbcommands.h" #include "commands/extension.h" #include "commands/trigger.h" -#include "distributed/backend_data.h" -#include "distributed/citus_depended_object.h" -#include "distributed/colocation_utils.h" -#include "distributed/connection_management.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/multi_executor.h" -#include "distributed/function_utils.h" -#include "distributed/listutils.h" -#include "distributed/foreign_key_relationship.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata/pg_dist_object.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/pg_dist_local_group.h" -#include "distributed/pg_dist_node_metadata.h" -#include "distributed/pg_dist_node.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/pg_dist_placement.h" -#include "distributed/shared_library_init.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/utils/array_type.h" -#include "distributed/utils/function.h" -#include "distributed/version_compat.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" +#include "common/hashfn.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "nodes/memnodes.h" @@ -74,15 +43,45 @@ #include "utils/catcache.h" #include "utils/datum.h" #include "utils/elog.h" -#include "utils/hsearch.h" -#include "utils/jsonb.h" -#include "common/hashfn.h" -#include "utils/inval.h" #include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/inval.h" +#include "utils/jsonb.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/palloc.h" #include "utils/rel.h" + +#include "citus_version.h" +#include "pg_version_compat.h" +#include "pg_version_constants.h" + +#include "distributed/backend_data.h" +#include "distributed/citus_depended_object.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/connection_management.h" +#include "distributed/foreign_key_relationship.h" +#include "distributed/function_utils.h" +#include "distributed/listutils.h" +#include "distributed/metadata/pg_dist_object.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_local_group.h" +#include "distributed/pg_dist_node.h" +#include "distributed/pg_dist_node_metadata.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_placement.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shared_library_init.h" +#include "distributed/utils/array_type.h" +#include "distributed/utils/function.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" #if PG_VERSION_NUM < PG_VERSION_16 #include "utils/relfilenodemap.h" #endif diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 40bdae0ea..0492e7b55 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -11,13 +11,15 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "miscadmin.h" - #include #include #include +#include "postgres.h" + +#include "miscadmin.h" +#include "pgstat.h" + #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" @@ -36,51 +38,13 @@ #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/async.h" -#include "distributed/argutils.h" -#include "distributed/backend_data.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/tenant_schema_metadata.h" -#include "distributed/commands.h" -#include "distributed/deparser.h" -#include "distributed/distribution_column.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/maintenanced.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "distributed/metadata/pg_dist_object.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/pg_dist_colocation.h" -#include "distributed/pg_dist_node.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/pg_dist_schema.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/remote_commands.h" -#include "distributed/resource_lock.h" -#include "distributed/utils/array_type.h" -#include "distributed/utils/function.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_transaction.h" -#include "distributed/version_compat.h" -#include "distributed/commands/utility_hook.h" #include "executor/spi.h" #include "foreign/foreign.h" -#include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" -#include "pgstat.h" +#include "parser/parse_type.h" #include "postmaster/bgworker.h" #include "postmaster/postmaster.h" -#include "parser/parse_type.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -89,6 +53,42 @@ #include "utils/snapmgr.h" #include "utils/syscache.h" +#include "distributed/argutils.h" +#include "distributed/backend_data.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/distribution_column.h" +#include "distributed/listutils.h" +#include "distributed/maintenanced.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata/pg_dist_object.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_colocation.h" +#include "distributed/pg_dist_node.h" +#include "distributed/pg_dist_schema.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/tenant_schema_metadata.h" +#include "distributed/utils/array_type.h" +#include "distributed/utils/function.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_transaction.h" + /* managed via a GUC */ char *EnableManualMetadataChangesForUser = ""; diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index 8668c0ca1..792a4aeba 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -13,12 +13,11 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" -#include "pg_version_constants.h" - #include "access/genam.h" #include "access/htup_details.h" #include "access/sysattr.h" @@ -29,44 +28,14 @@ #include "catalog/pg_constraint.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" + +#include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "catalog/pg_proc_d.h" #endif #include "catalog/pg_type.h" #include "commands/extension.h" #include "commands/sequence.h" -#include "distributed/background_jobs.h" -#include "distributed/colocation_utils.h" -#include "distributed/connection_management.h" -#include "distributed/citus_nodes.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/listutils.h" -#include "distributed/lock_graph.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/pg_dist_background_job.h" -#include "distributed/pg_dist_background_task.h" -#include "distributed/pg_dist_backrgound_task_depend.h" -#include "distributed/pg_dist_colocation.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/pg_dist_placement.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relay_utility.h" -#include "distributed/resource_lock.h" -#include "distributed/remote_commands.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/tuplestore.h" -#include "distributed/utils/array_type.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "parser/scansup.h" #include "storage/lmgr.h" @@ -81,6 +50,39 @@ #include "utils/rel.h" #include "utils/syscache.h" +#include "distributed/background_jobs.h" +#include "distributed/citus_nodes.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/colocation_utils.h" +#include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/lock_graph.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_background_job.h" +#include "distributed/pg_dist_background_task.h" +#include "distributed/pg_dist_backrgound_task_depend.h" +#include "distributed/pg_dist_colocation.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_placement.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relay_utility.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_rebalancer.h" +#include "distributed/tuplestore.h" +#include "distributed/utils/array_type.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" + #define DISK_SPACE_FIELDS 2 /* Local functions forward declarations */ diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c index a73f2e9d2..de3dbcbdf 100644 --- a/src/backend/distributed/metadata/node_metadata.c +++ b/src/backend/distributed/metadata/node_metadata.c @@ -5,34 +5,49 @@ * Copyright (c) Citus Data, Inc. */ #include "postgres.h" -#include "miscadmin.h" + #include "funcapi.h" -#include "utils/plancache.h" +#include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/skey.h" -#include "access/skey.h" #include "access/tupmacs.h" #include "access/xact.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "commands/sequence.h" +#include "executor/spi.h" +#include "lib/stringinfo.h" +#include "postmaster/postmaster.h" +#include "storage/bufmgr.h" +#include "storage/fd.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/plancache.h" +#include "utils/rel.h" +#include "utils/relcache.h" + #include "distributed/citus_acquire_lock.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" -#include "distributed/maintenanced.h" #include "distributed/coordinator_protocol.h" -#include "distributed/metadata_utility.h" +#include "distributed/maintenanced.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_join_order.h" +#include "distributed/multi_partitioning_utils.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_node_metadata.h" @@ -40,26 +55,12 @@ #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" -#include "distributed/multi_partitioning_utils.h" #include "distributed/shared_connection_stats.h" #include "distributed/string_utils.h" -#include "distributed/metadata/pg_dist_object.h" #include "distributed/transaction_recovery.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "executor/spi.h" -#include "lib/stringinfo.h" -#include "postmaster/postmaster.h" -#include "storage/bufmgr.h" -#include "storage/lmgr.h" -#include "storage/lock.h" -#include "storage/fd.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/relcache.h" #define INVALID_GROUP_ID -1 diff --git a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c index a7f40e2ad..abe378cdb 100644 --- a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c +++ b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c @@ -16,23 +16,26 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/objectaddress.h" #include "catalog/pg_type.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/metadata/dependency.h" -#include "distributed/metadata/distobject.h" -#include "pg_version_constants.h" -#include "distributed/version_compat.h" +#include "mb/pg_wchar.h" #include "nodes/value.h" +#include "parser/parse_type.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/varlena.h" -#include "mb/pg_wchar.h" -#include "parser/parse_type.h" + +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata/distobject.h" +#include "distributed/version_compat.h" static void ErrorIfCurrentUserCanNotDistributeObject(char *textType, ObjectType type, diff --git a/src/backend/distributed/operations/citus_create_restore_point.c b/src/backend/distributed/operations/citus_create_restore_point.c index 42fc5311f..8a5e738e4 100644 --- a/src/backend/distributed/operations/citus_create_restore_point.c +++ b/src/backend/distributed/operations/citus_create_restore_point.c @@ -10,22 +10,24 @@ */ #include "postgres.h" + #include "libpq-fe.h" #include "access/xlog.h" #include "access/xlog_internal.h" #include "catalog/pg_type.h" -#include "distributed/connection_management.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata_cache.h" -#include "distributed/remote_commands.h" #include "nodes/pg_list.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" +#include "distributed/connection_management.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/remote_commands.h" + #define CREATE_RESTORE_POINT_COMMAND "SELECT pg_catalog.pg_create_restore_point($1::text)" diff --git a/src/backend/distributed/operations/citus_split_shard_by_split_points.c b/src/backend/distributed/operations/citus_split_shard_by_split_points.c index 5bdbaf576..076e58d4c 100644 --- a/src/backend/distributed/operations/citus_split_shard_by_split_points.c +++ b/src/backend/distributed/operations/citus_split_shard_by_split_points.c @@ -10,19 +10,21 @@ */ #include "postgres.h" + #include "catalog/pg_type.h" -#include "nodes/pg_list.h" #include "lib/stringinfo.h" +#include "nodes/pg_list.h" #include "utils/builtins.h" #include "utils/lsyscache.h" -#include "distributed/utils/array_type.h" + #include "distributed/colocation_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/shard_split.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/utils/array_type.h" #include "distributed/utils/distribution_column_map.h" /* declarations for dynamic loading */ diff --git a/src/backend/distributed/operations/citus_tools.c b/src/backend/distributed/operations/citus_tools.c index 8f6f80c2b..bc4aa5560 100644 --- a/src/backend/distributed/operations/citus_tools.c +++ b/src/backend/distributed/operations/citus_tools.c @@ -12,8 +12,15 @@ #include "postgres.h" +#include "funcapi.h" +#include "libpq-fe.h" +#include "miscadmin.h" + #include "access/htup_details.h" #include "catalog/pg_type.h" +#include "lib/stringinfo.h" +#include "utils/builtins.h" + #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/metadata_cache.h" @@ -23,11 +30,6 @@ #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "funcapi.h" -#include "lib/stringinfo.h" -#include "libpq-fe.h" -#include "miscadmin.h" -#include "utils/builtins.h" PG_FUNCTION_INFO_V1(master_run_on_worker); diff --git a/src/backend/distributed/operations/create_shards.c b/src/backend/distributed/operations/create_shards.c index d0fcc9612..b5372d4e6 100644 --- a/src/backend/distributed/operations/create_shards.c +++ b/src/backend/distributed/operations/create_shards.c @@ -10,35 +10,22 @@ *------------------------------------------------------------------------- */ +#include +#include +#include +#include +#include + #include "postgres.h" + #include "c.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "port.h" -#include -#include -#include -#include -#include - #include "catalog/namespace.h" #include "catalog/pg_class.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/reference_table_utils.h" -#include "distributed/resource_lock.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/transaction_management.h" -#include "distributed/worker_manager.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" @@ -52,6 +39,21 @@ #include "utils/lsyscache.h" #include "utils/palloc.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/reference_table_utils.h" +#include "distributed/resource_lock.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/transaction_management.h" +#include "distributed/worker_manager.h" + /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(master_create_worker_shards); diff --git a/src/backend/distributed/operations/delete_protocol.c b/src/backend/distributed/operations/delete_protocol.c index 54cb568be..c36121b00 100644 --- a/src/backend/distributed/operations/delete_protocol.c +++ b/src/backend/distributed/operations/delete_protocol.c @@ -13,9 +13,9 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" +#include -#include "pg_version_constants.h" +#include "postgres.h" #include "c.h" #include "fmgr.h" @@ -23,17 +23,37 @@ #include "miscadmin.h" #include "port.h" -#include - #include "access/xact.h" #include "catalog/namespace.h" #include "commands/dbcommands.h" +#include "lib/stringinfo.h" +#include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" +#include "nodes/pathnodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" +#include "optimizer/restrictinfo.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "tcop/tcopprot.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/errcodes.h" +#include "utils/lsyscache.h" +#include "utils/varlena.h" + +#include "pg_version_constants.h" + #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_planner.h" @@ -47,25 +67,6 @@ #include "distributed/shard_cleaner.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" -#include "lib/stringinfo.h" -#include "nodes/nodeFuncs.h" -#include "nodes/nodes.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "optimizer/clauses.h" -#include "nodes/pathnodes.h" -#include "optimizer/optimizer.h" -#include "optimizer/restrictinfo.h" -#include "storage/lock.h" -#include "storage/lmgr.h" -#include "tcop/tcopprot.h" -#include "utils/array.h" -#include "utils/builtins.h" -#include "utils/elog.h" -#include "utils/errcodes.h" -#include "utils/lsyscache.h" -#include "utils/varlena.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/operations/health_check.c b/src/backend/distributed/operations/health_check.c index b3246f888..c908606c1 100644 --- a/src/backend/distributed/operations/health_check.c +++ b/src/backend/distributed/operations/health_check.c @@ -13,6 +13,8 @@ #include "postgres.h" +#include "utils/builtins.h" + #include "distributed/argutils.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" @@ -20,7 +22,6 @@ #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" -#include "utils/builtins.h" /* simple query to run on workers to check connectivity */ #define CONNECTIVITY_CHECK_QUERY "SELECT 1" diff --git a/src/backend/distributed/operations/isolate_shards.c b/src/backend/distributed/operations/isolate_shards.c index ec89ae402..502b00f5b 100644 --- a/src/backend/distributed/operations/isolate_shards.c +++ b/src/backend/distributed/operations/isolate_shards.c @@ -11,11 +11,20 @@ */ #include "postgres.h" + #include "c.h" #include "fmgr.h" #include "libpq-fe.h" #include "catalog/pg_class.h" +#include "nodes/pg_list.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/errcodes.h" +#include "utils/lsyscache.h" +#include "utils/typcache.h" + #include "distributed/colocation_utils.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" @@ -25,22 +34,15 @@ #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" -#include "distributed/remote_commands.h" #include "distributed/reference_table_utils.h" +#include "distributed/remote_commands.h" #include "distributed/resource_lock.h" +#include "distributed/shard_split.h" +#include "distributed/utils/distribution_column_map.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" -#include "distributed/version_compat.h" -#include "distributed/shard_split.h" -#include "distributed/utils/distribution_column_map.h" -#include "nodes/pg_list.h" -#include "storage/lock.h" -#include "utils/builtins.h" -#include "utils/elog.h" -#include "utils/errcodes.h" -#include "utils/lsyscache.h" -#include "utils/typcache.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(isolate_tenant_to_new_shard); diff --git a/src/backend/distributed/operations/modify_multiple_shards.c b/src/backend/distributed/operations/modify_multiple_shards.c index 8d596a10b..9e2879728 100644 --- a/src/backend/distributed/operations/modify_multiple_shards.c +++ b/src/backend/distributed/operations/modify_multiple_shards.c @@ -14,40 +14,17 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" - #include "catalog/pg_class.h" #include "commands/dbcommands.h" #include "commands/event_trigger.h" -#include "distributed/citus_clauses.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/distributed_planner.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/resource_lock.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shard_pruning.h" -#include "distributed/version_compat.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_transaction.h" +#include "nodes/makefuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" -#include "nodes/makefuncs.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/datum.h" @@ -55,6 +32,29 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "pg_version_constants.h" + +#include "distributed/citus_clauses.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_pruning.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_transaction.h" + PG_FUNCTION_INFO_V1(master_modify_multiple_shards); diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index eeaf34321..52e44bea0 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -11,17 +11,15 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" +#include -#include "pg_version_constants.h" +#include "postgres.h" #include "c.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" -#include - #include "access/attnum.h" #include "access/genam.h" #include "access/heapam.h" @@ -37,20 +35,9 @@ #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_index.h" -#include "catalog/pg_type.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_type.h" #include "commands/sequence.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/namespace_utils.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/shared_library_init.h" -#include "distributed/version_compat.h" -#include "distributed/worker_manager.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" @@ -65,6 +52,20 @@ #include "utils/ruleutils.h" #include "utils/varlena.h" +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/namespace_utils.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/shared_library_init.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" + /* Shard related configuration */ int ShardCount = 32; int ShardReplicationFactor = 1; /* desired replication factor for shards */ diff --git a/src/backend/distributed/operations/partitioning.c b/src/backend/distributed/operations/partitioning.c index 9e2057927..afcaa8ac1 100644 --- a/src/backend/distributed/operations/partitioning.c +++ b/src/backend/distributed/operations/partitioning.c @@ -9,13 +9,12 @@ */ #include "postgres.h" + #include "fmgr.h" #include "funcapi.h" #include "access/htup.h" #include "access/htup_details.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_utility.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "utils/builtins.h" @@ -23,6 +22,9 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" + /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(time_partition_range); diff --git a/src/backend/distributed/operations/replicate_none_dist_table_shard.c b/src/backend/distributed/operations/replicate_none_dist_table_shard.c index c28490367..33a98ee42 100644 --- a/src/backend/distributed/operations/replicate_none_dist_table_shard.c +++ b/src/backend/distributed/operations/replicate_none_dist_table_shard.c @@ -10,7 +10,9 @@ */ #include "postgres.h" + #include "miscadmin.h" + #include "nodes/pg_list.h" #include "distributed/adaptive_executor.h" diff --git a/src/backend/distributed/operations/shard_cleaner.c b/src/backend/distributed/operations/shard_cleaner.c index 42877bf10..790414530 100644 --- a/src/backend/distributed/operations/shard_cleaner.c +++ b/src/backend/distributed/operations/shard_cleaner.c @@ -10,27 +10,29 @@ */ #include "postgres.h" + #include "miscadmin.h" + #include "access/genam.h" #include "access/xact.h" #include "catalog/namespace.h" #include "commands/dbcommands.h" #include "commands/sequence.h" -#include "postmaster/postmaster.h" #include "nodes/makefuncs.h" +#include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "distributed/citus_safe_lib.h" -#include "distributed/listutils.h" #include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" #include "distributed/metadata_cache.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_rebalancer.h" +#include "distributed/pg_dist_cleanup.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" #include "distributed/worker_transaction.h" -#include "distributed/pg_dist_cleanup.h" #define REPLICATION_SLOT_CATALOG_TABLE_NAME "pg_replication_slots" #define STR_ERRCODE_OBJECT_IN_USE "55006" diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index d339ac56a..8b58f5993 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -12,30 +12,47 @@ */ -#include "postgres.h" -#include "libpq-fe.h" - #include +#include "postgres.h" + +#include "funcapi.h" +#include "libpq-fe.h" +#include "miscadmin.h" + +#include "access/genam.h" +#include "access/htup_details.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_type.h" +#include "commands/dbcommands.h" +#include "commands/sequence.h" +#include "common/hashfn.h" +#include "postmaster/postmaster.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/guc_tables.h" +#include "utils/json.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/pg_lsn.h" +#include "utils/syscache.h" +#include "utils/varlena.h" + #include "pg_version_constants.h" -#include "access/htup_details.h" -#include "access/genam.h" -#include "catalog/pg_type.h" -#include "catalog/pg_proc.h" -#include "commands/dbcommands.h" -#include "commands/sequence.h" #include "distributed/argutils.h" #include "distributed/background_jobs.h" -#include "distributed/citus_safe_lib.h" #include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" +#include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" #include "distributed/enterprise.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_logical_replication.h" @@ -45,27 +62,12 @@ #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" -#include "distributed/shard_rebalancer.h" #include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" #include "distributed/shard_transfer.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/worker_protocol.h" -#include "funcapi.h" -#include "miscadmin.h" -#include "postmaster/postmaster.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/json.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/pg_lsn.h" -#include "utils/syscache.h" -#include "common/hashfn.h" -#include "utils/varlena.h" -#include "utils/guc_tables.h" -#include "distributed/commands/utility_hook.h" /* RebalanceOptions are the options used to control the rebalance algorithm */ typedef struct RebalanceOptions diff --git a/src/backend/distributed/operations/shard_split.c b/src/backend/distributed/operations/shard_split.c index 0772b03b4..cf9f301b7 100644 --- a/src/backend/distributed/operations/shard_split.c +++ b/src/backend/distributed/operations/shard_split.c @@ -10,41 +10,43 @@ */ #include "postgres.h" + #include "miscadmin.h" + +#include "commands/dbcommands.h" #include "common/hashfn.h" -#include "nodes/pg_list.h" -#include "utils/array.h" -#include "distributed/utils/array_type.h" #include "lib/stringinfo.h" +#include "nodes/pg_list.h" +#include "postmaster/postmaster.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" -#include "distributed/shared_library_init.h" + #include "distributed/adaptive_executor.h" #include "distributed/colocation_utils.h" +#include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" #include "distributed/hash_helpers.h" #include "distributed/metadata_cache.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/connection_management.h" -#include "distributed/remote_commands.h" -#include "distributed/shard_split.h" -#include "distributed/reference_table_utils.h" -#include "distributed/shard_transfer.h" -#include "distributed/resource_lock.h" +#include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/reference_table_utils.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" +#include "distributed/shard_split.h" +#include "distributed/shard_transfer.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shardsplit_logical_replication.h" +#include "distributed/shared_library_init.h" +#include "distributed/utils/array_type.h" +#include "distributed/utils/distribution_column_map.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shared_library_init.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/utils/distribution_column_map.h" -#include "commands/dbcommands.h" -#include "distributed/shardsplit_logical_replication.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/shard_rebalancer.h" -#include "postmaster/postmaster.h" /* * Entry for map that tracks ShardInterval -> Placement Node diff --git a/src/backend/distributed/operations/shard_transfer.c b/src/backend/distributed/operations/shard_transfer.c index 23925a315..49d468cb1 100644 --- a/src/backend/distributed/operations/shard_transfer.c +++ b/src/backend/distributed/operations/shard_transfer.c @@ -9,27 +9,39 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "fmgr.h" -#include "miscadmin.h" - #include #include +#include "postgres.h" + +#include "fmgr.h" +#include "miscadmin.h" + #include "access/htup_details.h" #include "catalog/pg_class.h" #include "catalog/pg_enum.h" +#include "lib/stringinfo.h" +#include "nodes/pg_list.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/errcodes.h" +#include "utils/lsyscache.h" +#include "utils/palloc.h" +#include "utils/rel.h" +#include "utils/syscache.h" + #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" -#include "distributed/shard_cleaner.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" @@ -39,24 +51,13 @@ #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_split.h" #include "distributed/shard_transfer.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" -#include "lib/stringinfo.h" -#include "nodes/pg_list.h" -#include "storage/lmgr.h" -#include "storage/lock.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/elog.h" -#include "utils/errcodes.h" -#include "utils/lsyscache.h" -#include "utils/palloc.h" -#include "utils/rel.h" -#include "utils/syscache.h" /* local type declarations */ diff --git a/src/backend/distributed/operations/stage_protocol.c b/src/backend/distributed/operations/stage_protocol.c index 421593c66..5770d648e 100644 --- a/src/backend/distributed/operations/stage_protocol.c +++ b/src/backend/distributed/operations/stage_protocol.c @@ -15,31 +15,40 @@ */ #include "postgres.h" + #include "funcapi.h" -#include "miscadmin.h" #include "libpq-fe.h" +#include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" -#include "commands/tablecmds.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/partition.h" +#include "commands/tablecmds.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +#include "distributed/adaptive_executor.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" -#include "distributed/adaptive_executor.h" #include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/foreign_key_relationship.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" -#include "distributed/multi_executor.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_partition.h" @@ -50,16 +59,9 @@ #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/inval.h" -#include "utils/lsyscache.h" -#include "utils/syscache.h" -#include "utils/rel.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/operations/worker_copy_table_to_node_udf.c b/src/backend/distributed/operations/worker_copy_table_to_node_udf.c index f0f83744d..c603de72a 100644 --- a/src/backend/distributed/operations/worker_copy_table_to_node_udf.c +++ b/src/backend/distributed/operations/worker_copy_table_to_node_udf.c @@ -15,6 +15,7 @@ #include "utils/builtins.h" #include "utils/lsyscache.h" + #include "distributed/citus_ruleutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" diff --git a/src/backend/distributed/operations/worker_node_manager.c b/src/backend/distributed/operations/worker_node_manager.c index 76f2732ba..192f6b466 100644 --- a/src/backend/distributed/operations/worker_node_manager.c +++ b/src/backend/distributed/operations/worker_node_manager.c @@ -12,16 +12,13 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "commands/dbcommands.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/hash_helpers.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/worker_manager.h" -#include "libpq/hba.h" +#include "common/hashfn.h" #include "common/ip.h" +#include "libpq/hba.h" #include "libpq/libpq-be.h" #include "postmaster/postmaster.h" #include "storage/fd.h" @@ -31,7 +28,12 @@ #include "utils/guc.h" #include "utils/hsearch.h" #include "utils/memutils.h" -#include "common/hashfn.h" + +#include "distributed/coordinator_protocol.h" +#include "distributed/hash_helpers.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/worker_manager.h" /* Config variables managed via guc.c */ diff --git a/src/backend/distributed/operations/worker_shard_copy.c b/src/backend/distributed/operations/worker_shard_copy.c index ba65635a7..f99c9b537 100644 --- a/src/backend/distributed/operations/worker_shard_copy.c +++ b/src/backend/distributed/operations/worker_shard_copy.c @@ -8,23 +8,26 @@ *------------------------------------------------------------------------- */ -#include "libpq-fe.h" #include "postgres.h" + +#include "libpq-fe.h" + #include "commands/copy.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" -#include "utils/lsyscache.h" #include "utils/builtins.h" -#include "distributed/remote_commands.h" -#include "distributed/worker_shard_copy.h" +#include "utils/lsyscache.h" + #include "distributed/commands/multi_copy.h" -#include "distributed/local_multi_copy.h" -#include "distributed/worker_manager.h" #include "distributed/connection_management.h" -#include "distributed/relation_utils.h" -#include "distributed/version_compat.h" #include "distributed/local_executor.h" +#include "distributed/local_multi_copy.h" +#include "distributed/relation_utils.h" +#include "distributed/remote_commands.h" #include "distributed/replication_origin_session_utils.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_shard_copy.h" /* * LocalCopyBuffer is used in copy callback to return the copied rows. diff --git a/src/backend/distributed/operations/worker_split_copy_udf.c b/src/backend/distributed/operations/worker_split_copy_udf.c index 18fdbfc4a..03354ea04 100644 --- a/src/backend/distributed/operations/worker_split_copy_udf.c +++ b/src/backend/distributed/operations/worker_split_copy_udf.c @@ -8,7 +8,13 @@ */ #include "postgres.h" + +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" + #include "pg_version_compat.h" + #include "distributed/citus_ruleutils.h" #include "distributed/distribution_column.h" #include "distributed/intermediate_results.h" @@ -16,9 +22,6 @@ #include "distributed/multi_executor.h" #include "distributed/utils/array_type.h" #include "distributed/worker_shard_copy.h" -#include "utils/lsyscache.h" -#include "utils/array.h" -#include "utils/builtins.h" PG_FUNCTION_INFO_V1(worker_split_copy); diff --git a/src/backend/distributed/operations/worker_split_shard_release_dsm_udf.c b/src/backend/distributed/operations/worker_split_shard_release_dsm_udf.c index 94ce40cdb..7f3f3ff7a 100644 --- a/src/backend/distributed/operations/worker_split_shard_release_dsm_udf.c +++ b/src/backend/distributed/operations/worker_split_shard_release_dsm_udf.c @@ -9,6 +9,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_shared_memory.h" diff --git a/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c b/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c index 4d116dfa1..d4775995c 100644 --- a/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c +++ b/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c @@ -9,24 +9,27 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "miscadmin.h" -#include "postmaster/postmaster.h" + +#include "commands/dbcommands.h" #include "common/hashfn.h" -#include "distributed/distribution_column.h" -#include "distributed/hash_helpers.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_utils.h" -#include "distributed/shardsplit_shared_memory.h" -#include "distributed/connection_management.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/listutils.h" -#include "distributed/remote_commands.h" -#include "distributed/tuplestore.h" -#include "distributed/shardsplit_logical_replication.h" +#include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/lsyscache.h" -#include "commands/dbcommands.h" + +#include "distributed/citus_safe_lib.h" +#include "distributed/connection_management.h" +#include "distributed/distribution_column.h" +#include "distributed/hash_helpers.h" +#include "distributed/listutils.h" +#include "distributed/remote_commands.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_utils.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shardsplit_logical_replication.h" +#include "distributed/shardsplit_shared_memory.h" +#include "distributed/tuplestore.h" /* declarations for dynamic loading */ diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index 6a171dac1..e3aa7b3e6 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -11,21 +11,22 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "catalog/pg_type.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/insert_select_planner.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/combine_query_planner.h" -#include "distributed/multi_physical_planner.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/planner.h" #include "rewrite/rewriteManip.h" +#include "pg_version_constants.h" + +#include "distributed/citus_ruleutils.h" +#include "distributed/combine_query_planner.h" +#include "distributed/insert_select_planner.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_physical_planner.h" + static List * RemoteScanTargetList(List *workerTargetList); static PlannedStmt * BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetList, diff --git a/src/backend/distributed/planner/cte_inline.c b/src/backend/distributed/planner/cte_inline.c index 9a1bbab96..d6f88525c 100644 --- a/src/backend/distributed/planner/cte_inline.c +++ b/src/backend/distributed/planner/cte_inline.c @@ -12,13 +12,15 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + +#include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" +#include "rewrite/rewriteManip.h" + #include "pg_version_compat.h" #include "pg_version_constants.h" #include "distributed/cte_inline.h" -#include "nodes/nodeFuncs.h" -#include "optimizer/optimizer.h" -#include "rewrite/rewriteManip.h" typedef struct inline_cte_walker_context { diff --git a/src/backend/distributed/planner/deparse_shard_query.c b/src/backend/distributed/planner/deparse_shard_query.c index ac37b1399..43b5f1493 100644 --- a/src/backend/distributed/planner/deparse_shard_query.c +++ b/src/backend/distributed/planner/deparse_shard_query.c @@ -10,11 +10,24 @@ */ #include "postgres.h" + #include "c.h" #include "access/heapam.h" #include "access/htup_details.h" #include "catalog/pg_constraint.h" +#include "lib/stringinfo.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" +#include "nodes/pg_list.h" +#include "parser/parsetree.h" +#include "storage/lock.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/combine_query_planner.h" @@ -28,17 +41,6 @@ #include "distributed/shard_utils.h" #include "distributed/utils/citus_stat_tenants.h" #include "distributed/version_compat.h" -#include "lib/stringinfo.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/nodes.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" -#include "parser/parsetree.h" -#include "storage/lock.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/syscache.h" static void UpdateTaskQueryString(Query *query, Task *task); diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index ed644c8a7..f00fd4090 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -7,65 +7,66 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - -#include "funcapi.h" - #include #include +#include "postgres.h" + +#include "funcapi.h" + #include "access/htup_details.h" #include "access/xact.h" #include "catalog/pg_class.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" +#include "executor/executor.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pg_list.h" + +#include "pg_version_constants.h" + #include "distributed/citus_depended_object.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" +#include "distributed/combine_query_planner.h" #include "distributed/commands.h" +#include "distributed/coordinator_protocol.h" #include "distributed/cte_inline.h" +#include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_planner.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/merge_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" -#include "distributed/distributed_planner.h" -#include "distributed/query_pushdown_planning.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" -#include "distributed/combine_query_planner.h" #include "distributed/multi_router_planner.h" +#include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" -#include "distributed/shardinterval_utils.h" #include "distributed/shard_utils.h" +#include "distributed/shardinterval_utils.h" #include "distributed/utils/citus_stat_tenants.h" #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" -#include "executor/executor.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pg_list.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "parser/parse_relation.h" #endif -#include "parser/parsetree.h" -#include "parser/parse_type.h" #include "optimizer/optimizer.h" -#include "optimizer/plancat.h" #include "optimizer/pathnode.h" -#include "optimizer/planner.h" +#include "optimizer/plancat.h" #include "optimizer/planmain.h" +#include "optimizer/planner.h" +#include "parser/parse_type.h" +#include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/planner/extended_op_node_utils.c b/src/backend/distributed/planner/extended_op_node_utils.c index bb87b6949..7912de1d9 100644 --- a/src/backend/distributed/planner/extended_op_node_utils.c +++ b/src/backend/distributed/planner/extended_op_node_utils.c @@ -9,6 +9,12 @@ */ #include "postgres.h" + +#include "nodes/nodeFuncs.h" +#include "nodes/pg_list.h" +#include "optimizer/optimizer.h" +#include "optimizer/restrictinfo.h" + #include "pg_version_constants.h" #include "distributed/extended_op_node_utils.h" @@ -16,10 +22,6 @@ #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/pg_dist_partition.h" -#include "optimizer/optimizer.h" -#include "optimizer/restrictinfo.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pg_list.h" static bool GroupedByPartitionColumn(MultiNode *node, MultiExtendedOp *opNode); diff --git a/src/backend/distributed/planner/fast_path_router_planner.c b/src/backend/distributed/planner/fast_path_router_planner.c index 6037da534..fdc13332e 100644 --- a/src/backend/distributed/planner/fast_path_router_planner.c +++ b/src/backend/distributed/planner/fast_path_router_planner.c @@ -34,16 +34,6 @@ */ #include "postgres.h" -#include "pg_version_constants.h" - -#include "distributed/distributed_planner.h" -#include "distributed/insert_select_planner.h" -#include "distributed/multi_physical_planner.h" /* only to use some utility functions */ -#include "distributed/metadata_cache.h" -#include "distributed/multi_router_planner.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shard_pruning.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" @@ -51,6 +41,17 @@ #include "optimizer/optimizer.h" #include "tcop/pquery.h" +#include "pg_version_constants.h" + +#include "distributed/distributed_planner.h" +#include "distributed/insert_select_planner.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_physical_planner.h" /* only to use some utility functions */ +#include "distributed/multi_router_planner.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/shard_pruning.h" +#include "distributed/shardinterval_utils.h" + bool EnableFastPathRouterPlanner = true; static bool ColumnAppearsMultipleTimes(Node *quals, Var *distributionKey); diff --git a/src/backend/distributed/planner/function_call_delegation.c b/src/backend/distributed/planner/function_call_delegation.c index aeea3b714..c72982e2a 100644 --- a/src/backend/distributed/planner/function_call_delegation.c +++ b/src/backend/distributed/planner/function_call_delegation.c @@ -12,32 +12,11 @@ #include "postgres.h" -#include "pg_version_constants.h" +#include "miscadmin.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/defrem.h" -#include "distributed/backend_data.h" -#include "distributed/metadata_utility.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/connection_management.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/function_call_delegation.h" -#include "distributed/insert_select_planner.h" -#include "distributed/citus_custom_scan.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/remote_commands.h" -#include "distributed/shard_pruning.h" -#include "distributed/recursive_planning.h" -#include "distributed/version_compat.h" -#include "distributed/worker_manager.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" @@ -46,11 +25,34 @@ #include "optimizer/clauses.h" #include "parser/parse_coerce.h" #include "parser/parsetree.h" -#include "miscadmin.h" #include "tcop/dest.h" #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "pg_version_constants.h" + +#include "distributed/backend_data.h" +#include "distributed/citus_custom_scan.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/function_call_delegation.h" +#include "distributed/insert_select_planner.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/recursive_planning.h" +#include "distributed/remote_commands.h" +#include "distributed/shard_pruning.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" + struct ParamWalkerContext { bool hasParam; diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index dd4bee90f..60d6ce466 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -10,22 +10,39 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "catalog/pg_class.h" #include "catalog/pg_type.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/parsenodes.h" +#include "nodes/print.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" +#include "optimizer/planner.h" +#include "optimizer/restrictinfo.h" +#include "optimizer/tlist.h" +#include "parser/parse_coerce.h" +#include "parser/parse_relation.h" +#include "parser/parsetree.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" + +#include "pg_version_constants.h" + #include "distributed/citus_clauses.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/errormessage.h" -#include "distributed/listutils.h" -#include "distributed/log_utils.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" -#include "distributed/multi_logical_planner.h" #include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" @@ -34,22 +51,6 @@ #include "distributed/repartition_executor.h" #include "distributed/resource_lock.h" #include "distributed/version_compat.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/parsenodes.h" -#include "optimizer/clauses.h" -#include "optimizer/planner.h" -#include "optimizer/restrictinfo.h" -#include "optimizer/tlist.h" -#include "optimizer/optimizer.h" -#include "parser/parsetree.h" -#include "parser/parse_coerce.h" -#include "parser/parse_relation.h" -#include "tcop/tcopprot.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include static void PrepareInsertSelectForCitusPlanner(Query *insertSelectQuery); diff --git a/src/backend/distributed/planner/intermediate_result_pruning.c b/src/backend/distributed/planner/intermediate_result_pruning.c index cefbfb833..474d767d7 100644 --- a/src/backend/distributed/planner/intermediate_result_pruning.c +++ b/src/backend/distributed/planner/intermediate_result_pruning.c @@ -12,6 +12,9 @@ *------------------------------------------------------------------------- */ +#include "common/hashfn.h" +#include "utils/builtins.h" + #include "distributed/citus_custom_scan.h" #include "distributed/citus_ruleutils.h" #include "distributed/intermediate_result_pruning.h" @@ -20,8 +23,6 @@ #include "distributed/metadata_cache.h" #include "distributed/query_utils.h" #include "distributed/worker_manager.h" -#include "utils/builtins.h" -#include "common/hashfn.h" /* controlled via GUC, used mostly for testing */ bool LogIntermediateResults = false; diff --git a/src/backend/distributed/planner/local_distributed_join_planner.c b/src/backend/distributed/planner/local_distributed_join_planner.c index 1867a790c..a6502bf43 100644 --- a/src/backend/distributed/planner/local_distributed_join_planner.c +++ b/src/backend/distributed/planner/local_distributed_join_planner.c @@ -71,55 +71,53 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "funcapi.h" -#include "catalog/pg_type.h" #include "catalog/pg_class.h" #include "catalog/pg_index.h" -#include "distributed/citus_nodes.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/distributed_planner.h" -#include "distributed/errormessage.h" -#include "distributed/local_distributed_join_planner.h" -#include "distributed/listutils.h" -#include "distributed/log_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_router_planner.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/multi_router_planner.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/query_colocation_checker.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/recursive_planning.h" -#include "distributed/relation_restriction_equivalence.h" -#include "distributed/log_utils.h" -#include "distributed/shard_pruning.h" -#include "distributed/version_compat.h" +#include "catalog/pg_type.h" #include "lib/stringinfo.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" +#include "nodes/pathnodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/nodes.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "nodes/pathnodes.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" +#include "pg_version_constants.h" + +#include "distributed/citus_nodes.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" +#include "distributed/errormessage.h" +#include "distributed/listutils.h" +#include "distributed/local_distributed_join_planner.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/query_colocation_checker.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/recursive_planning.h" +#include "distributed/relation_restriction_equivalence.h" +#include "distributed/shard_pruning.h" +#include "distributed/version_compat.h" + #define INVALID_RTE_IDENTITY -1 /* diff --git a/src/backend/distributed/planner/local_plan_cache.c b/src/backend/distributed/planner/local_plan_cache.c index 1ac8e24a3..2e5ca4e55 100644 --- a/src/backend/distributed/planner/local_plan_cache.c +++ b/src/backend/distributed/planner/local_plan_cache.c @@ -9,19 +9,20 @@ */ #include "postgres.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" + #include "pg_version_constants.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_plan_cache.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/insert_select_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/version_compat.h" -#include "optimizer/optimizer.h" -#include "optimizer/clauses.h" static Query * GetLocalShardQueryForCache(Query *jobQuery, Task *task, diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 5c593d153..4d64b8f56 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -12,6 +12,7 @@ #include #include "postgres.h" + #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" @@ -20,6 +21,8 @@ #include "tcop/tcopprot.h" #include "utils/lsyscache.h" +#include "pg_version_constants.h" + #include "distributed/citus_clauses.h" #include "distributed/citus_custom_scan.h" #include "distributed/insert_select_planner.h" @@ -29,12 +32,11 @@ #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_node_metadata.h" -#include "pg_version_constants.h" -#include "distributed/query_pushdown_planning.h" #include "distributed/query_colocation_checker.h" +#include "distributed/query_pushdown_planning.h" #include "distributed/repartition_executor.h" -#include "distributed/shared_library_init.h" #include "distributed/shard_pruning.h" +#include "distributed/shared_library_init.h" #if PG_VERSION_NUM >= PG_VERSION_15 diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index bf9a1871e..1d6a88934 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -8,11 +8,11 @@ */ #include "postgres.h" + +#include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" -#include "pg_version_constants.h" - #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" @@ -24,41 +24,13 @@ #include "commands/dbcommands.h" #include "commands/explain.h" #include "commands/tablecmds.h" -#include "optimizer/cost.h" -#include "distributed/citus_depended_object.h" -#include "distributed/citus_nodefuncs.h" -#include "distributed/connection_management.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/executor_util.h" -#include "distributed/insert_select_planner.h" -#include "distributed/insert_select_executor.h" -#include "distributed/listutils.h" -#include "distributed/merge_planner.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_explain.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/combine_query_planner.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_router_planner.h" -#include "distributed/distributed_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/remote_commands.h" -#include "distributed/recursive_planning.h" -#include "distributed/placement_connection.h" -#include "distributed/tuple_destination.h" -#include "distributed/tuplestore.h" -#include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" -#include "distributed/jsonbutils.h" -#include "distributed/commands/utility_hook.h" #include "executor/tstoreReceiver.h" -#include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/plannodes.h" #include "nodes/primnodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" +#include "optimizer/cost.h" #include "optimizer/planner.h" #include "parser/analyze.h" #include "portability/instr_time.h" @@ -71,6 +43,36 @@ #include "utils/lsyscache.h" #include "utils/snapmgr.h" +#include "pg_version_constants.h" + +#include "distributed/citus_depended_object.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/combine_query_planner.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/connection_management.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/distributed_planner.h" +#include "distributed/executor_util.h" +#include "distributed/insert_select_executor.h" +#include "distributed/insert_select_planner.h" +#include "distributed/jsonbutils.h" +#include "distributed/listutils.h" +#include "distributed/merge_planner.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_explain.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/placement_connection.h" +#include "distributed/recursive_planning.h" +#include "distributed/remote_commands.h" +#include "distributed/tuple_destination.h" +#include "distributed/tuplestore.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" + /* Config variables that enable printing distributed query plans */ bool ExplainDistributedQueries = true; diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index 0eede6b9b..908ed206e 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -11,31 +11,31 @@ *------------------------------------------------------------------------- */ +#include + #include "postgres.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/nbtree.h" +#include "catalog/pg_am.h" +#include "lib/stringinfo.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + #include "pg_version_constants.h" -#include - -#include "access/nbtree.h" -#include "access/heapam.h" -#include "access/htup_details.h" -#include "catalog/pg_am.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/worker_protocol.h" -#include "lib/stringinfo.h" -#include "optimizer/optimizer.h" -#include "utils/builtins.h" -#include "nodes/nodeFuncs.h" -#include "utils/builtins.h" -#include "utils/datum.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/syscache.h" /* Config variables managed via guc.c */ diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 9001d724d..76e38237a 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -11,12 +11,10 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - #include +#include "postgres.h" + #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" @@ -27,6 +25,23 @@ #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/extension.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" +#include "optimizer/tlist.h" +#include "parser/parse_agg.h" +#include "parser/parse_coerce.h" +#include "parser/parse_oper.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteManip.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" @@ -42,22 +57,8 @@ #include "distributed/query_pushdown_planning.h" #include "distributed/string_utils.h" #include "distributed/tdigest_extension.h" -#include "distributed/worker_protocol.h" #include "distributed/version_compat.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "optimizer/clauses.h" -#include "optimizer/tlist.h" -#include "optimizer/optimizer.h" -#include "parser/parse_agg.h" -#include "parser/parse_coerce.h" -#include "parser/parse_oper.h" -#include "parser/parsetree.h" -#include "rewrite/rewriteManip.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/syscache.h" +#include "distributed/worker_protocol.h" /* Config variable managed via guc.c */ int LimitClauseRowFetchCount = -1; /* number of rows to fetch from each task */ diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index d6897d17b..f62e309f2 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -14,42 +14,43 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "access/heapam.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_class.h" #include "commands/defrem.h" -#include "distributed/citus_clauses.h" -#include "distributed/colocation_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/insert_select_planner.h" -#include "distributed/listutils.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_restriction_equivalence.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/query_utils.h" -#include "distributed/multi_router_planner.h" -#include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" -#include "optimizer/optimizer.h" #include "optimizer/clauses.h" +#include "optimizer/optimizer.h" #include "optimizer/prep.h" #include "optimizer/tlist.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" -#include "utils/syscache.h" #include "utils/rel.h" #include "utils/relcache.h" +#include "utils/syscache.h" + +#include "pg_version_constants.h" + +#include "distributed/citus_clauses.h" +#include "distributed/colocation_utils.h" +#include "distributed/insert_select_planner.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/query_utils.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_restriction_equivalence.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" /* Struct to differentiate different qualifier types in an expression tree walker */ diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index aa2c2b5b4..fb7f844c7 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -11,13 +11,11 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - #include #include +#include "postgres.h" + #include "miscadmin.h" #include "access/genam.h" @@ -33,39 +31,11 @@ #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/sequence.h" -#include "distributed/backend_data.h" -#include "distributed/listutils.h" -#include "distributed/citus_nodefuncs.h" -#include "distributed/citus_nodes.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/colocation_utils.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/intermediate_results.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_router_planner.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/log_utils.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/query_utils.h" -#include "distributed/recursive_planning.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shard_pruning.h" -#include "distributed/string_utils.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#include "nodes/pathnodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" -#include "nodes/pathnodes.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" @@ -84,6 +54,37 @@ #include "utils/syscache.h" #include "utils/typcache.h" +#include "pg_version_constants.h" + +#include "distributed/backend_data.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_nodes.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/intermediate_results.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/query_utils.h" +#include "distributed/recursive_planning.h" +#include "distributed/shard_pruning.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/string_utils.h" +#include "distributed/version_compat.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" + /* RepartitionJoinBucketCountPerNode determines bucket amount during repartitions */ int RepartitionJoinBucketCountPerNode = 4; diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 60aa81e73..1f2ad4751 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -11,50 +11,15 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - #include +#include "postgres.h" + #include "access/stratnum.h" #include "access/xact.h" #include "catalog/pg_opfamily.h" +#include "catalog/pg_proc.h" #include "catalog/pg_type.h" -#include "distributed/colocation_utils.h" -#include "distributed/citus_clauses.h" -#include "distributed/citus_nodes.h" -#include "distributed/citus_nodefuncs.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/distribution_column.h" -#include "distributed/errormessage.h" -#include "distributed/executor_util.h" -#include "distributed/log_utils.h" -#include "distributed/insert_select_planner.h" -#include "distributed/intermediate_result_pruning.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/merge_planner.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_router_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/listutils.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/query_utils.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_restriction_equivalence.h" -#include "distributed/relay_utility.h" -#include "distributed/recursive_planning.h" -#include "distributed/resource_lock.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shard_pruning.h" #include "executor/execdesc.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" @@ -65,12 +30,13 @@ #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/joininfo.h" +#include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" -#include "optimizer/optimizer.h" +#include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" -#include "parser/parsetree.h" #include "parser/parse_oper.h" +#include "parser/parsetree.h" #include "postmaster/postmaster.h" #include "storage/lock.h" #include "utils/builtins.h" @@ -80,8 +46,42 @@ #include "utils/rel.h" #include "utils/typcache.h" -#include "catalog/pg_proc.h" -#include "optimizer/planmain.h" +#include "pg_version_constants.h" + +#include "distributed/citus_clauses.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_nodes.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/colocation_utils.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/distribution_column.h" +#include "distributed/errormessage.h" +#include "distributed/executor_util.h" +#include "distributed/insert_select_planner.h" +#include "distributed/intermediate_result_pruning.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" +#include "distributed/merge_planner.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/query_utils.h" +#include "distributed/recursive_planning.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_restriction_equivalence.h" +#include "distributed/relay_utility.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_pruning.h" +#include "distributed/shardinterval_utils.h" /* intermediate value for INSERT processing */ typedef struct InsertValues diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index fd1df1be9..827a0286c 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -21,26 +21,26 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "access/relation.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/query_colocation_checker.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/relation_restriction_equivalence.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_logical_planner.h" /* only to access utility functions */ - #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" -#include "parser/parsetree.h" -#include "distributed/listutils.h" -#include "parser/parse_relation.h" #include "optimizer/planner.h" #include "optimizer/prep.h" +#include "parser/parse_relation.h" +#include "parser/parsetree.h" #include "utils/rel.h" +#include "pg_version_constants.h" + +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_logical_planner.h" /* only to access utility functions */ +#include "distributed/pg_dist_partition.h" +#include "distributed/query_colocation_checker.h" +#include "distributed/relation_restriction_equivalence.h" + static RangeTblEntry * AnchorRte(Query *subquery); static List * UnionRelationRestrictionLists(List *firstRelationList, diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 8ccc35c82..2eda4e42a 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -21,6 +21,13 @@ #include "postgres.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pg_list.h" +#include "optimizer/clauses.h" +#include "optimizer/optimizer.h" +#include "parser/parsetree.h" + #include "pg_version_constants.h" #include "distributed/citus_clauses.h" @@ -32,17 +39,11 @@ #include "distributed/multi_logical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" -#include "distributed/query_utils.h" #include "distributed/query_pushdown_planning.h" +#include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/version_compat.h" -#include "nodes/nodeFuncs.h" -#include "nodes/makefuncs.h" -#include "optimizer/optimizer.h" -#include "nodes/pg_list.h" -#include "optimizer/clauses.h" -#include "parser/parsetree.h" #define INVALID_RELID -1 diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index d16280662..6c42046ff 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -48,51 +48,50 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "funcapi.h" -#include "catalog/pg_type.h" #include "catalog/pg_class.h" -#include "distributed/citus_nodes.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/distributed_planner.h" -#include "distributed/errormessage.h" -#include "distributed/local_distributed_join_planner.h" -#include "distributed/listutils.h" -#include "distributed/log_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/multi_router_planner.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/query_colocation_checker.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/recursive_planning.h" -#include "distributed/relation_restriction_equivalence.h" -#include "distributed/log_utils.h" -#include "distributed/shard_pruning.h" -#include "distributed/version_compat.h" +#include "catalog/pg_type.h" #include "lib/stringinfo.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" +#include "nodes/pathnodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/nodes.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "nodes/pathnodes.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" +#include "pg_version_constants.h" + +#include "distributed/citus_nodes.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/distributed_planner.h" +#include "distributed/errormessage.h" +#include "distributed/listutils.h" +#include "distributed/local_distributed_join_planner.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/query_colocation_checker.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/recursive_planning.h" +#include "distributed/relation_restriction_equivalence.h" +#include "distributed/shard_pruning.h" +#include "distributed/version_compat.h" + /* * RecursivePlanningContext is used to recursively plan subqueries * and CTEs, pull results to the coordinator, and push it back into diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 4b51a537d..83d7cbcdb 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -10,31 +10,31 @@ */ #include "postgres.h" +#include "catalog/pg_type.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pathnodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" +#include "optimizer/optimizer.h" +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "parser/parsetree.h" + #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" -#include "distributed/multi_logical_planner.h" #include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/query_utils.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/shard_pruning.h" -#include "catalog/pg_type.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "nodes/pathnodes.h" -#include "optimizer/optimizer.h" -#include "nodes/makefuncs.h" -#include "optimizer/paths.h" -#include "parser/parsetree.h" -#include "optimizer/pathnode.h" - static uint32 AttributeEquivalenceId = 1; diff --git a/src/backend/distributed/planner/shard_pruning.c b/src/backend/distributed/planner/shard_pruning.c index ef244ea66..e68ac72b0 100644 --- a/src/backend/distributed/planner/shard_pruning.c +++ b/src/backend/distributed/planner/shard_pruning.c @@ -66,28 +66,14 @@ */ #include "postgres.h" -#include "pg_version_constants.h" - #include "fmgr.h" -#include "distributed/shard_pruning.h" - #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" -#include "distributed/distributed_planner.h" -#include "distributed/listutils.h" -#include "distributed/log_utils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/version_compat.h" -#include "distributed/worker_protocol.h" -#include "nodes/nodeFuncs.h" #include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/planner.h" #include "parser/parse_coerce.h" @@ -98,6 +84,20 @@ #include "utils/memutils.h" #include "utils/ruleutils.h" +#include "pg_version_constants.h" + +#include "distributed/distributed_planner.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/shard_pruning.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" + /* * Tree node for compact representation of the given query logical tree. diff --git a/src/backend/distributed/planner/tdigest_extension.c b/src/backend/distributed/planner/tdigest_extension.c index 123b170d4..3a3701940 100644 --- a/src/backend/distributed/planner/tdigest_extension.c +++ b/src/backend/distributed/planner/tdigest_extension.c @@ -12,13 +12,14 @@ #include "access/htup_details.h" #include "catalog/pg_extension.h" #include "catalog/pg_type.h" -#include "distributed/metadata_cache.h" -#include "distributed/tdigest_extension.h" -#include "distributed/version_compat.h" #include "parser/parse_func.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "distributed/metadata_cache.h" +#include "distributed/tdigest_extension.h" +#include "distributed/version_compat.h" + static Oid LookupTDigestFunction(const char *functionName, int argcount, Oid *argtypes); diff --git a/src/backend/distributed/progress/multi_progress.c b/src/backend/distributed/progress/multi_progress.c index 8a3adf4bc..64e0a5b47 100644 --- a/src/backend/distributed/progress/multi_progress.c +++ b/src/backend/distributed/progress/multi_progress.c @@ -8,15 +8,17 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "pgstat.h" +#include "storage/dsm.h" +#include "utils/builtins.h" + #include "distributed/function_utils.h" #include "distributed/listutils.h" #include "distributed/multi_progress.h" #include "distributed/version_compat.h" -#include "storage/dsm.h" -#include "utils/builtins.h" /* dynamic shared memory handle of the current progress */ diff --git a/src/backend/distributed/relay/relay_event_utility.c b/src/backend/distributed/relay/relay_event_utility.c index 3284ead11..d0267025b 100644 --- a/src/backend/distributed/relay/relay_event_utility.c +++ b/src/backend/distributed/relay/relay_event_utility.c @@ -12,34 +12,28 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "c.h" - #include #include +#include "postgres.h" + +#include "c.h" + #include "access/genam.h" -#include "access/heapam.h" -#include "access/htup_details.h" #include "access/hash.h" +#include "access/heapam.h" #include "access/htup.h" +#include "access/htup_details.h" #include "access/skey.h" #include "access/stratnum.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/relay_utility.h" -#include "distributed/version_compat.h" #include "lib/stringinfo.h" #include "mb/pg_wchar.h" -#include "nodes/nodes.h" #include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" @@ -53,6 +47,14 @@ #include "utils/palloc.h" #include "utils/relcache.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/commands.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/relay_utility.h" +#include "distributed/version_compat.h" + /* Local functions forward declarations */ static void RelayEventExtendConstraintAndIndexNames(AlterTableStmt *alterTableStmt, Constraint *constraint, diff --git a/src/backend/distributed/replication/multi_logical_replication.c b/src/backend/distributed/replication/multi_logical_replication.c index 30491f42c..ea61b77ca 100644 --- a/src/backend/distributed/replication/multi_logical_replication.c +++ b/src/backend/distributed/replication/multi_logical_replication.c @@ -10,55 +10,32 @@ *------------------------------------------------------------------------- */ #include "postgres.h" -#include "miscadmin.h" -#include "fmgr.h" -#include "pgstat.h" -#include "libpq-fe.h" -#include "pg_version_constants.h" +#include "fmgr.h" +#include "libpq-fe.h" +#include "miscadmin.h" +#include "pgstat.h" #include "access/genam.h" - -#include "postmaster/interrupt.h" - #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" -#include "commands/dbcommands.h" -#include "common/hashfn.h" -#include "catalog/pg_subscription_rel.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" -#include "distributed/adaptive_executor.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/colocation_utils.h" -#include "distributed/connection_management.h" -#include "distributed/hash_helpers.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_logical_replication.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/priority.h" -#include "distributed/distributed_planner.h" -#include "distributed/remote_commands.h" -#include "distributed/resource_lock.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/shard_transfer.h" -#include "distributed/version_compat.h" +#include "catalog/pg_subscription_rel.h" +#include "commands/dbcommands.h" +#include "common/hashfn.h" #include "nodes/bitmapset.h" #include "parser/scansup.h" +#include "postmaster/interrupt.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/lock.h" -#include "utils/guc.h" #include "utils/builtins.h" -#include "utils/fmgrprotos.h" #include "utils/fmgroids.h" +#include "utils/fmgrprotos.h" #include "utils/formatting.h" +#include "utils/guc.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/pg_lsn.h" @@ -66,6 +43,29 @@ #include "utils/ruleutils.h" #include "utils/syscache.h" +#include "pg_version_constants.h" + +#include "distributed/adaptive_executor.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/colocation_utils.h" +#include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" +#include "distributed/hash_helpers.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_logical_replication.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/priority.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" +#include "distributed/shard_transfer.h" +#include "distributed/version_compat.h" + #define CURRENT_LOG_POSITION_COMMAND "SELECT pg_current_wal_lsn()" /* decimal representation of Adler-16 hash value of citus_shard_move_publication */ diff --git a/src/backend/distributed/shardsplit/shardsplit_decoder.c b/src/backend/distributed/shardsplit/shardsplit_decoder.c index 1386a21b0..51ab769f3 100644 --- a/src/backend/distributed/shardsplit/shardsplit_decoder.c +++ b/src/backend/distributed/shardsplit/shardsplit_decoder.c @@ -8,16 +8,18 @@ *------------------------------------------------------------------------- */ #include "postgres.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shardsplit_shared_memory.h" -#include "distributed/worker_shard_visibility.h" -#include "distributed/worker_protocol.h" + +#include "catalog/pg_namespace.h" +#include "replication/logical.h" +#include "utils/lsyscache.h" +#include "utils/typcache.h" + #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" -#include "replication/logical.h" -#include "utils/typcache.h" -#include "utils/lsyscache.h" -#include "catalog/pg_namespace.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shardsplit_shared_memory.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_shard_visibility.h" extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); static LogicalDecodeChangeCB pgOutputPluginChangeCB; diff --git a/src/backend/distributed/shardsplit/shardsplit_logical_replication.c b/src/backend/distributed/shardsplit/shardsplit_logical_replication.c index 8ffccb90c..328dc9af9 100644 --- a/src/backend/distributed/shardsplit/shardsplit_logical_replication.c +++ b/src/backend/distributed/shardsplit/shardsplit_logical_replication.c @@ -10,23 +10,26 @@ */ #include "postgres.h" + #include "miscadmin.h" + +#include "commands/dbcommands.h" #include "nodes/pg_list.h" +#include "utils/builtins.h" + #include "distributed/colocation_utils.h" +#include "distributed/connection_management.h" #include "distributed/hash_helpers.h" +#include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/priority.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/connection_management.h" #include "distributed/remote_commands.h" -#include "distributed/shard_split.h" -#include "distributed/shared_library_init.h" -#include "distributed/listutils.h" -#include "distributed/shardsplit_logical_replication.h" #include "distributed/resource_lock.h" -#include "utils/builtins.h" -#include "commands/dbcommands.h" +#include "distributed/shard_split.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shardsplit_logical_replication.h" +#include "distributed/shared_library_init.h" static HTAB *ShardInfoHashMapForPublications = NULL; diff --git a/src/backend/distributed/shardsplit/shardsplit_shared_memory.c b/src/backend/distributed/shardsplit/shardsplit_shared_memory.c index 3e8745758..16ed79ad7 100644 --- a/src/backend/distributed/shardsplit/shardsplit_shared_memory.c +++ b/src/backend/distributed/shardsplit/shardsplit_shared_memory.c @@ -12,13 +12,15 @@ */ #include "postgres.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/shardsplit_shared_memory.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/multi_logical_replication.h" + +#include "common/hashfn.h" #include "storage/ipc.h" #include "utils/memutils.h" -#include "common/hashfn.h" + +#include "distributed/citus_safe_lib.h" +#include "distributed/multi_logical_replication.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/shardsplit_shared_memory.h" const char *SharedMemoryNameForHandleManagement = "Shared memory handle for shard split"; diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index e5d593295..898c276bf 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -7,12 +7,12 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - #include #include #include +#include "postgres.h" + /* necessary to get alloca on illumos */ #ifdef __sun #include @@ -20,92 +20,23 @@ #include "fmgr.h" #include "miscadmin.h" - #include "safe_lib.h" -#include "catalog/pg_authid.h" #include "catalog/objectaccess.h" +#include "catalog/pg_authid.h" #include "catalog/pg_extension.h" -#include "citus_version.h" #include "commands/explain.h" #include "commands/extension.h" #include "common/string.h" #include "executor/executor.h" -#include "distributed/backend_data.h" -#include "distributed/background_jobs.h" -#include "distributed/causal_clock.h" -#include "distributed/citus_depended_object.h" -#include "distributed/citus_nodefuncs.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/commands.h" -#include "distributed/commands/multi_copy.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/connection_management.h" -#include "distributed/cte_inline.h" -#include "distributed/distributed_deadlock_detection.h" -#include "distributed/errormessage.h" -#include "distributed/repartition_executor.h" -#include "distributed/intermediate_result_pruning.h" -#include "distributed/local_multi_copy.h" -#include "distributed/local_executor.h" -#include "distributed/local_distributed_join_planner.h" -#include "distributed/locally_reserved_shared_connections.h" -#include "distributed/log_utils.h" -#include "distributed/maintenanced.h" -#include "distributed/shard_cleaner.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_explain.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_logical_replication.h" -#include "distributed/multi_logical_optimizer.h" -#include "distributed/distributed_planner.h" -#include "distributed/combine_query_planner.h" -#include "distributed/multi_router_planner.h" -#include "distributed/multi_server_executor.h" -#include "distributed/pg_dist_partition.h" -#include "distributed/placement_connection.h" -#include "distributed/priority.h" -#include "distributed/query_stats.h" -#include "distributed/recursive_planning.h" -#include "distributed/reference_table_utils.h" -#include "distributed/relation_access_tracking.h" -#include "distributed/replication_origin_session_utils.h" -#include "distributed/run_from_same_connection.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_transfer.h" -#include "distributed/shared_connection_stats.h" -#include "distributed/shardsplit_shared_memory.h" -#include "distributed/query_pushdown_planning.h" -#include "distributed/time_constants.h" -#include "distributed/query_stats.h" -#include "distributed/remote_commands.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/shared_library_init.h" -#include "distributed/statistics_collection.h" -#include "distributed/subplan_execution.h" -#include "distributed/resource_lock.h" -#include "distributed/transaction_management.h" -#include "distributed/transaction_recovery.h" -#include "distributed/utils/citus_stat_tenants.h" -#include "distributed/utils/directory.h" -#include "distributed/worker_log_messages.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_shard_visibility.h" -#include "distributed/adaptive_executor.h" #include "libpq/auth.h" +#include "optimizer/paths.h" +#include "optimizer/plancat.h" +#include "optimizer/planner.h" #include "port/atomics.h" #include "postmaster/postmaster.h" #include "replication/walsender.h" #include "storage/ipc.h" -#include "optimizer/planner.h" -#include "optimizer/plancat.h" -#include "optimizer/paths.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/guc_tables.h" @@ -114,8 +45,76 @@ #include "utils/syscache.h" #include "utils/varlena.h" +#include "citus_version.h" + #include "columnar/columnar.h" +#include "distributed/adaptive_executor.h" +#include "distributed/backend_data.h" +#include "distributed/background_jobs.h" +#include "distributed/causal_clock.h" +#include "distributed/citus_depended_object.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/combine_query_planner.h" +#include "distributed/commands.h" +#include "distributed/commands/multi_copy.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/connection_management.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/cte_inline.h" +#include "distributed/distributed_deadlock_detection.h" +#include "distributed/distributed_planner.h" +#include "distributed/errormessage.h" +#include "distributed/intermediate_result_pruning.h" +#include "distributed/local_distributed_join_planner.h" +#include "distributed/local_executor.h" +#include "distributed/local_multi_copy.h" +#include "distributed/locally_reserved_shared_connections.h" +#include "distributed/log_utils.h" +#include "distributed/maintenanced.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_explain.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_logical_replication.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/multi_router_planner.h" +#include "distributed/multi_server_executor.h" +#include "distributed/pg_dist_partition.h" +#include "distributed/placement_connection.h" +#include "distributed/priority.h" +#include "distributed/query_pushdown_planning.h" +#include "distributed/query_stats.h" +#include "distributed/recursive_planning.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relation_access_tracking.h" +#include "distributed/remote_commands.h" +#include "distributed/repartition_executor.h" +#include "distributed/replication_origin_session_utils.h" +#include "distributed/resource_lock.h" +#include "distributed/run_from_same_connection.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" +#include "distributed/shard_transfer.h" +#include "distributed/shardsplit_shared_memory.h" +#include "distributed/shared_connection_stats.h" +#include "distributed/shared_library_init.h" +#include "distributed/statistics_collection.h" +#include "distributed/subplan_execution.h" +#include "distributed/time_constants.h" +#include "distributed/transaction_management.h" +#include "distributed/transaction_recovery.h" +#include "distributed/utils/citus_stat_tenants.h" +#include "distributed/utils/directory.h" +#include "distributed/worker_log_messages.h" +#include "distributed/worker_manager.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_shard_visibility.h" + /* marks shared object as one loadable by the postgres version compiled against */ PG_MODULE_MAGIC; diff --git a/src/backend/distributed/test/backend_counter.c b/src/backend/distributed/test/backend_counter.c index 1b9984ac9..f3f19f0d3 100644 --- a/src/backend/distributed/test/backend_counter.c +++ b/src/backend/distributed/test/backend_counter.c @@ -11,6 +11,7 @@ */ #include "postgres.h" + #include "fmgr.h" #include "distributed/backend_data.h" diff --git a/src/backend/distributed/test/citus_depended_object.c b/src/backend/distributed/test/citus_depended_object.c index 4e1e919e8..77fc2e482 100644 --- a/src/backend/distributed/test/citus_depended_object.c +++ b/src/backend/distributed/test/citus_depended_object.c @@ -13,17 +13,17 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" -#include "catalog/pg_attribute.h" #include "catalog/pg_attrdef.h" -#include "catalog/pg_constraint.h" +#include "catalog/pg_attribute.h" #include "catalog/pg_class.h" +#include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" -#include "catalog/pg_operator.h" #include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_rewrite.h" @@ -34,11 +34,12 @@ #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" + #include "distributed/citus_depended_object.h" #include "distributed/listutils.h" -#include "distributed/metadata_cache.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" +#include "distributed/metadata_cache.h" static bool IsCitusDependentObject(ObjectAddress objectAddress); diff --git a/src/backend/distributed/test/citus_stat_tenants.c b/src/backend/distributed/test/citus_stat_tenants.c index 2cfe0029b..b8fe305c6 100644 --- a/src/backend/distributed/test/citus_stat_tenants.c +++ b/src/backend/distributed/test/citus_stat_tenants.c @@ -10,11 +10,13 @@ */ #include "postgres.h" + #include "fmgr.h" -#include "distributed/utils/citus_stat_tenants.h" #include "sys/time.h" +#include "distributed/utils/citus_stat_tenants.h" + PG_FUNCTION_INFO_V1(sleep_until_next_period); /* diff --git a/src/backend/distributed/test/colocation_utils.c b/src/backend/distributed/test/colocation_utils.c index 19a4e1664..6a87539c4 100644 --- a/src/backend/distributed/test/colocation_utils.c +++ b/src/backend/distributed/test/colocation_utils.c @@ -11,9 +11,11 @@ */ #include "postgres.h" + #include "fmgr.h" #include "catalog/pg_type.h" + #include "distributed/colocation_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" diff --git a/src/backend/distributed/test/create_shards.c b/src/backend/distributed/test/create_shards.c index 4ed1db7c7..4ef13f1cb 100644 --- a/src/backend/distributed/test/create_shards.c +++ b/src/backend/distributed/test/create_shards.c @@ -10,16 +10,18 @@ *------------------------------------------------------------------------- */ +#include + #include "postgres.h" + #include "c.h" #include "fmgr.h" -#include - -#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" +#include "distributed/listutils.h" + /* local function forward declarations */ static int CompareStrings(const void *leftElement, const void *rightElement); diff --git a/src/backend/distributed/test/deparse_function_query.c b/src/backend/distributed/test/deparse_function_query.c index 7a6e54424..8971f597a 100644 --- a/src/backend/distributed/test/deparse_function_query.c +++ b/src/backend/distributed/test/deparse_function_query.c @@ -13,9 +13,10 @@ #include "postgres.h" +#include "utils/builtins.h" + #include "distributed/deparser.h" #include "distributed/multi_executor.h" -#include "utils/builtins.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(deparse_test); diff --git a/src/backend/distributed/test/deparse_shard_query.c b/src/backend/distributed/test/deparse_shard_query.c index a6196146f..a9b4ced1d 100644 --- a/src/backend/distributed/test/deparse_shard_query.c +++ b/src/backend/distributed/test/deparse_shard_query.c @@ -10,18 +10,14 @@ *------------------------------------------------------------------------- */ +#include + #include "postgres.h" + #include "c.h" #include "fmgr.h" -#include - #include "catalog/pg_type.h" -#include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/insert_select_planner.h" -#include "distributed/multi_router_planner.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" @@ -33,6 +29,12 @@ #include "utils/builtins.h" #include "utils/palloc.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/insert_select_planner.h" +#include "distributed/listutils.h" +#include "distributed/multi_router_planner.h" + /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(deparse_shard_query_test); diff --git a/src/backend/distributed/test/dependency.c b/src/backend/distributed/test/dependency.c index 82e818b8c..7afbfdec7 100644 --- a/src/backend/distributed/test/dependency.c +++ b/src/backend/distributed/test/dependency.c @@ -9,6 +9,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/test/distributed_deadlock_detection.c b/src/backend/distributed/test/distributed_deadlock_detection.c index d3fa34db2..68b5622a7 100644 --- a/src/backend/distributed/test/distributed_deadlock_detection.c +++ b/src/backend/distributed/test/distributed_deadlock_detection.c @@ -10,10 +10,15 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" #include "access/hash.h" +#include "nodes/pg_list.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" + #include "distributed/backend_data.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/hash_helpers.h" @@ -22,9 +27,6 @@ #include "distributed/metadata_cache.h" #include "distributed/transaction_identifier.h" #include "distributed/tuplestore.h" -#include "nodes/pg_list.h" -#include "utils/hsearch.h" -#include "utils/timestamp.h" PG_FUNCTION_INFO_V1(get_adjacency_list_wait_graph); diff --git a/src/backend/distributed/test/distributed_intermediate_results.c b/src/backend/distributed/test/distributed_intermediate_results.c index c3b286f52..843bda476 100644 --- a/src/backend/distributed/test/distributed_intermediate_results.c +++ b/src/backend/distributed/test/distributed_intermediate_results.c @@ -13,12 +13,15 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "catalog/pg_type.h" +#include "tcop/tcopprot.h" + #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/intermediate_results.h" @@ -26,10 +29,8 @@ #include "distributed/multi_executor.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" -#include "distributed/listutils.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" -#include "tcop/tcopprot.h" PG_FUNCTION_INFO_V1(partition_task_list_results); PG_FUNCTION_INFO_V1(redistribute_task_list_results); diff --git a/src/backend/distributed/test/distribution_metadata.c b/src/backend/distributed/test/distribution_metadata.c index c3bc7fb51..01117922e 100644 --- a/src/backend/distributed/test/distribution_metadata.c +++ b/src/backend/distributed/test/distribution_metadata.c @@ -10,37 +10,39 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "c.h" -#include "fmgr.h" - #include #include +#include "postgres.h" + +#include "c.h" +#include "fmgr.h" + #include "access/heapam.h" #include "catalog/pg_type.h" -#include "distributed/distribution_column.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/query_utils.h" -#include "distributed/resource_lock.h" -#include "distributed/utils/array_type.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "storage/lock.h" #include "tcop/tcopprot.h" #include "utils/array.h" +#include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" -#include "utils/builtins.h" #include "utils/palloc.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distribution_column.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/pg_dist_shard.h" +#include "distributed/query_utils.h" +#include "distributed/resource_lock.h" +#include "distributed/utils/array_type.h" + /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(load_shard_id_array); diff --git a/src/backend/distributed/test/fake_am.c b/src/backend/distributed/test/fake_am.c index 4b11d7871..cff124961 100644 --- a/src/backend/distributed/test/fake_am.c +++ b/src/backend/distributed/test/fake_am.c @@ -19,14 +19,10 @@ #include "postgres.h" -#include "pg_version_constants.h" -#include "pg_version_compat.h" - - #include "access/amapi.h" #include "access/heapam.h" -#include "access/tableam.h" #include "access/multixact.h" +#include "access/tableam.h" #include "access/xact.h" #include "catalog/index.h" #include "catalog/storage.h" @@ -36,6 +32,9 @@ #include "storage/smgr.h" #include "utils/snapmgr.h" +#include "pg_version_compat.h" +#include "pg_version_constants.h" + PG_FUNCTION_INFO_V1(fake_am_handler); static const TableAmRoutine fake_methods; diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index f53242f7f..585e61d41 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -10,27 +10,27 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" +#include -#include "pg_version_constants.h" +#include "postgres.h" #include "c.h" #include "fmgr.h" -#include - #include "executor/tuptable.h" #include "foreign/fdwapi.h" #include "nodes/execnodes.h" #include "nodes/nodes.h" +#include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/plannodes.h" -#include "nodes/pathnodes.h" #include "optimizer/pathnode.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" #include "utils/palloc.h" +#include "pg_version_constants.h" + /* local function forward declarations */ static void FakeGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); diff --git a/src/backend/distributed/test/foreign_key_relationship_query.c b/src/backend/distributed/test/foreign_key_relationship_query.c index 545c2e970..af187111a 100644 --- a/src/backend/distributed/test/foreign_key_relationship_query.c +++ b/src/backend/distributed/test/foreign_key_relationship_query.c @@ -11,18 +11,20 @@ */ #include "postgres.h" + #include "fmgr.h" #include "funcapi.h" #include "catalog/dependency.h" #include "catalog/pg_constraint.h" -#include "distributed/foreign_key_relationship.h" +#include "utils/builtins.h" + #include "distributed/coordinator_protocol.h" +#include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/tuplestore.h" #include "distributed/version_compat.h" -#include "utils/builtins.h" #define GET_FKEY_CONNECTED_RELATIONS_COLUMNS 1 diff --git a/src/backend/distributed/test/global_pid.c b/src/backend/distributed/test/global_pid.c index de54f1929..b63b39b44 100644 --- a/src/backend/distributed/test/global_pid.c +++ b/src/backend/distributed/test/global_pid.c @@ -10,6 +10,7 @@ */ #include "postgres.h" + #include "fmgr.h" #include "distributed/backend_data.h" diff --git a/src/backend/distributed/test/hide_shards.c b/src/backend/distributed/test/hide_shards.c index 59e738c36..b1adf61b4 100644 --- a/src/backend/distributed/test/hide_shards.c +++ b/src/backend/distributed/test/hide_shards.c @@ -10,6 +10,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" diff --git a/src/backend/distributed/test/intermediate_results.c b/src/backend/distributed/test/intermediate_results.c index b4f14bca6..8681a6ca9 100644 --- a/src/backend/distributed/test/intermediate_results.c +++ b/src/backend/distributed/test/intermediate_results.c @@ -13,6 +13,7 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" diff --git a/src/backend/distributed/test/make_external_connection.c b/src/backend/distributed/test/make_external_connection.c index 424793dea..14be057ab 100644 --- a/src/backend/distributed/test/make_external_connection.c +++ b/src/backend/distributed/test/make_external_connection.c @@ -11,10 +11,17 @@ */ #include "postgres.h" -#include "miscadmin.h" + #include "libpq-fe.h" +#include "miscadmin.h" #include "access/xact.h" +#include "executor/spi.h" +#include "lib/stringinfo.h" +#include "postmaster/postmaster.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_utils.h" @@ -23,13 +30,7 @@ #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" - #include "distributed/version_compat.h" -#include "executor/spi.h" -#include "lib/stringinfo.h" -#include "postmaster/postmaster.h" -#include "utils/builtins.h" -#include "utils/memutils.h" PG_FUNCTION_INFO_V1(make_external_connection_to_node); diff --git a/src/backend/distributed/test/metadata_sync.c b/src/backend/distributed/test/metadata_sync.c index 46d2303d6..4a5779b63 100644 --- a/src/backend/distributed/test/metadata_sync.c +++ b/src/backend/distributed/test/metadata_sync.c @@ -10,10 +10,17 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "c.h" #include "fmgr.h" +#include "miscadmin.h" #include "catalog/pg_type.h" +#include "postmaster/postmaster.h" +#include "storage/latch.h" +#include "utils/array.h" +#include "utils/builtins.h" + #include "distributed/connection_management.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" @@ -22,11 +29,6 @@ #include "distributed/remote_commands.h" #include "distributed/utils/array_type.h" #include "distributed/worker_manager.h" -#include "postmaster/postmaster.h" -#include "miscadmin.h" -#include "storage/latch.h" -#include "utils/array.h" -#include "utils/builtins.h" /* declarations for dynamic loading */ diff --git a/src/backend/distributed/test/partitioning_utils.c b/src/backend/distributed/test/partitioning_utils.c index 95adaddf6..be9163561 100644 --- a/src/backend/distributed/test/partitioning_utils.c +++ b/src/backend/distributed/test/partitioning_utils.c @@ -10,16 +10,18 @@ *------------------------------------------------------------------------- */ #include "postgres.h" + #include "fmgr.h" #include "catalog/pg_type.h" -#include "distributed/listutils.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/reference_table_utils.h" #include "lib/stringinfo.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "distributed/listutils.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/reference_table_utils.h" + PG_FUNCTION_INFO_V1(generate_alter_table_detach_partition_command); PG_FUNCTION_INFO_V1(generate_alter_table_attach_partition_command); diff --git a/src/backend/distributed/test/progress_utils.c b/src/backend/distributed/test/progress_utils.c index 42b065dae..e1ea09e3d 100644 --- a/src/backend/distributed/test/progress_utils.c +++ b/src/backend/distributed/test/progress_utils.c @@ -11,18 +11,20 @@ */ +#include + #include "postgres.h" -#include "miscadmin.h" + #include "fmgr.h" #include "funcapi.h" +#include "miscadmin.h" -#include +#include "nodes/execnodes.h" +#include "utils/tuplestore.h" #include "distributed/listutils.h" #include "distributed/multi_progress.h" #include "distributed/tuplestore.h" -#include "nodes/execnodes.h" -#include "utils/tuplestore.h" PG_FUNCTION_INFO_V1(create_progress); diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index 023a759cb..f972281ec 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -10,25 +10,15 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" +#include -#include "pg_version_constants.h" +#include "postgres.h" #include "c.h" #include "fmgr.h" -#include - #include "access/stratnum.h" #include "catalog/pg_type.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_utility.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/resource_lock.h" -#include "distributed/shard_pruning.h" -#include "distributed/utils/array_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" @@ -38,6 +28,17 @@ #include "utils/array.h" #include "utils/palloc.h" +#include "pg_version_constants.h" + +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/resource_lock.h" +#include "distributed/shard_pruning.h" +#include "distributed/utils/array_type.h" + /* local function forward declarations */ static Expr * MakeTextPartitionExpression(Oid distributedTableId, text *value); diff --git a/src/backend/distributed/test/relation_access_tracking.c b/src/backend/distributed/test/relation_access_tracking.c index 5715bd03d..85c0ff2aa 100644 --- a/src/backend/distributed/test/relation_access_tracking.c +++ b/src/backend/distributed/test/relation_access_tracking.c @@ -10,6 +10,7 @@ */ #include "postgres.h" + #include "c.h" #include "fmgr.h" diff --git a/src/backend/distributed/test/run_from_same_connection.c b/src/backend/distributed/test/run_from_same_connection.c index 04a3149da..5663a42fa 100644 --- a/src/backend/distributed/test/run_from_same_connection.c +++ b/src/backend/distributed/test/run_from_same_connection.c @@ -12,10 +12,17 @@ */ #include "postgres.h" -#include "miscadmin.h" + #include "libpq-fe.h" +#include "miscadmin.h" #include "access/xact.h" +#include "executor/spi.h" +#include "lib/stringinfo.h" +#include "postmaster/postmaster.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_utils.h" @@ -24,13 +31,7 @@ #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" - #include "distributed/version_compat.h" -#include "executor/spi.h" -#include "lib/stringinfo.h" -#include "postmaster/postmaster.h" -#include "utils/builtins.h" -#include "utils/memutils.h" #define ALTER_CURRENT_PROCESS_ID \ diff --git a/src/backend/distributed/test/sequential_execution.c b/src/backend/distributed/test/sequential_execution.c index 9b88e3b7a..f967eb75f 100644 --- a/src/backend/distributed/test/sequential_execution.c +++ b/src/backend/distributed/test/sequential_execution.c @@ -11,6 +11,7 @@ */ #include "postgres.h" + #include "fmgr.h" #include "distributed/multi_executor.h" diff --git a/src/backend/distributed/test/shard_rebalancer.c b/src/backend/distributed/test/shard_rebalancer.c index 56a063982..32bfd9f46 100644 --- a/src/backend/distributed/test/shard_rebalancer.c +++ b/src/backend/distributed/test/shard_rebalancer.c @@ -11,27 +11,28 @@ */ #include "postgres.h" -#include "libpq-fe.h" +#include "funcapi.h" +#include "libpq-fe.h" +#include "miscadmin.h" #include "safe_lib.h" #include "catalog/pg_type.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/connection_management.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/shard_cleaner.h" -#include "distributed/shard_rebalancer.h" -#include "distributed/relay_utility.h" -#include "funcapi.h" -#include "miscadmin.h" #include "utils/builtins.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/connection_management.h" +#include "distributed/listutils.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_physical_planner.h" +#include "distributed/relay_utility.h" +#include "distributed/shard_cleaner.h" +#include "distributed/shard_rebalancer.h" + /* static declarations for json conversion */ static List * JsonArrayToShardPlacementTestInfoList( ArrayType *shardPlacementJsonArrayObject); diff --git a/src/backend/distributed/test/shared_connection_counters.c b/src/backend/distributed/test/shared_connection_counters.c index 641cfd314..c59602887 100644 --- a/src/backend/distributed/test/shared_connection_counters.c +++ b/src/backend/distributed/test/shared_connection_counters.c @@ -11,14 +11,16 @@ */ #include "postgres.h" -#include "miscadmin.h" -#include "fmgr.h" -#include "distributed/shared_connection_stats.h" -#include "distributed/listutils.h" +#include "fmgr.h" +#include "miscadmin.h" + #include "nodes/parsenodes.h" #include "utils/guc.h" +#include "distributed/listutils.h" +#include "distributed/shared_connection_stats.h" + /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(wake_up_connection_pool_waiters); PG_FUNCTION_INFO_V1(set_max_shared_pool_size); diff --git a/src/backend/distributed/test/xact_stats.c b/src/backend/distributed/test/xact_stats.c index 87e15aa64..a968f8cb6 100644 --- a/src/backend/distributed/test/xact_stats.c +++ b/src/backend/distributed/test/xact_stats.c @@ -13,6 +13,7 @@ #include #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index c1981b77a..5f868f548 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -12,18 +12,29 @@ #include "postgres.h" -#include "pg_version_constants.h" - +#include "funcapi.h" #include "miscadmin.h" +#include "safe_lib.h" #include "unistd.h" -#include "safe_lib.h" - -#include "funcapi.h" #include "access/htup_details.h" #include "catalog/pg_authid.h" #include "catalog/pg_type.h" #include "datatype/timestamp.h" +#include "nodes/execnodes.h" +#include "postmaster/autovacuum.h" /* to access autovacuum_max_workers */ +#include "replication/walsender.h" +#include "storage/ipc.h" +#include "storage/lmgr.h" +#include "storage/lwlock.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/s_lock.h" +#include "storage/spin.h" +#include "utils/timestamp.h" + +#include "pg_version_constants.h" + #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" @@ -34,17 +45,6 @@ #include "distributed/transaction_identifier.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" -#include "nodes/execnodes.h" -#include "postmaster/autovacuum.h" /* to access autovacuum_max_workers */ -#include "replication/walsender.h" -#include "storage/ipc.h" -#include "storage/lmgr.h" -#include "storage/lwlock.h" -#include "storage/procarray.h" -#include "storage/proc.h" -#include "storage/spin.h" -#include "storage/s_lock.h" -#include "utils/timestamp.h" #define GET_ACTIVE_TRANSACTION_QUERY "SELECT * FROM get_all_active_transactions();" diff --git a/src/backend/distributed/transaction/citus_dist_stat_activity.c b/src/backend/distributed/transaction/citus_dist_stat_activity.c index 3aa6372e6..b0ffc05eb 100644 --- a/src/backend/distributed/transaction/citus_dist_stat_activity.c +++ b/src/backend/distributed/transaction/citus_dist_stat_activity.c @@ -10,8 +10,9 @@ */ #include "postgres.h" -#include "miscadmin.h" + #include "funcapi.h" +#include "miscadmin.h" PG_FUNCTION_INFO_V1(citus_dist_stat_activity); PG_FUNCTION_INFO_V1(citus_worker_stat_activity); diff --git a/src/backend/distributed/transaction/distributed_deadlock_detection.c b/src/backend/distributed/transaction/distributed_deadlock_detection.c index cf8dd43f5..27bb48ee3 100644 --- a/src/backend/distributed/transaction/distributed_deadlock_detection.c +++ b/src/backend/distributed/transaction/distributed_deadlock_detection.c @@ -15,8 +15,11 @@ #include "pgstat.h" #include "access/hash.h" +#include "nodes/pg_list.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" + #include "distributed/backend_data.h" -#include "distributed/errormessage.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/errormessage.h" #include "distributed/hash_helpers.h" @@ -25,9 +28,6 @@ #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/transaction_identifier.h" -#include "nodes/pg_list.h" -#include "utils/hsearch.h" -#include "utils/timestamp.h" /* used only for finding the deadlock cycle path */ diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index 0b4c0f02e..82f936243 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -18,6 +18,11 @@ #include "miscadmin.h" #include "access/hash.h" +#include "storage/proc.h" +#include "utils/builtins.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" + #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" @@ -26,10 +31,6 @@ #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" -#include "storage/proc.h" -#include "utils/builtins.h" -#include "utils/hsearch.h" -#include "utils/timestamp.h" /* diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index 3ad61ac79..5044941c4 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -15,22 +15,23 @@ */ #include "postgres.h" -#include "pg_version_constants.h" - #include "miscadmin.h" #include "access/xact.h" +#include "common/hashfn.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" + +#include "pg_version_constants.h" + #include "distributed/colocation_utils.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" +#include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" -#include "distributed/metadata_cache.h" #include "distributed/relation_access_tracking.h" -#include "utils/hsearch.h" -#include "common/hashfn.h" -#include "utils/lsyscache.h" /* Config variables managed via guc.c */ diff --git a/src/backend/distributed/transaction/remote_transaction.c b/src/backend/distributed/transaction/remote_transaction.c index 0f6241793..3dc89c995 100644 --- a/src/backend/distributed/transaction/remote_transaction.c +++ b/src/backend/distributed/transaction/remote_transaction.c @@ -16,10 +16,12 @@ #include "postgres.h" #include "libpq-fe.h" - #include "miscadmin.h" #include "access/xact.h" +#include "utils/builtins.h" +#include "utils/hsearch.h" + #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" @@ -32,8 +34,6 @@ #include "distributed/transaction_management.h" #include "distributed/transaction_recovery.h" #include "distributed/worker_manager.h" -#include "utils/builtins.h" -#include "utils/hsearch.h" #define PREPARED_TRANSACTION_NAME_FORMAT "citus_%u_%u_"UINT64_FORMAT "_%u" diff --git a/src/backend/distributed/transaction/transaction_management.c b/src/backend/distributed/transaction/transaction_management.c index 9a7bd9089..d133d7be6 100644 --- a/src/backend/distributed/transaction/transaction_management.c +++ b/src/backend/distributed/transaction/transaction_management.c @@ -14,15 +14,22 @@ #include "postgres.h" #include "libpq-fe.h" - #include "miscadmin.h" #include "access/twophase.h" #include "access/xact.h" #include "catalog/dependency.h" #include "common/hashfn.h" +#include "nodes/print.h" +#include "storage/fd.h" +#include "utils/datum.h" +#include "utils/guc.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" + #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" +#include "distributed/commands.h" #include "distributed/connection_management.h" #include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" @@ -33,27 +40,20 @@ #include "distributed/locally_reserved_shared_connections.h" #include "distributed/maintenanced.h" #include "distributed/metadata/dependency.h" +#include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" -#include "distributed/multi_logical_replication.h" #include "distributed/multi_explain.h" -#include "distributed/repartition_join_execution.h" -#include "distributed/replication_origin_session_utils.h" -#include "distributed/transaction_management.h" +#include "distributed/multi_logical_replication.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" -#include "distributed/shared_connection_stats.h" +#include "distributed/repartition_join_execution.h" +#include "distributed/replication_origin_session_utils.h" #include "distributed/shard_cleaner.h" +#include "distributed/shared_connection_stats.h" #include "distributed/subplan_execution.h" +#include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_log_messages.h" -#include "distributed/commands.h" -#include "distributed/metadata_cache.h" -#include "utils/hsearch.h" -#include "utils/guc.h" -#include "utils/memutils.h" -#include "utils/datum.h" -#include "storage/fd.h" -#include "nodes/print.h" CoordinatedTransactionState CurrentCoordinatedTransactionState = COORD_TRANS_NONE; diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index a833f5a46..0ec5ba0a3 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -12,22 +12,30 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - -#include "miscadmin.h" -#include "libpq-fe.h" - #include #include +#include "postgres.h" + +#include "libpq-fe.h" +#include "miscadmin.h" + #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/relscan.h" #include "access/xact.h" #include "catalog/indexing.h" +#include "lib/stringinfo.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +#include "pg_version_constants.h" + #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" @@ -36,15 +44,8 @@ #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_recovery.h" -#include "distributed/worker_manager.h" #include "distributed/version_compat.h" -#include "lib/stringinfo.h" -#include "storage/lmgr.h" -#include "storage/lock.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/memutils.h" -#include "utils/rel.h" +#include "distributed/worker_manager.h" /* exports for SQL callable functions */ diff --git a/src/backend/distributed/transaction/worker_transaction.c b/src/backend/distributed/transaction/worker_transaction.c index 03ecbea72..f008c2974 100644 --- a/src/backend/distributed/transaction/worker_transaction.c +++ b/src/backend/distributed/transaction/worker_transaction.c @@ -11,28 +11,30 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "miscadmin.h" -#include "libpq-fe.h" - #include #include +#include "postgres.h" + +#include "libpq-fe.h" +#include "miscadmin.h" + #include "access/xact.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + #include "distributed/connection_management.h" +#include "distributed/jsonbutils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" -#include "distributed/resource_lock.h" #include "distributed/metadata_sync.h" -#include "distributed/remote_commands.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_transaction.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" #include "distributed/transaction_recovery.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "distributed/jsonbutils.h" -#include "utils/memutils.h" -#include "utils/builtins.h" static void SendCommandToMetadataWorkersParams(const char *command, const char *user, int parameterCount, diff --git a/src/backend/distributed/utils/acquire_lock.c b/src/backend/distributed/utils/acquire_lock.c index f414167b3..d0f6193c2 100644 --- a/src/backend/distributed/utils/acquire_lock.c +++ b/src/backend/distributed/utils/acquire_lock.c @@ -22,12 +22,12 @@ #include "postgres.h" +#include "miscadmin.h" +#include "pgstat.h" #include "access/xact.h" #include "catalog/pg_type.h" #include "executor/spi.h" -#include "miscadmin.h" -#include "pgstat.h" #include "portability/instr_time.h" #include "storage/ipc.h" #include "storage/latch.h" diff --git a/src/backend/distributed/utils/aggregate_utils.c b/src/backend/distributed/utils/aggregate_utils.c index 773e0aa25..3fd584df9 100644 --- a/src/backend/distributed/utils/aggregate_utils.c +++ b/src/backend/distributed/utils/aggregate_utils.c @@ -16,11 +16,14 @@ #include "postgres.h" +#include "fmgr.h" +#include "miscadmin.h" +#include "pg_config_manual.h" + #include "access/htup_details.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" -#include "distributed/version_compat.h" #include "nodes/nodeFuncs.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -28,9 +31,8 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/typcache.h" -#include "fmgr.h" -#include "miscadmin.h" -#include "pg_config_manual.h" + +#include "distributed/version_compat.h" PG_FUNCTION_INFO_V1(worker_partial_agg_sfunc); PG_FUNCTION_INFO_V1(worker_partial_agg_ffunc); diff --git a/src/backend/distributed/utils/array_type.c b/src/backend/distributed/utils/array_type.c index 70c7dde14..1c3663d43 100644 --- a/src/backend/distributed/utils/array_type.c +++ b/src/backend/distributed/utils/array_type.c @@ -10,16 +10,19 @@ */ #include "postgres.h" + #include "miscadmin.h" -#include "pg_version_compat.h" #include "catalog/pg_type.h" #include "nodes/pg_list.h" -#include "distributed/utils/array_type.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "pg_version_compat.h" + +#include "distributed/utils/array_type.h" + /* * DeconstructArrayObject takes in a single dimensional array, and deserializes diff --git a/src/backend/distributed/utils/background_jobs.c b/src/backend/distributed/utils/background_jobs.c index 2b5ce2dca..a7a124c74 100644 --- a/src/backend/distributed/utils/background_jobs.c +++ b/src/backend/distributed/utils/background_jobs.c @@ -27,17 +27,17 @@ #include "postgres.h" +#include "libpq-fe.h" +#include "pgstat.h" #include "safe_mem_lib.h" #include "access/xact.h" #include "commands/dbcommands.h" #include "common/hashfn.h" -#include "libpq-fe.h" #include "libpq/pqformat.h" #include "libpq/pqmq.h" #include "libpq/pqsignal.h" #include "parser/analyze.h" -#include "pgstat.h" #include "storage/dsm.h" #include "storage/ipc.h" #include "storage/procarray.h" @@ -62,9 +62,9 @@ #include "distributed/maintenanced.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" +#include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" -#include "distributed/resource_lock.h" /* Table-of-contents constants for our dynamic shared memory segment. */ #define CITUS_BACKGROUND_TASK_MAGIC 0x51028081 diff --git a/src/backend/distributed/utils/cancel_utils.c b/src/backend/distributed/utils/cancel_utils.c index 17383c034..f135212e4 100644 --- a/src/backend/distributed/utils/cancel_utils.c +++ b/src/backend/distributed/utils/cancel_utils.c @@ -8,7 +8,9 @@ #include "postgres.h" + #include "miscadmin.h" + #include "distributed/cancel_utils.h" diff --git a/src/backend/distributed/utils/citus_clauses.c b/src/backend/distributed/utils/citus_clauses.c index 82900ea1a..f88b173af 100644 --- a/src/backend/distributed/utils/citus_clauses.c +++ b/src/backend/distributed/utils/citus_clauses.c @@ -8,12 +8,6 @@ #include "postgres.h" -#include "distributed/citus_clauses.h" -#include "distributed/insert_select_planner.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_router_planner.h" -#include "distributed/version_compat.h" - #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "executor/executor.h" @@ -28,6 +22,12 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/citus_clauses.h" +#include "distributed/insert_select_planner.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_router_planner.h" +#include "distributed/version_compat.h" + /* private function declarations */ static bool IsVariableExpression(Node *node); diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index 7e1379ef3..abbb62839 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -11,11 +11,11 @@ */ #include "postgres.h" +#include "utils/datum.h" #include "distributed/citus_nodefuncs.h" -#include "distributed/multi_server_executor.h" #include "distributed/listutils.h" -#include "utils/datum.h" +#include "distributed/multi_server_executor.h" /* diff --git a/src/backend/distributed/utils/citus_depended_object.c b/src/backend/distributed/utils/citus_depended_object.c index 3b5a34b54..a160fcd56 100644 --- a/src/backend/distributed/utils/citus_depended_object.c +++ b/src/backend/distributed/utils/citus_depended_object.c @@ -7,6 +7,7 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/namespace.h" @@ -14,17 +15,17 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" -#include "catalog/pg_attribute.h" #include "catalog/pg_attrdef.h" -#include "catalog/pg_constraint.h" +#include "catalog/pg_attribute.h" #include "catalog/pg_class.h" +#include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" -#include "catalog/pg_operator.h" #include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_rewrite.h" @@ -35,12 +36,6 @@ #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" -#include "distributed/citus_depended_object.h" -#include "distributed/metadata_cache.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/log_utils.h" -#include "distributed/shared_library_init.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" @@ -49,6 +44,13 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "distributed/citus_depended_object.h" +#include "distributed/commands.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/shared_library_init.h" + /* * GUC hides any objects, which depends on citus extension, from pg meta class queries, * it is intended to be used in vanilla tests to not break postgres test logs diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index e74f598d8..0b03926f8 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -10,16 +10,17 @@ #include "postgres.h" +#include "catalog/pg_type.h" + #include "pg_version_constants.h" -#include "catalog/pg_type.h" -#include "distributed/citus_nodes.h" #include "distributed/citus_nodefuncs.h" +#include "distributed/citus_nodes.h" #include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" -#include "distributed/distributed_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" diff --git a/src/backend/distributed/utils/citus_safe_lib.c b/src/backend/distributed/utils/citus_safe_lib.c index cbd06fc50..2d504a644 100644 --- a/src/backend/distributed/utils/citus_safe_lib.c +++ b/src/backend/distributed/utils/citus_safe_lib.c @@ -12,16 +12,17 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" +#include -#include "pg_version_constants.h" +#include "postgres.h" #include "safe_lib.h" -#include +#include "lib/stringinfo.h" + +#include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" -#include "lib/stringinfo.h" /* diff --git a/src/backend/distributed/utils/citus_stat_tenants.c b/src/backend/distributed/utils/citus_stat_tenants.c index aa813e152..6af5c0d58 100644 --- a/src/backend/distributed/utils/citus_stat_tenants.c +++ b/src/backend/distributed/utils/citus_stat_tenants.c @@ -8,21 +8,13 @@ *------------------------------------------------------------------------- */ +#include + #include "postgres.h" + #include "unistd.h" #include "access/hash.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/colocation_utils.h" -#include "distributed/distributed_planner.h" -#include "distributed/jsonbutils.h" -#include "distributed/log_utils.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_executor.h" -#include "distributed/tenant_schema_metadata.h" -#include "distributed/tuplestore.h" -#include "distributed/utils/citus_stat_tenants.h" #include "executor/execdesc.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -34,7 +26,17 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" -#include +#include "distributed/citus_safe_lib.h" +#include "distributed/colocation_utils.h" +#include "distributed/distributed_planner.h" +#include "distributed/jsonbutils.h" +#include "distributed/listutils.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_executor.h" +#include "distributed/tenant_schema_metadata.h" +#include "distributed/tuplestore.h" +#include "distributed/utils/citus_stat_tenants.h" #if (PG_VERSION_NUM >= PG_VERSION_15) #include "common/pg_prng.h" diff --git a/src/backend/distributed/utils/citus_version.c b/src/backend/distributed/utils/citus_version.c index 95945a30f..edae4f927 100644 --- a/src/backend/distributed/utils/citus_version.c +++ b/src/backend/distributed/utils/citus_version.c @@ -11,9 +11,10 @@ #include "postgres.h" -#include "citus_version.h" #include "utils/builtins.h" +#include "citus_version.h" + /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_version); diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index e7007874b..b8db3c80f 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -10,6 +10,7 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "access/genam.h" @@ -19,28 +20,29 @@ #include "catalog/indexing.h" #include "catalog/pg_type.h" #include "commands/sequence.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" + #include "distributed/colocation_utils.h" #include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" #include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_colocation.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/tenant_schema_metadata.h" -#include "distributed/version_compat.h" #include "distributed/utils/array_type.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" /* local function forward declarations */ diff --git a/src/backend/distributed/utils/directory.c b/src/backend/distributed/utils/directory.c index b749b9cd6..bad585809 100644 --- a/src/backend/distributed/utils/directory.c +++ b/src/backend/distributed/utils/directory.c @@ -12,6 +12,7 @@ #include #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" diff --git a/src/backend/distributed/utils/distribution_column.c b/src/backend/distributed/utils/distribution_column.c index 474133f73..5927be612 100644 --- a/src/backend/distributed/utils/distribution_column.c +++ b/src/backend/distributed/utils/distribution_column.c @@ -12,19 +12,14 @@ #include "postgres.h" - #include "access/attnum.h" #include "access/heapam.h" #include "access/htup_details.h" -#include "distributed/distribution_column.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "nodes/primnodes.h" -#include "parser/scansup.h" #include "parser/parse_relation.h" +#include "parser/scansup.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" @@ -33,6 +28,11 @@ #include "utils/relcache.h" #include "utils/syscache.h" +#include "distributed/distribution_column.h" +#include "distributed/metadata_cache.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/version_compat.h" + /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(column_name_to_column); diff --git a/src/backend/distributed/utils/distribution_column_map.c b/src/backend/distributed/utils/distribution_column_map.c index c3c0db01f..43f9939b1 100644 --- a/src/backend/distributed/utils/distribution_column_map.c +++ b/src/backend/distributed/utils/distribution_column_map.c @@ -11,12 +11,13 @@ #include "postgres.h" #include "common/hashfn.h" +#include "nodes/primnodes.h" + #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/utils/distribution_column_map.h" -#include "nodes/primnodes.h" /* diff --git a/src/backend/distributed/utils/enable_ssl.c b/src/backend/distributed/utils/enable_ssl.c index 35b1e0f1a..261225450 100644 --- a/src/backend/distributed/utils/enable_ssl.c +++ b/src/backend/distributed/utils/enable_ssl.c @@ -18,16 +18,18 @@ * it otherwise we get warnings about redefining this value. This needs to be * done before including libpq.h. */ +#include "miscadmin.h" + +#include "libpq/libpq.h" +#include "nodes/parsenodes.h" +#include "postmaster/postmaster.h" +#include "utils/guc.h" + #include "pg_version_constants.h" #include "distributed/connection_management.h" #include "distributed/memutils.h" #include "distributed/worker_protocol.h" -#include "libpq/libpq.h" -#include "miscadmin.h" -#include "nodes/parsenodes.h" -#include "postmaster/postmaster.h" -#include "utils/guc.h" #ifdef USE_OPENSSL #include "openssl/dsa.h" diff --git a/src/backend/distributed/utils/errormessage.c b/src/backend/distributed/utils/errormessage.c index 72758f9ca..dbc55019d 100644 --- a/src/backend/distributed/utils/errormessage.c +++ b/src/backend/distributed/utils/errormessage.c @@ -6,13 +6,14 @@ */ #include "postgres.h" -#include "utils/memutils.h" #include "common/sha2.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" -#include "utils/builtins.h" /* diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index d69d9044d..1abb7ae07 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -12,29 +12,29 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "access/genam.h" #include "access/htup_details.h" #include "access/stratnum.h" #include "access/table.h" #include "catalog/pg_constraint.h" -#include "distributed/commands.h" -#include "distributed/hash_helpers.h" -#include "distributed/foreign_key_relationship.h" -#include "distributed/hash_helpers.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/version_compat.h" +#include "common/hashfn.h" #include "nodes/pg_list.h" #include "storage/lockdefs.h" #include "utils/catcache.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" -#include "common/hashfn.h" #include "utils/inval.h" #include "utils/memutils.h" +#include "pg_version_constants.h" + +#include "distributed/commands.h" +#include "distributed/foreign_key_relationship.h" +#include "distributed/hash_helpers.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/version_compat.h" + /* * ForeignConstraintRelationshipGraph holds the graph data structure for foreign constraint relationship diff --git a/src/backend/distributed/utils/function.c b/src/backend/distributed/utils/function.c index bfb59181c..dcfcff6fe 100644 --- a/src/backend/distributed/utils/function.c +++ b/src/backend/distributed/utils/function.c @@ -10,13 +10,15 @@ */ #include "postgres.h" + #include "fmgr.h" #include "miscadmin.h" #include "commands/defrem.h" -#include "distributed/utils/function.h" #include "utils/lsyscache.h" +#include "distributed/utils/function.h" + /* * GetFunctionInfo first resolves the operator for the given data type, access diff --git a/src/backend/distributed/utils/function_utils.c b/src/backend/distributed/utils/function_utils.c index 48f878e13..0770b8cb9 100644 --- a/src/backend/distributed/utils/function_utils.c +++ b/src/backend/distributed/utils/function_utils.c @@ -10,12 +10,13 @@ #include "postgres.h" #include "catalog/namespace.h" -#include "distributed/function_utils.h" -#include "distributed/version_compat.h" #include "executor/executor.h" #include "utils/builtins.h" #include "utils/regproc.h" +#include "distributed/function_utils.h" +#include "distributed/version_compat.h" + /* * FunctionOid searches for a function that has the given name and the given diff --git a/src/backend/distributed/utils/hash_helpers.c b/src/backend/distributed/utils/hash_helpers.c index d2bfe38fa..2aaaaef12 100644 --- a/src/backend/distributed/utils/hash_helpers.c +++ b/src/backend/distributed/utils/hash_helpers.c @@ -11,9 +11,10 @@ #include "postgres.h" #include "common/hashfn.h" +#include "utils/hsearch.h" + #include "distributed/citus_safe_lib.h" #include "distributed/hash_helpers.h" -#include "utils/hsearch.h" /* diff --git a/src/backend/distributed/utils/jsonbutils.c b/src/backend/distributed/utils/jsonbutils.c index 4855ee004..47e6aa2c7 100644 --- a/src/backend/distributed/utils/jsonbutils.c +++ b/src/backend/distributed/utils/jsonbutils.c @@ -1,21 +1,21 @@ #include "postgres.h" -#include "pg_version_compat.h" +#include "fmgr.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" - #include "utils/array.h" +#include "utils/builtins.h" #include "utils/json.h" +#include "utils/lsyscache.h" + +#include "pg_version_compat.h" + #include "distributed/jsonbutils.h" #include "distributed/metadata_cache.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "fmgr.h" - /* * ExtractFieldJsonb gets value of fieldName from jsonbDoc and puts it diff --git a/src/backend/distributed/utils/listutils.c b/src/backend/distributed/utils/listutils.c index dd54443c4..eddef1fea 100644 --- a/src/backend/distributed/utils/listutils.c +++ b/src/backend/distributed/utils/listutils.c @@ -10,15 +10,17 @@ */ #include "postgres.h" + #include "c.h" #include "port.h" -#include "utils/lsyscache.h" #include "lib/stringinfo.h" +#include "nodes/pg_list.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" + #include "distributed/citus_safe_lib.h" #include "distributed/listutils.h" -#include "nodes/pg_list.h" -#include "utils/memutils.h" /* diff --git a/src/backend/distributed/utils/log_utils.c b/src/backend/distributed/utils/log_utils.c index 7d808591b..2e9d94c44 100644 --- a/src/backend/distributed/utils/log_utils.c +++ b/src/backend/distributed/utils/log_utils.c @@ -9,16 +9,15 @@ #include "postgres.h" +#include "common/cryptohash.h" +#include "common/sha2.h" +#include "utils/builtins.h" +#include "utils/guc.h" + #include "pg_version_constants.h" -#include "utils/guc.h" -#include "distributed/log_utils.h" #include "distributed/errormessage.h" -#include "common/sha2.h" - -#include "utils/builtins.h" - -#include "common/cryptohash.h" +#include "distributed/log_utils.h" /* diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index db5365a81..49f2266c7 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -14,52 +14,52 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include "pg_version_constants.h" - #include +#include "postgres.h" + #include "miscadmin.h" #include "pgstat.h" #include "access/xact.h" #include "access/xlog.h" -#include "catalog/pg_extension.h" -#include "citus_version.h" +#include "catalog/namespace.h" #include "catalog/pg_authid.h" +#include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "commands/async.h" #include "commands/extension.h" +#include "common/hashfn.h" #include "libpq/pqsignal.h" -#include "catalog/namespace.h" -#include "distributed/background_jobs.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/distributed_deadlock_detection.h" -#include "distributed/maintenanced.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/shard_cleaner.h" -#include "distributed/metadata_sync.h" -#include "distributed/query_stats.h" -#include "distributed/statistics_collection.h" -#include "distributed/transaction_recovery.h" -#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "postmaster/bgworker.h" #include "postmaster/postmaster.h" -#include "nodes/makefuncs.h" #include "storage/ipc.h" -#include "storage/proc.h" #include "storage/latch.h" #include "storage/lmgr.h" #include "storage/lwlock.h" +#include "storage/proc.h" #include "tcop/tcopprot.h" -#include "common/hashfn.h" #include "utils/builtins.h" -#include "utils/memutils.h" #include "utils/lsyscache.h" +#include "utils/memutils.h" + +#include "citus_version.h" +#include "pg_version_constants.h" + +#include "distributed/background_jobs.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_deadlock_detection.h" +#include "distributed/maintenanced.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/query_stats.h" #include "distributed/resource_lock.h" +#include "distributed/shard_cleaner.h" +#include "distributed/statistics_collection.h" +#include "distributed/transaction_recovery.h" +#include "distributed/version_compat.h" /* * Shared memory data for all maintenance workers. diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index 404d792f9..ede2008ca 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -6,7 +6,7 @@ */ #include "postgres.h" -#include "pg_version_constants.h" +#include "pgstat.h" #include "access/genam.h" #include "access/heapam.h" @@ -19,8 +19,21 @@ #include "catalog/pg_inherits.h" #include "commands/tablecmds.h" #include "common/string.h" -#include "distributed/citus_nodes.h" +#include "lib/stringinfo.h" +#include "nodes/makefuncs.h" +#include "nodes/pg_list.h" +#include "partitioning/partdesc.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" +#include "utils/varlena.h" + +#include "pg_version_constants.h" + #include "distributed/adaptive_executor.h" +#include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" @@ -36,17 +49,6 @@ #include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "lib/stringinfo.h" -#include "nodes/makefuncs.h" -#include "nodes/pg_list.h" -#include "pgstat.h" -#include "partitioning/partdesc.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/syscache.h" -#include "utils/varlena.h" static char * PartitionBound(Oid partitionId); static Relation try_relation_open_nolock(Oid relationId); diff --git a/src/backend/distributed/utils/namespace_utils.c b/src/backend/distributed/utils/namespace_utils.c index 4f822b7d2..a5401b00c 100644 --- a/src/backend/distributed/utils/namespace_utils.c +++ b/src/backend/distributed/utils/namespace_utils.c @@ -11,10 +11,11 @@ #include "postgres.h" -#include "distributed/namespace_utils.h" #include "utils/guc.h" #include "utils/regproc.h" +#include "distributed/namespace_utils.h" + /* * We use the equivalent of a function SET option to allow the setting to * persist for the exact duration of the transaction, guc.c takes care of diff --git a/src/backend/distributed/utils/param_utils.c b/src/backend/distributed/utils/param_utils.c index 8aefecb7d..a500b5b65 100644 --- a/src/backend/distributed/utils/param_utils.c +++ b/src/backend/distributed/utils/param_utils.c @@ -9,12 +9,13 @@ #include "postgres.h" -#include -#include -#include -#include -#include -#include +#include "nodes/bitmapset.h" +#include "nodes/nodeFuncs.h" +#include "nodes/nodes.h" +#include "nodes/params.h" +#include "nodes/parsenodes.h" +#include "nodes/primnodes.h" + #include "distributed/param_utils.h" /* diff --git a/src/backend/distributed/utils/priority.c b/src/backend/distributed/utils/priority.c index 2e7972d2d..ceb75ac26 100644 --- a/src/backend/distributed/utils/priority.c +++ b/src/backend/distributed/utils/priority.c @@ -7,13 +7,13 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - -#include #include #include #include #include +#include + +#include "postgres.h" #include "distributed/priority.h" diff --git a/src/backend/distributed/utils/query_utils.c b/src/backend/distributed/utils/query_utils.c index 4ae49ed81..ac33bdd52 100644 --- a/src/backend/distributed/utils/query_utils.c +++ b/src/backend/distributed/utils/query_utils.c @@ -11,13 +11,14 @@ */ #include "postgres.h" -#include "nodes/primnodes.h" #include "catalog/pg_class.h" +#include "nodes/nodeFuncs.h" +#include "nodes/primnodes.h" + +#include "distributed/listutils.h" #include "distributed/query_utils.h" #include "distributed/version_compat.h" -#include "distributed/listutils.h" -#include "nodes/nodeFuncs.h" static bool CitusQueryableRangeTableRelation(RangeTblEntry *rangeTableEntry); diff --git a/src/backend/distributed/utils/reference_table_utils.c b/src/backend/distributed/utils/reference_table_utils.c index 314044ab5..b1710c1d6 100644 --- a/src/backend/distributed/utils/reference_table_utils.c +++ b/src/backend/distributed/utils/reference_table_utils.c @@ -10,19 +10,27 @@ */ #include "postgres.h" + #include "miscadmin.h" +#include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" -#include "access/genam.h" +#include "postmaster/postmaster.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" + #include "distributed/backend_data.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" -#include "distributed/listutils.h" #include "distributed/coordinator_protocol.h" -#include "distributed/metadata_utility.h" +#include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_planner.h" #include "distributed/reference_table_utils.h" @@ -33,12 +41,6 @@ #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" -#include "postmaster/postmaster.h" -#include "storage/lmgr.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" /* local function forward declarations */ static List * WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode); diff --git a/src/backend/distributed/utils/replication_origin_session_utils.c b/src/backend/distributed/utils/replication_origin_session_utils.c index 800d82ef7..63038c0ee 100644 --- a/src/backend/distributed/utils/replication_origin_session_utils.c +++ b/src/backend/distributed/utils/replication_origin_session_utils.c @@ -8,12 +8,14 @@ *------------------------------------------------------------------------- */ -#include "distributed/replication_origin_session_utils.h" -#include "distributed/remote_commands.h" -#include "distributed/metadata_cache.h" -#include "utils/builtins.h" #include "miscadmin.h" +#include "utils/builtins.h" + +#include "distributed/metadata_cache.h" +#include "distributed/remote_commands.h" +#include "distributed/replication_origin_session_utils.h" + static bool IsRemoteReplicationOriginSessionSetup(MultiConnection *connection); static void SetupMemoryContextResetReplicationOriginHandler(void); diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index c76830c1d..13e88a16e 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -14,39 +14,41 @@ */ #include "postgres.h" + #include "c.h" #include "miscadmin.h" #include "access/xact.h" #include "catalog/namespace.h" #include "commands/tablecmds.h" -#include "distributed/colocation_utils.h" -#include "distributed/commands.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" -#include "distributed/multi_executor.h" -#include "distributed/multi_join_order.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/distributed_planner.h" -#include "distributed/relay_utility.h" -#include "distributed/reference_table_utils.h" -#include "distributed/remote_commands.h" -#include "distributed/resource_lock.h" -#include "distributed/shardinterval_utils.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_transaction.h" -#include "distributed/utils/array_type.h" -#include "distributed/version_compat.h" -#include "distributed/local_executor.h" -#include "distributed/worker_shard_visibility.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/varlena.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distributed_planner.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_join_order.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/reference_table_utils.h" +#include "distributed/relay_utility.h" +#include "distributed/remote_commands.h" +#include "distributed/resource_lock.h" +#include "distributed/shardinterval_utils.h" +#include "distributed/utils/array_type.h" +#include "distributed/version_compat.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_shard_visibility.h" +#include "distributed/worker_transaction.h" + #define LOCK_RELATION_IF_EXISTS \ "SELECT pg_catalog.lock_relation_if_exists(%s, %s);" diff --git a/src/backend/distributed/utils/role.c b/src/backend/distributed/utils/role.c index 3a9a90f9f..9e92a3290 100644 --- a/src/backend/distributed/utils/role.c +++ b/src/backend/distributed/utils/role.c @@ -10,13 +10,15 @@ */ #include "postgres.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/worker_protocol.h" #include "fmgr.h" + #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/builtins.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/worker_protocol.h" + PG_FUNCTION_INFO_V1(alter_role_if_exists); PG_FUNCTION_INFO_V1(worker_create_or_alter_role); diff --git a/src/backend/distributed/utils/shard_utils.c b/src/backend/distributed/utils/shard_utils.c index d6d41f192..cd688b745 100644 --- a/src/backend/distributed/utils/shard_utils.c +++ b/src/backend/distributed/utils/shard_utils.c @@ -12,9 +12,11 @@ #include "postgres.h" #include "miscadmin.h" + #include "utils/builtins.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" + #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" diff --git a/src/backend/distributed/utils/shardinterval_utils.c b/src/backend/distributed/utils/shardinterval_utils.c index 6c18e201e..16d43ffdc 100644 --- a/src/backend/distributed/utils/shardinterval_utils.c +++ b/src/backend/distributed/utils/shardinterval_utils.c @@ -8,23 +8,25 @@ * *------------------------------------------------------------------------- */ -#include "stdint.h" #include "postgres.h" +#include "stdint.h" + #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "utils/catcache.h" +#include "utils/memutils.h" + +#include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" -#include "distributed/distributed_planner.h" +#include "distributed/pg_dist_partition.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" -#include "distributed/pg_dist_partition.h" #include "distributed/worker_protocol.h" -#include "utils/catcache.h" -#include "utils/memutils.h" /* diff --git a/src/backend/distributed/utils/statistics_collection.c b/src/backend/distributed/utils/statistics_collection.c index a442aac95..1cadea968 100644 --- a/src/backend/distributed/utils/statistics_collection.c +++ b/src/backend/distributed/utils/statistics_collection.c @@ -10,10 +10,12 @@ #include "postgres.h" -#include "citus_version.h" #include "fmgr.h" + #include "utils/uuid.h" +#include "citus_version.h" + #if defined(HAVE_LIBCURL) && defined(ENABLE_CITUS_STATISTICS_COLLECTION) bool EnableStatisticsCollection = true; /* send basic usage statistics to Citus */ #else @@ -28,18 +30,19 @@ PG_FUNCTION_INFO_V1(citus_server_id); #include #include "access/xact.h" +#include "lib/stringinfo.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/json.h" +#include "utils/jsonb.h" + #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/shardinterval_utils.h" #include "distributed/statistics_collection.h" -#include "distributed/worker_manager.h" #include "distributed/version_compat.h" -#include "lib/stringinfo.h" -#include "utils/builtins.h" -#include "utils/json.h" -#include "utils/jsonb.h" -#include "utils/fmgrprotos.h" +#include "distributed/worker_manager.h" static size_t StatisticsCallback(char *contents, size_t size, size_t count, void *userData); diff --git a/src/backend/distributed/utils/task_execution_utils.c b/src/backend/distributed/utils/task_execution_utils.c index 7251514b5..5a6f74283 100644 --- a/src/backend/distributed/utils/task_execution_utils.c +++ b/src/backend/distributed/utils/task_execution_utils.c @@ -1,16 +1,21 @@ -#include "postgres.h" -#include "miscadmin.h" - +#include #include #include -#include + +#include "postgres.h" + +#include "miscadmin.h" + +#include "commands/dbcommands.h" +#include "common/hashfn.h" +#include "storage/fd.h" +#include "utils/builtins.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" #include "pg_version_constants.h" -#include "common/hashfn.h" - -#include "commands/dbcommands.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_nodes.h" #include "distributed/connection_management.h" @@ -27,12 +32,8 @@ #include "distributed/resource_lock.h" #include "distributed/subplan_execution.h" #include "distributed/task_execution_utils.h" -#include "distributed/worker_protocol.h" #include "distributed/version_compat.h" -#include "storage/fd.h" -#include "utils/builtins.h" -#include "utils/hsearch.h" -#include "utils/timestamp.h" +#include "distributed/worker_protocol.h" /* TaskMapKey is used as a key in task hash */ typedef struct TaskMapKey diff --git a/src/backend/distributed/utils/tenant_schema_metadata.c b/src/backend/distributed/utils/tenant_schema_metadata.c index e634795a2..57ae1d151 100644 --- a/src/backend/distributed/utils/tenant_schema_metadata.c +++ b/src/backend/distributed/utils/tenant_schema_metadata.c @@ -14,14 +14,15 @@ #include "access/genam.h" #include "access/htup.h" #include "access/table.h" +#include "storage/lockdefs.h" +#include "utils/fmgroids.h" +#include "utils/relcache.h" + #include "distributed/colocation_utils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/pg_dist_schema.h" #include "distributed/tenant_schema_metadata.h" -#include "storage/lockdefs.h" -#include "utils/relcache.h" -#include "utils/fmgroids.h" /* diff --git a/src/backend/distributed/utils/tuplestore.c b/src/backend/distributed/utils/tuplestore.c index 4473c1f3e..ea59e7040 100644 --- a/src/backend/distributed/utils/tuplestore.c +++ b/src/backend/distributed/utils/tuplestore.c @@ -10,9 +10,10 @@ #include "postgres.h" -#include "distributed/tuplestore.h" #include "miscadmin.h" +#include "distributed/tuplestore.h" + /* * CheckTuplestoreReturn checks if a tuplestore can be returned in the callsite * of the UDF. diff --git a/src/backend/distributed/utils/type_utils.c b/src/backend/distributed/utils/type_utils.c index 66a924a02..fca331374 100644 --- a/src/backend/distributed/utils/type_utils.c +++ b/src/backend/distributed/utils/type_utils.c @@ -10,13 +10,14 @@ */ #include "postgres.h" + #include "fmgr.h" #include "libpq-fe.h" #include "catalog/pg_type.h" +#include "libpq/pqformat.h" #include "nodes/pg_list.h" #include "utils/syscache.h" -#include "libpq/pqformat.h" #include "distributed/causal_clock.h" diff --git a/src/backend/distributed/worker/task_tracker_protocol.c b/src/backend/distributed/worker/task_tracker_protocol.c index 9b2016f67..abe1f765e 100644 --- a/src/backend/distributed/worker/task_tracker_protocol.c +++ b/src/backend/distributed/worker/task_tracker_protocol.c @@ -12,6 +12,7 @@ */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" diff --git a/src/backend/distributed/worker/worker_create_or_replace.c b/src/backend/distributed/worker/worker_create_or_replace.c index 804e71125..2fab84ac6 100644 --- a/src/backend/distributed/worker/worker_create_or_replace.c +++ b/src/backend/distributed/worker/worker_create_or_replace.c @@ -9,23 +9,24 @@ #include "postgres.h" +#include "fmgr.h" +#include "funcapi.h" + #include "access/htup_details.h" #include "catalog/dependency.h" #include "catalog/pg_collation.h" #include "catalog/pg_proc.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_type.h" -#include "fmgr.h" -#include "funcapi.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "parser/parse_type.h" #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/builtins.h" -#include "utils/syscache.h" #include "utils/lsyscache.h" #include "utils/regproc.h" +#include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 11fdda287..f51d9c80c 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -12,12 +12,14 @@ *------------------------------------------------------------------------- */ +#include +#include + #include "postgres.h" + #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" -#include -#include #include "access/xact.h" #include "catalog/dependency.h" @@ -27,6 +29,17 @@ #include "commands/dbcommands.h" #include "commands/extension.h" #include "commands/sequence.h" +#include "executor/spi.h" +#include "nodes/makefuncs.h" +#include "parser/parse_relation.h" +#include "storage/lmgr.h" +#include "tcop/tcopprot.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/regproc.h" +#include "utils/varlena.h" + #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" @@ -44,20 +57,9 @@ #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" - +#include "distributed/version_compat.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_protocol.h" -#include "distributed/version_compat.h" -#include "executor/spi.h" -#include "nodes/makefuncs.h" -#include "parser/parse_relation.h" -#include "storage/lmgr.h" -#include "tcop/tcopprot.h" -#include "tcop/utility.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" -#include "utils/regproc.h" -#include "utils/varlena.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/worker/worker_drop_protocol.c b/src/backend/distributed/worker/worker_drop_protocol.c index 16b7bb66a..6d7b5326a 100644 --- a/src/backend/distributed/worker/worker_drop_protocol.c +++ b/src/backend/distributed/worker/worker_drop_protocol.c @@ -19,22 +19,23 @@ #include "catalog/dependency.h" #include "catalog/pg_depend.h" #include "catalog/pg_foreign_server.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/distribution_column.h" -#include "distributed/listutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/commands/utility_hook.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata/distobject.h" -#include "distributed/multi_partitioning_utils.h" -#include "distributed/worker_protocol.h" #include "foreign/foreign.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/distribution_column.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/worker_protocol.h" + PG_FUNCTION_INFO_V1(worker_drop_distributed_table); PG_FUNCTION_INFO_V1(worker_drop_shell_table); PG_FUNCTION_INFO_V1(worker_drop_sequence_dependency); diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index 2291633d4..cdbda6d3e 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -10,6 +10,7 @@ */ #include "postgres.h" + #include "funcapi.h" #include "miscadmin.h" diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index 63a9cca34..49131ef6d 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -8,26 +8,28 @@ */ #include "postgres.h" + #include "miscadmin.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_type.h" -#include "distributed/backend_data.h" -#include "distributed/metadata_cache.h" -#include "distributed/coordinator_protocol.h" -#include "distributed/listutils.h" -#include "distributed/local_executor.h" -#include "distributed/query_colocation_checker.h" -#include "distributed/worker_protocol.h" -#include "distributed/worker_shard_visibility.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/varlena.h" +#include "distributed/backend_data.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/listutils.h" +#include "distributed/local_executor.h" +#include "distributed/metadata_cache.h" +#include "distributed/query_colocation_checker.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_shard_visibility.h" + /* HideShardsMode is used to determine whether to hide shards */ typedef enum HideShardsMode diff --git a/src/backend/distributed/worker/worker_sql_task_protocol.c b/src/backend/distributed/worker/worker_sql_task_protocol.c index 38dba5e35..a3adbb9d8 100644 --- a/src/backend/distributed/worker/worker_sql_task_protocol.c +++ b/src/backend/distributed/worker/worker_sql_task_protocol.c @@ -10,16 +10,18 @@ */ #include "postgres.h" + #include "funcapi.h" #include "pgstat.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + #include "distributed/commands/multi_copy.h" #include "distributed/multi_executor.h" #include "distributed/transmit.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" -#include "utils/builtins.h" -#include "utils/memutils.h" /* necessary to get S_IRUSR, S_IWUSR definitions on illumos */ #include diff --git a/src/backend/distributed/worker/worker_truncate_trigger_protocol.c b/src/backend/distributed/worker/worker_truncate_trigger_protocol.c index dc4e7ffd8..3f8f96b2d 100644 --- a/src/backend/distributed/worker/worker_truncate_trigger_protocol.c +++ b/src/backend/distributed/worker/worker_truncate_trigger_protocol.c @@ -12,16 +12,18 @@ */ #include "postgres.h" + #include "fmgr.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/metadata_utility.h" -#include "distributed/metadata_cache.h" -#include "distributed/metadata_sync.h" #include "utils/elog.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/metadata_utility.h" + PG_FUNCTION_INFO_V1(worker_create_truncate_trigger); diff --git a/src/include/columnar/columnar.h b/src/include/columnar/columnar.h index 64cf745e1..7167bcc1c 100644 --- a/src/include/columnar/columnar.h +++ b/src/include/columnar/columnar.h @@ -14,11 +14,13 @@ #include "postgres.h" #include "fmgr.h" + #include "lib/stringinfo.h" #include "nodes/parsenodes.h" -#include "pg_version_compat.h" #include "storage/bufpage.h" #include "storage/lockdefs.h" + +#include "pg_version_compat.h" #if PG_VERSION_NUM >= PG_VERSION_16 #include "storage/relfilelocator.h" #else diff --git a/src/include/columnar/columnar_tableam.h b/src/include/columnar/columnar_tableam.h index 657491ef8..18331bd70 100644 --- a/src/include/columnar/columnar_tableam.h +++ b/src/include/columnar/columnar_tableam.h @@ -1,16 +1,18 @@ #ifndef COLUMNAR_TABLEAM_H #define COLUMNAR_TABLEAM_H -#include "citus_version.h" - #include "postgres.h" + #include "fmgr.h" -#include "access/tableam.h" -#include "access/skey.h" -#include "nodes/bitmapset.h" + #include "access/heapam.h" +#include "access/skey.h" +#include "access/tableam.h" #include "catalog/indexing.h" +#include "nodes/bitmapset.h" #include "utils/acl.h" +#include "citus_version.h" + /* * Number of valid ItemPointer Offset's for "row number" <> "ItemPointer" * mapping. diff --git a/src/include/distributed/backend_data.h b/src/include/distributed/backend_data.h index 1fcd31141..8014fe5a6 100644 --- a/src/include/distributed/backend_data.h +++ b/src/include/distributed/backend_data.h @@ -15,12 +15,13 @@ #include "access/twophase.h" #include "datatype/timestamp.h" -#include "distributed/transaction_identifier.h" #include "nodes/pg_list.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "storage/s_lock.h" +#include "distributed/transaction_identifier.h" + /* * Each backend's active distributed transaction information is tracked via diff --git a/src/include/distributed/citus_custom_scan.h b/src/include/distributed/citus_custom_scan.h index a3da4958c..db1f0ce1f 100644 --- a/src/include/distributed/citus_custom_scan.h +++ b/src/include/distributed/citus_custom_scan.h @@ -10,11 +10,12 @@ #ifndef CITUS_CUSTOM_SCAN_H #define CITUS_CUSTOM_SCAN_H -#include "distributed/distributed_planner.h" -#include "distributed/multi_server_executor.h" #include "executor/execdesc.h" #include "nodes/plannodes.h" +#include "distributed/distributed_planner.h" +#include "distributed/multi_server_executor.h" + typedef struct CitusScanState { CustomScanState customScanState; /* underlying custom scan node */ diff --git a/src/include/distributed/citus_depended_object.h b/src/include/distributed/citus_depended_object.h index b52018411..1efbe4e2a 100644 --- a/src/include/distributed/citus_depended_object.h +++ b/src/include/distributed/citus_depended_object.h @@ -12,10 +12,11 @@ #ifndef CITUS_DEPENDED_OBJECT_H #define CITUS_DEPENDED_OBJECT_H -#include "distributed/commands.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" +#include "distributed/commands.h" + extern bool HideCitusDependentObjects; /* DistOpsValidationState to be used to determine validity of dist ops */ diff --git a/src/include/distributed/citus_nodefuncs.h b/src/include/distributed/citus_nodefuncs.h index caeda3a72..f7c0061b9 100644 --- a/src/include/distributed/citus_nodefuncs.h +++ b/src/include/distributed/citus_nodefuncs.h @@ -11,10 +11,11 @@ #ifndef CITUS_NODEFUNCS_H #define CITUS_NODEFUNCS_H -#include "distributed/multi_physical_planner.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" +#include "distributed/multi_physical_planner.h" + /* citus_nodefuncs.c */ extern void SetRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSchemaName, char *fragmentTableName, diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index e45ddb269..3a9c36482 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -15,11 +15,12 @@ #include "catalog/pg_sequence.h" #include "commands/sequence.h" -#include "distributed/coordinator_protocol.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" +#include "distributed/coordinator_protocol.h" + /* Function declarations for version independent Citus ruleutils wrapper functions */ extern char * pg_get_extensiondef_string(Oid tableRelationId); extern Oid get_extension_schema(Oid ext_oid); diff --git a/src/include/distributed/colocation_utils.h b/src/include/distributed/colocation_utils.h index bba78afd1..018f97570 100644 --- a/src/include/distributed/colocation_utils.h +++ b/src/include/distributed/colocation_utils.h @@ -12,9 +12,10 @@ #ifndef COLOCATION_UTILS_H_ #define COLOCATION_UTILS_H_ -#include "distributed/shardinterval_utils.h" #include "nodes/pg_list.h" +#include "distributed/shardinterval_utils.h" + #define INVALID_COLOCATION_ID 0 extern uint32 TableColocationId(Oid distributedTableId); diff --git a/src/include/distributed/combine_query_planner.h b/src/include/distributed/combine_query_planner.h index 710010913..2afc8aa5f 100644 --- a/src/include/distributed/combine_query_planner.h +++ b/src/include/distributed/combine_query_planner.h @@ -14,9 +14,8 @@ #include "lib/stringinfo.h" #include "nodes/parsenodes.h" -#include "nodes/plannodes.h" - #include "nodes/pathnodes.h" +#include "nodes/plannodes.h" /* Function declarations for building local plans on the coordinator node */ diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 309149293..ec2cb8a57 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -15,12 +15,13 @@ #include "postgres.h" -#include "distributed/metadata_utility.h" -#include "utils/rel.h" #include "nodes/parsenodes.h" #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/acl.h" +#include "utils/rel.h" + +#include "distributed/metadata_utility.h" extern bool AddAllLocalTablesToMetadata; diff --git a/src/include/distributed/commands/multi_copy.h b/src/include/distributed/commands/multi_copy.h index fa59894ad..1fc42df60 100644 --- a/src/include/distributed/commands/multi_copy.h +++ b/src/include/distributed/commands/multi_copy.h @@ -13,14 +13,15 @@ #define MULTI_COPY_H -#include "distributed/metadata_utility.h" -#include "distributed/metadata_cache.h" -#include "distributed/version_compat.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_coerce.h" #include "tcop/dest.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "distributed/version_compat.h" + #define INVALID_PARTITION_COLUMN_INDEX -1 diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 3a42cf55e..a3011edbc 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -10,12 +10,12 @@ #ifndef MULTI_UTILITY_H #define MULTI_UTILITY_H -#include "pg_version_constants.h" - #include "postgres.h" -#include "utils/relcache.h" #include "tcop/utility.h" +#include "utils/relcache.h" + +#include "pg_version_constants.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_call_delegation.h" diff --git a/src/include/distributed/connection_management.h b/src/include/distributed/connection_management.h index 158f0b1ce..9eadbde9d 100644 --- a/src/include/distributed/connection_management.h +++ b/src/include/distributed/connection_management.h @@ -13,16 +13,18 @@ #include "postgres.h" -#include "distributed/transaction_management.h" -#include "distributed/remote_transaction.h" -#include "lib/ilist.h" #include "pg_config.h" + +#include "lib/ilist.h" #include "portability/instr_time.h" #include "storage/latch.h" #include "utils/guc.h" #include "utils/hsearch.h" #include "utils/timestamp.h" +#include "distributed/remote_transaction.h" +#include "distributed/transaction_management.h" + /* maximum (textual) lengths of hostname and port */ #define MAX_NODE_LENGTH 255 /* includes 0 byte */ diff --git a/src/include/distributed/coordinator_protocol.h b/src/include/distributed/coordinator_protocol.h index 0dcc66141..b2170fd2e 100644 --- a/src/include/distributed/coordinator_protocol.h +++ b/src/include/distributed/coordinator_protocol.h @@ -13,16 +13,18 @@ #define COORDINATOR_PROTOCOL_H #include "postgres.h" + #include "c.h" #include "fmgr.h" -#include "distributed/connection_management.h" -#include "distributed/shardinterval_utils.h" #include "nodes/pg_list.h" -#include "distributed/metadata_utility.h" #include "columnar/columnar.h" +#include "distributed/connection_management.h" +#include "distributed/metadata_utility.h" +#include "distributed/shardinterval_utils.h" + /* * In our distributed database, we need a mechanism to make remote procedure * calls between clients, the coordinator node, and worker nodes. These remote calls diff --git a/src/include/distributed/deparse_shard_query.h b/src/include/distributed/deparse_shard_query.h index 9370e51e2..8fb012588 100644 --- a/src/include/distributed/deparse_shard_query.h +++ b/src/include/distributed/deparse_shard_query.h @@ -18,6 +18,7 @@ #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" + #include "distributed/citus_custom_scan.h" diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index aeefc811d..33bc8ac62 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -15,10 +15,10 @@ #include "postgres.h" -#include "nodes/nodes.h" -#include "nodes/parsenodes.h" #include "catalog/objectaddress.h" #include "lib/stringinfo.h" +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" /* forward declarations for format_collate.c */ /* Control flags for FormatCollateExtended, compatible with format_type_extended */ diff --git a/src/include/distributed/distributed_deadlock_detection.h b/src/include/distributed/distributed_deadlock_detection.h index 23f6554ef..5f391cc70 100644 --- a/src/include/distributed/distributed_deadlock_detection.h +++ b/src/include/distributed/distributed_deadlock_detection.h @@ -14,11 +14,12 @@ #include "postgres.h" #include "access/hash.h" +#include "nodes/pg_list.h" + #include "distributed/backend_data.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/transaction_identifier.h" -#include "nodes/pg_list.h" typedef struct TransactionNode { diff --git a/src/include/distributed/distributed_execution_locks.h b/src/include/distributed/distributed_execution_locks.h index e789843ae..3ca31b330 100644 --- a/src/include/distributed/distributed_execution_locks.h +++ b/src/include/distributed/distributed_execution_locks.h @@ -14,6 +14,7 @@ #include "nodes/pg_list.h" #include "storage/lockdefs.h" + #include "distributed/multi_physical_planner.h" extern void AcquireExecutorShardLocksForExecution(RowModifyLevel modLevel, diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index bc8f5bc94..d7234e4bc 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -12,11 +12,10 @@ #include "postgres.h" -#include "pg_version_constants.h" - +#include "nodes/pathnodes.h" #include "nodes/plannodes.h" -#include "nodes/pathnodes.h" +#include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" diff --git a/src/include/distributed/enterprise.h b/src/include/distributed/enterprise.h index 26a882bd6..2ba2fa1ff 100644 --- a/src/include/distributed/enterprise.h +++ b/src/include/distributed/enterprise.h @@ -13,6 +13,7 @@ #define CITUS_ENTERPRISE_H #include "postgres.h" + #include "fmgr.h" diff --git a/src/include/distributed/errormessage.h b/src/include/distributed/errormessage.h index 3c19a9c83..7a38d513c 100644 --- a/src/include/distributed/errormessage.h +++ b/src/include/distributed/errormessage.h @@ -11,9 +11,11 @@ #define ERRORMESSAGE_H #include "c.h" -#include "distributed/citus_nodes.h" + #include "pg_version_compat.h" +#include "distributed/citus_nodes.h" + typedef struct DeferredErrorMessage { diff --git a/src/include/distributed/executor_util.h b/src/include/distributed/executor_util.h index 8560c6dfd..b39122d26 100644 --- a/src/include/distributed/executor_util.h +++ b/src/include/distributed/executor_util.h @@ -12,10 +12,11 @@ #include "funcapi.h" #include "access/tupdesc.h" -#include "distributed/multi_physical_planner.h" #include "nodes/params.h" #include "nodes/pg_list.h" +#include "distributed/multi_physical_planner.h" + /* utility functions for dealing with tasks in the executor */ extern bool TaskListModifiesDatabase(RowModifyLevel modLevel, List *taskList); diff --git a/src/include/distributed/foreign_key_relationship.h b/src/include/distributed/foreign_key_relationship.h index fbbee831e..bbaf8be73 100644 --- a/src/include/distributed/foreign_key_relationship.h +++ b/src/include/distributed/foreign_key_relationship.h @@ -10,10 +10,12 @@ #define FOREIGN_KEY_RELATIONSHIP_H #include "postgres.h" + #include "postgres_ext.h" -#include "utils/relcache.h" -#include "utils/hsearch.h" + #include "nodes/primnodes.h" +#include "utils/hsearch.h" +#include "utils/relcache.h" extern List * GetForeignKeyConnectedRelationIdList(Oid relationId); extern bool ShouldUndistributeCitusLocalTable(Oid relationId); diff --git a/src/include/distributed/hash_helpers.h b/src/include/distributed/hash_helpers.h index 168879b4d..b64bfde71 100644 --- a/src/include/distributed/hash_helpers.h +++ b/src/include/distributed/hash_helpers.h @@ -11,10 +11,10 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "utils/hsearch.h" +#include "pg_version_constants.h" + /* * assert_valid_hash_key2 checks if a type that contains 2 fields contains no * padding bytes. This is necessary to use a type as a hash key with tag_hash. diff --git a/src/include/distributed/insert_select_planner.h b/src/include/distributed/insert_select_planner.h index 771d1d60f..a9100b02d 100644 --- a/src/include/distributed/insert_select_planner.h +++ b/src/include/distributed/insert_select_planner.h @@ -16,12 +16,13 @@ #include "postgres.h" -#include "distributed/multi_physical_planner.h" -#include "distributed/distributed_planner.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h" +#include "distributed/distributed_planner.h" +#include "distributed/multi_physical_planner.h" + extern bool InsertSelectIntoCitusTable(Query *query); extern bool CheckInsertSelectQuery(Query *query); diff --git a/src/include/distributed/intermediate_results.h b/src/include/distributed/intermediate_results.h index 63eca5ad1..ca4fa581e 100644 --- a/src/include/distributed/intermediate_results.h +++ b/src/include/distributed/intermediate_results.h @@ -14,13 +14,14 @@ #include "fmgr.h" -#include "distributed/commands/multi_copy.h" #include "nodes/execnodes.h" #include "nodes/pg_list.h" #include "tcop/dest.h" #include "utils/builtins.h" #include "utils/palloc.h" +#include "distributed/commands/multi_copy.h" + /* * DistributedResultFragment represents a fragment of a distributed result. diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index 833c77d22..2a52cbc75 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -13,13 +13,15 @@ #define CITUS_LISTUTILS_H #include "postgres.h" + #include "c.h" #include "nodes/pg_list.h" -#include "pg_version_compat.h" #include "utils/array.h" #include "utils/hsearch.h" +#include "pg_version_compat.h" + /* * ListCellAndListWrapper stores a list and list cell. diff --git a/src/include/distributed/local_distributed_join_planner.h b/src/include/distributed/local_distributed_join_planner.h index dfb45f149..3390ab213 100644 --- a/src/include/distributed/local_distributed_join_planner.h +++ b/src/include/distributed/local_distributed_join_planner.h @@ -14,6 +14,7 @@ #define LOCAL_DISTRIBUTED_JOIN_PLANNER_H #include "postgres.h" + #include "distributed/recursive_planning.h" /* managed via guc.c */ diff --git a/src/include/distributed/lock_graph.h b/src/include/distributed/lock_graph.h index f204ebb03..e14a51580 100644 --- a/src/include/distributed/lock_graph.h +++ b/src/include/distributed/lock_graph.h @@ -14,12 +14,14 @@ #include "postgres.h" + #include "libpq-fe.h" #include "datatype/timestamp.h" -#include "distributed/backend_data.h" #include "storage/lock.h" +#include "distributed/backend_data.h" + /* * Describes an edge in a waiting-for graph of locks. This isn't used for diff --git a/src/include/distributed/merge_planner.h b/src/include/distributed/merge_planner.h index 898292603..b6636687a 100644 --- a/src/include/distributed/merge_planner.h +++ b/src/include/distributed/merge_planner.h @@ -15,6 +15,7 @@ #include "c.h" #include "nodes/parsenodes.h" + #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/multi_physical_planner.h" diff --git a/src/include/distributed/metadata/dependency.h b/src/include/distributed/metadata/dependency.h index 2d3759e1f..2cfefc87e 100644 --- a/src/include/distributed/metadata/dependency.h +++ b/src/include/distributed/metadata/dependency.h @@ -16,9 +16,10 @@ #include "catalog/objectaddress.h" #include "catalog/pg_depend.h" -#include "distributed/errormessage.h" #include "nodes/pg_list.h" +#include "distributed/errormessage.h" + typedef bool (*AddressPredicate)(const ObjectAddress *); extern List * GetUniqueDependenciesList(List *objectAddressesList); diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index 34b95b859..f1120497b 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -14,10 +14,12 @@ #include "postgres.h" #include "fmgr.h" + +#include "utils/hsearch.h" + #include "distributed/metadata_utility.h" #include "distributed/pg_dist_partition.h" #include "distributed/worker_manager.h" -#include "utils/hsearch.h" extern bool EnableVersionChecks; diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index 237df363a..4e190ec39 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -13,10 +13,11 @@ #define METADATA_SYNC_H +#include "nodes/pg_list.h" + #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" -#include "nodes/pg_list.h" /* managed via guc.c */ typedef enum diff --git a/src/include/distributed/metadata_utility.h b/src/include/distributed/metadata_utility.h index 9234adc76..d93e2255f 100644 --- a/src/include/distributed/metadata_utility.h +++ b/src/include/distributed/metadata_utility.h @@ -21,13 +21,14 @@ #include "access/tupdesc.h" #include "catalog/indexing.h" #include "catalog/objectaddress.h" +#include "utils/acl.h" +#include "utils/relcache.h" + #include "distributed/citus_nodes.h" #include "distributed/connection_management.h" #include "distributed/errormessage.h" #include "distributed/relay_utility.h" #include "distributed/worker_manager.h" -#include "utils/acl.h" -#include "utils/relcache.h" /* total number of hash tokens (2^32) */ diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h index 5ae010d87..6708d9a64 100644 --- a/src/include/distributed/multi_executor.h +++ b/src/include/distributed/multi_executor.h @@ -11,8 +11,8 @@ #define MULTI_EXECUTOR_H #include "executor/execdesc.h" -#include "nodes/parsenodes.h" #include "nodes/execnodes.h" +#include "nodes/parsenodes.h" #include "distributed/citus_custom_scan.h" #include "distributed/multi_physical_planner.h" diff --git a/src/include/distributed/multi_explain.h b/src/include/distributed/multi_explain.h index 296634905..f6dad83c2 100644 --- a/src/include/distributed/multi_explain.h +++ b/src/include/distributed/multi_explain.h @@ -10,9 +10,10 @@ #ifndef MULTI_EXPLAIN_H #define MULTI_EXPLAIN_H -#include "executor/executor.h" #include "tuple_destination.h" +#include "executor/executor.h" + typedef enum { EXPLAIN_ANALYZE_SORT_BY_TIME = 0, diff --git a/src/include/distributed/multi_logical_planner.h b/src/include/distributed/multi_logical_planner.h index de4901ea2..f68fd3ed5 100644 --- a/src/include/distributed/multi_logical_planner.h +++ b/src/include/distributed/multi_logical_planner.h @@ -14,15 +14,16 @@ #ifndef MULTI_LOGICAL_PLANNER_H #define MULTI_LOGICAL_PLANNER_H +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" + #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_join_order.h" #include "distributed/relation_restriction_equivalence.h" -#include "nodes/nodes.h" -#include "nodes/primnodes.h" -#include "nodes/parsenodes.h" -#include "nodes/pg_list.h" #define SUBQUERY_RANGE_TABLE_ID -1 diff --git a/src/include/distributed/multi_logical_replication.h b/src/include/distributed/multi_logical_replication.h index f5a9dc342..2a57c0224 100644 --- a/src/include/distributed/multi_logical_replication.h +++ b/src/include/distributed/multi_logical_replication.h @@ -15,6 +15,7 @@ #include "c.h" #include "nodes/pg_list.h" + #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" #include "distributed/shard_cleaner.h" diff --git a/src/include/distributed/multi_partitioning_utils.h b/src/include/distributed/multi_partitioning_utils.h index b8cfe38c0..7d76b9aa3 100644 --- a/src/include/distributed/multi_partitioning_utils.h +++ b/src/include/distributed/multi_partitioning_utils.h @@ -8,9 +8,10 @@ #define MULTI_PARTITIONING_UTILS_H_ -#include "distributed/metadata_utility.h" #include "nodes/pg_list.h" +#include "distributed/metadata_utility.h" + extern bool PartitionedTable(Oid relationId); extern bool PartitionedTableNoLock(Oid relationId); diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 89d144a92..9278be5af 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -16,22 +16,23 @@ #include "postgres.h" -#include "pg_version_constants.h" - #include "c.h" #include "datatype/timestamp.h" -#include "distributed/citus_nodes.h" -#include "distributed/errormessage.h" -#include "distributed/log_utils.h" -#include "distributed/metadata_utility.h" -#include "distributed/worker_manager.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/distributed_planner.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/array.h" +#include "pg_version_constants.h" + +#include "distributed/citus_nodes.h" +#include "distributed/distributed_planner.h" +#include "distributed/errormessage.h" +#include "distributed/log_utils.h" +#include "distributed/metadata_utility.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/worker_manager.h" + /* Definitions local to the physical planner */ #define NON_PRUNABLE_JOIN -1 diff --git a/src/include/distributed/multi_progress.h b/src/include/distributed/multi_progress.h index 64bad527f..2a9cf1cbb 100644 --- a/src/include/distributed/multi_progress.h +++ b/src/include/distributed/multi_progress.h @@ -16,6 +16,7 @@ #include "postgres.h" #include "fmgr.h" + #include "nodes/pg_list.h" #include "storage/dsm.h" diff --git a/src/include/distributed/multi_router_planner.h b/src/include/distributed/multi_router_planner.h index 160cf6605..ae75ee631 100644 --- a/src/include/distributed/multi_router_planner.h +++ b/src/include/distributed/multi_router_planner.h @@ -14,12 +14,13 @@ #include "c.h" +#include "nodes/parsenodes.h" + +#include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" -#include "distributed/distributed_planner.h" -#include "nodes/parsenodes.h" /* reserved alias name for UPSERTs */ diff --git a/src/include/distributed/placement_access.h b/src/include/distributed/placement_access.h index 28b05baae..0eafa678a 100644 --- a/src/include/distributed/placement_access.h +++ b/src/include/distributed/placement_access.h @@ -11,7 +11,9 @@ #define PLACEMENT_ACCESS_H #include "postgres.h" + #include "nodes/pg_list.h" + #include "distributed/multi_physical_planner.h" /* forward declare, to avoid dependency on ShardPlacement definition */ diff --git a/src/include/distributed/query_colocation_checker.h b/src/include/distributed/query_colocation_checker.h index 562869a92..2a46d364c 100644 --- a/src/include/distributed/query_colocation_checker.h +++ b/src/include/distributed/query_colocation_checker.h @@ -11,10 +11,11 @@ #define QUERY_COLOCATION_CHECKER_H -#include "distributed/distributed_planner.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" +#include "distributed/distributed_planner.h" + /* * ColocatedJoinChecker is a helper structure that is used to decide diff --git a/src/include/distributed/query_pushdown_planning.h b/src/include/distributed/query_pushdown_planning.h index 061a4a730..e0d4f25dd 100644 --- a/src/include/distributed/query_pushdown_planning.h +++ b/src/include/distributed/query_pushdown_planning.h @@ -13,10 +13,10 @@ #include "postgres.h" #include "distributed/distributed_planner.h" -#include "distributed/multi_logical_planner.h" -#include "distributed/multi_physical_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_physical_planner.h" /* Config variables managed via guc.c */ diff --git a/src/include/distributed/query_utils.h b/src/include/distributed/query_utils.h index 7e1ba54e6..0b216d158 100644 --- a/src/include/distributed/query_utils.h +++ b/src/include/distributed/query_utils.h @@ -12,6 +12,7 @@ #define QUERY_UTILS_H #include "postgres.h" + #include "nodes/pg_list.h" #include "nodes/primnodes.h" diff --git a/src/include/distributed/recursive_planning.h b/src/include/distributed/recursive_planning.h index 87df7fba2..c37eba343 100644 --- a/src/include/distributed/recursive_planning.h +++ b/src/include/distributed/recursive_planning.h @@ -10,13 +10,15 @@ #ifndef RECURSIVE_PLANNING_H #define RECURSIVE_PLANNING_H +#include "nodes/pathnodes.h" +#include "nodes/pg_list.h" +#include "nodes/primnodes.h" + #include "pg_version_constants.h" + #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/relation_restriction_equivalence.h" -#include "nodes/pg_list.h" -#include "nodes/primnodes.h" -#include "nodes/pathnodes.h" typedef struct RecursivePlanningContextInternal RecursivePlanningContext; diff --git a/src/include/distributed/relay_utility.h b/src/include/distributed/relay_utility.h index f5a37da45..6fa2172e3 100644 --- a/src/include/distributed/relay_utility.h +++ b/src/include/distributed/relay_utility.h @@ -16,6 +16,7 @@ #define RELAY_UTILITY_H #include "fmgr.h" + #include "lib/stringinfo.h" #include "nodes/nodes.h" diff --git a/src/include/distributed/remote_transaction.h b/src/include/distributed/remote_transaction.h index 6136f25c9..1c422da20 100644 --- a/src/include/distributed/remote_transaction.h +++ b/src/include/distributed/remote_transaction.h @@ -12,8 +12,9 @@ #include "libpq-fe.h" -#include "nodes/pg_list.h" + #include "lib/ilist.h" +#include "nodes/pg_list.h" /* forward declare, to avoid recursive includes */ diff --git a/src/include/distributed/replication_origin_session_utils.h b/src/include/distributed/replication_origin_session_utils.h index e90bd8ab8..b11d56ffc 100644 --- a/src/include/distributed/replication_origin_session_utils.h +++ b/src/include/distributed/replication_origin_session_utils.h @@ -12,7 +12,9 @@ #define REPLICATION_ORIGIN_SESSION_UTILS_H #include "postgres.h" + #include "replication/origin.h" + #include "distributed/connection_management.h" extern void InitializeReplicationOriginSessionUtils(void); diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h index 9efa1b767..8e0e36735 100644 --- a/src/include/distributed/resource_lock.h +++ b/src/include/distributed/resource_lock.h @@ -11,13 +11,15 @@ #define RESOURCE_LOCK_H #include "postgres.h" /* IWYU pragma: keep */ + #include "c.h" -#include "distributed/worker_transaction.h" #include "nodes/pg_list.h" #include "storage/lock.h" #include "tcop/utility.h" +#include "distributed/worker_transaction.h" + /* * Postgres' advisory locks use 'field4' to discern between different kind of diff --git a/src/include/distributed/shard_pruning.h b/src/include/distributed/shard_pruning.h index 04176314e..1b1ffce8a 100644 --- a/src/include/distributed/shard_pruning.h +++ b/src/include/distributed/shard_pruning.h @@ -11,9 +11,10 @@ #ifndef SHARD_PRUNING_H_ #define SHARD_PRUNING_H_ -#include "distributed/metadata_cache.h" #include "nodes/primnodes.h" +#include "distributed/metadata_cache.h" + #define INVALID_SHARD_INDEX -1 /* Function declarations for shard pruning */ diff --git a/src/include/distributed/shard_rebalancer.h b/src/include/distributed/shard_rebalancer.h index 38ce4f485..8e47ac1e5 100644 --- a/src/include/distributed/shard_rebalancer.h +++ b/src/include/distributed/shard_rebalancer.h @@ -17,7 +17,9 @@ #include "postgres.h" #include "fmgr.h" + #include "nodes/pg_list.h" + #include "distributed/coordinator_protocol.h" #include "distributed/worker_manager.h" diff --git a/src/include/distributed/shard_transfer.h b/src/include/distributed/shard_transfer.h index a6d024a2e..c1621879b 100644 --- a/src/include/distributed/shard_transfer.h +++ b/src/include/distributed/shard_transfer.h @@ -9,9 +9,10 @@ #include "postgres.h" -#include "distributed/shard_rebalancer.h" #include "nodes/pg_list.h" +#include "distributed/shard_rebalancer.h" + extern Datum citus_move_shard_placement(PG_FUNCTION_ARGS); extern Datum citus_move_shard_placement_with_nodeid(PG_FUNCTION_ARGS); diff --git a/src/include/distributed/shardinterval_utils.h b/src/include/distributed/shardinterval_utils.h index 4cc99e6d5..ed5600a11 100644 --- a/src/include/distributed/shardinterval_utils.h +++ b/src/include/distributed/shardinterval_utils.h @@ -12,10 +12,11 @@ #ifndef SHARDINTERVAL_UTILS_H_ #define SHARDINTERVAL_UTILS_H_ -#include "distributed/metadata_utility.h" -#include "distributed/metadata_cache.h" #include "nodes/primnodes.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" + #define INVALID_SHARD_INDEX -1 /* OperatorCacheEntry contains information for each element in OperatorCache */ diff --git a/src/include/distributed/transaction_management.h b/src/include/distributed/transaction_management.h index ca4e632a9..fa762682b 100644 --- a/src/include/distributed/transaction_management.h +++ b/src/include/distributed/transaction_management.h @@ -14,7 +14,6 @@ #include "lib/ilist.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" -#include "lib/stringinfo.h" #include "nodes/primnodes.h" #include "utils/hsearch.h" diff --git a/src/include/distributed/tuple_destination.h b/src/include/distributed/tuple_destination.h index 0480ffdc4..5b4f64983 100644 --- a/src/include/distributed/tuple_destination.h +++ b/src/include/distributed/tuple_destination.h @@ -11,10 +11,11 @@ #define TUPLE_DESTINATION_H #include "access/tupdesc.h" -#include "distributed/multi_physical_planner.h" #include "tcop/dest.h" #include "utils/tuplestore.h" +#include "distributed/multi_physical_planner.h" + typedef struct TupleDestination TupleDestination; diff --git a/src/include/distributed/utils/citus_stat_tenants.h b/src/include/distributed/utils/citus_stat_tenants.h index 0a482b241..573502606 100644 --- a/src/include/distributed/utils/citus_stat_tenants.h +++ b/src/include/distributed/utils/citus_stat_tenants.h @@ -11,13 +11,14 @@ #ifndef CITUS_ATTRIBUTE_H #define CITUS_ATTRIBUTE_H -#include "distributed/hash_helpers.h" #include "executor/execdesc.h" #include "executor/executor.h" #include "storage/lwlock.h" #include "utils/datetime.h" #include "utils/hsearch.h" +#include "distributed/hash_helpers.h" + #define MAX_TENANT_ATTRIBUTE_LENGTH 100 /* diff --git a/src/include/distributed/utils/directory.h b/src/include/distributed/utils/directory.h index 7ed8a3f95..76b6cf1df 100644 --- a/src/include/distributed/utils/directory.h +++ b/src/include/distributed/utils/directory.h @@ -12,6 +12,7 @@ #define CITUS_DIRECTORY_H #include "postgres.h" + #include "lib/stringinfo.h" diff --git a/src/include/distributed/utils/function.h b/src/include/distributed/utils/function.h index 91d4ab84b..6f527218c 100644 --- a/src/include/distributed/utils/function.h +++ b/src/include/distributed/utils/function.h @@ -12,6 +12,7 @@ #define CITUS_FUNCTION_H #include "postgres.h" + #include "fmgr.h" diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index b990b82ef..f450dc1ce 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -13,21 +13,21 @@ #include "postgres.h" -#include "access/sdir.h" #include "access/heapam.h" -#include "commands/explain.h" +#include "access/sdir.h" #include "catalog/namespace.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/citus_safe_lib.h" +#include "commands/explain.h" #include "executor/tuptable.h" #include "nodes/parsenodes.h" -#include "parser/parse_func.h" #include "optimizer/optimizer.h" - +#include "parser/parse_func.h" #include "tcop/tcopprot.h" #include "pg_version_compat.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/citus_safe_lib.h" + typedef struct { File fd; diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h index 5ad7f4962..84d9a9200 100644 --- a/src/include/distributed/worker_manager.h +++ b/src/include/distributed/worker_manager.h @@ -16,9 +16,9 @@ #include "postgres.h" +#include "nodes/pg_list.h" #include "storage/lmgr.h" #include "storage/lockdefs.h" -#include "nodes/pg_list.h" /* Worker nodeName's, nodePort's, and nodeCluster's maximum length */ diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index 29d364247..21c0c44c8 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -17,11 +17,13 @@ #include "postgres.h" #include "fmgr.h" -#include "distributed/shardinterval_utils.h" + #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "storage/fd.h" #include "utils/array.h" + +#include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" diff --git a/src/include/distributed/worker_transaction.h b/src/include/distributed/worker_transaction.h index 631940edf..59ca8f869 100644 --- a/src/include/distributed/worker_transaction.h +++ b/src/include/distributed/worker_transaction.h @@ -12,9 +12,10 @@ #ifndef WORKER_TRANSACTION_H #define WORKER_TRANSACTION_H +#include "storage/lockdefs.h" + #include "distributed/connection_management.h" #include "distributed/worker_manager.h" -#include "storage/lockdefs.h" /* From 26178fb53895e0fcaef53f27351caa0024c44fd8 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 16 Apr 2024 11:41:13 +0200 Subject: [PATCH 016/155] Add missing postgres.h includes After sorting includes in the previous commit some files were now invalid because they were not including postgres.h --- src/backend/distributed/planner/intermediate_result_pruning.c | 2 ++ .../distributed/utils/replication_origin_session_utils.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/backend/distributed/planner/intermediate_result_pruning.c b/src/backend/distributed/planner/intermediate_result_pruning.c index 474d767d7..d4d9c0e21 100644 --- a/src/backend/distributed/planner/intermediate_result_pruning.c +++ b/src/backend/distributed/planner/intermediate_result_pruning.c @@ -12,6 +12,8 @@ *------------------------------------------------------------------------- */ +#include "postgres.h" + #include "common/hashfn.h" #include "utils/builtins.h" diff --git a/src/backend/distributed/utils/replication_origin_session_utils.c b/src/backend/distributed/utils/replication_origin_session_utils.c index 63038c0ee..eaf64d2cc 100644 --- a/src/backend/distributed/utils/replication_origin_session_utils.c +++ b/src/backend/distributed/utils/replication_origin_session_utils.c @@ -8,6 +8,8 @@ *------------------------------------------------------------------------- */ +#include "postgres.h" + #include "miscadmin.h" #include "utils/builtins.h" From fa6743d43629253ed581dac5389ab32f77125105 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Mon, 15 Apr 2024 12:28:11 +0200 Subject: [PATCH 017/155] Speed up EnsureSequenceTypeSupported (#7575) DESCRIPTION: Fix performance issue when creating distributed tables and many already exist EnsureSequenceTypeSupported was doing an O(number of distributed tables) operation. This can become very slow with lots of Citus tables, which now happens much more frequently in practice due to schema based sharding. Partially addresses #7022 (cherry picked from commit 381f31756e6de0e0522aeaa489fe671df0ddf731) --- .../commands/create_distributed_table.c | 73 +++++++--------- .../distributed/metadata/metadata_sync.c | 84 +++++++++++++++++++ src/include/distributed/metadata_sync.h | 4 + 3 files changed, 119 insertions(+), 42 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 9f3975a1e..fced0fe3a 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -22,6 +22,7 @@ #include "catalog/dependency.h" #include "catalog/index.h" #include "catalog/pg_am.h" +#include "catalog/pg_attrdef.h" #include "catalog/pg_attribute.h" #include "catalog/pg_enum.h" #include "catalog/pg_extension.h" @@ -50,6 +51,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -1699,52 +1701,39 @@ PropagatePrerequisiteObjectsForDistributedTable(Oid relationId) void EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid ownerRelationId) { - List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); - citusTableIdList = list_append_unique_oid(citusTableIdList, ownerRelationId); + Oid attrDefOid; + List *attrDefOids = GetAttrDefsFromSequence(seqOid); - Oid citusTableId = InvalidOid; - foreach_oid(citusTableId, citusTableIdList) + foreach_oid(attrDefOid, attrDefOids) { - List *seqInfoList = NIL; - GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0, DEPENDENCY_AUTO); + ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); - SequenceInfo *seqInfo = NULL; - foreach_ptr(seqInfo, seqInfoList) + /* + * If another distributed table is using the same sequence + * in one of its column defaults, make sure the types of the + * columns match. + * + * We skip non-distributed tables, but we need to check the current + * table as it might reference the same sequence multiple times. + */ + if (columnAddress.objectId != ownerRelationId && + !IsCitusTable(columnAddress.objectId)) { - AttrNumber currentAttnum = seqInfo->attributeNumber; - Oid currentSeqOid = seqInfo->sequenceOid; - - if (!seqInfo->isNextValDefault) - { - /* - * If a sequence is not on the nextval, we don't need any check. - * This is a dependent sequence via ALTER SEQUENCE .. OWNED BY col - */ - continue; - } - - /* - * If another distributed table is using the same sequence - * in one of its column defaults, make sure the types of the - * columns match - */ - if (currentSeqOid == seqOid) - { - Oid currentAttributeTypId = GetAttributeTypeOid(citusTableId, - currentAttnum); - if (attributeTypeId != currentAttributeTypId) - { - char *sequenceName = generate_qualified_relation_name( - seqOid); - char *citusTableName = - generate_qualified_relation_name(citusTableId); - ereport(ERROR, (errmsg( - "The sequence %s is already used for a different" - " type in column %d of the table %s", - sequenceName, currentAttnum, - citusTableName))); - } - } + continue; + } + Oid currentAttributeTypId = GetAttributeTypeOid(columnAddress.objectId, + columnAddress.objectSubId); + if (attributeTypeId != currentAttributeTypId) + { + char *sequenceName = generate_qualified_relation_name( + seqOid); + char *citusTableName = + generate_qualified_relation_name(columnAddress.objectId); + ereport(ERROR, (errmsg( + "The sequence %s is already used for a different" + " type in column %d of the table %s", + sequenceName, columnAddress.objectSubId, + citusTableName))); } } } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 0492e7b55..511cc45a9 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1671,6 +1671,90 @@ GetSequencesFromAttrDef(Oid attrdefOid) } +#if PG_VERSION_NUM < PG_VERSION_15 + +/* + * Given a pg_attrdef OID, return the relation OID and column number of + * the owning column (represented as an ObjectAddress for convenience). + * + * Returns InvalidObjectAddress if there is no such pg_attrdef entry. + */ +ObjectAddress +GetAttrDefaultColumnAddress(Oid attrdefoid) +{ + ObjectAddress result = InvalidObjectAddress; + ScanKeyData skey[1]; + HeapTuple tup; + + Relation attrdef = table_open(AttrDefaultRelationId, AccessShareLock); + ScanKeyInit(&skey[0], + Anum_pg_attrdef_oid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(attrdefoid)); + SysScanDesc scan = systable_beginscan(attrdef, AttrDefaultOidIndexId, true, + NULL, 1, skey); + + if (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_attrdef atdform = (Form_pg_attrdef) GETSTRUCT(tup); + + result.classId = RelationRelationId; + result.objectId = atdform->adrelid; + result.objectSubId = atdform->adnum; + } + + systable_endscan(scan); + table_close(attrdef, AccessShareLock); + + return result; +} + + +#endif + + +/* + * GetAttrDefsFromSequence returns a list of attrdef OIDs that have + * a dependency on the given sequence + */ +List * +GetAttrDefsFromSequence(Oid seqOid) +{ + List *attrDefsResult = NIL; + ScanKeyData key[2]; + HeapTuple tup; + + Relation depRel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(seqOid)); + SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, lengthof(key), key); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); + + if (deprec->classid == AttrDefaultRelationId && + deprec->deptype == DEPENDENCY_NORMAL) + { + attrDefsResult = lappend_oid(attrDefsResult, deprec->objid); + } + } + + systable_endscan(scan); + + table_close(depRel, AccessShareLock); + + return attrDefsResult; +} + + /* * GetDependentFunctionsWithRelation returns the dependent functions for the * given relation id. diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index 4e190ec39..01929b0c5 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -128,6 +128,10 @@ extern List * IdentitySequenceDependencyCommandList(Oid targetRelationId); extern List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName); extern List * GetSequencesFromAttrDef(Oid attrdefOid); +#if PG_VERSION_NUM < PG_VERSION_15 +ObjectAddress GetAttrDefaultColumnAddress(Oid attrdefoid); +#endif +extern List * GetAttrDefsFromSequence(Oid seqOid); extern void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, AttrNumber attnum, char depType); extern List * GetDependentFunctionsWithRelation(Oid relationId); From d9069b1e01133c536c91052aa451e3dd7d875ef4 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Mon, 15 Apr 2024 14:01:55 +0200 Subject: [PATCH 018/155] Speed up SequenceUsedInDistributedTable (#7579) DESCRIPTION: Fix performance issue when creating distributed tables if many already exist This builds on the work to speed up EnsureSequenceTypeSupported, and now does something similar for SequenceUsedInDistributedTable. SequenceUsedInDistributedTable had a similar O(number of citus tables) operation. This fixes that and speeds up creation of distributed tables significantly when many distributed tables already exist. Fixes #7022 (cherry picked from commit cdf51da45842a41304a6a2ce9d878f21c8ec0782) --- src/backend/distributed/commands/sequence.c | 21 ++---- .../distributed/metadata/metadata_sync.c | 68 +++++++++++++++++++ src/include/distributed/metadata_sync.h | 1 + 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index 4d838a882..cfb55faf7 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -14,6 +14,7 @@ #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" +#include "catalog/pg_attrdef.h" #include "commands/defrem.h" #include "commands/extension.h" #include "nodes/makefuncs.h" @@ -507,22 +508,14 @@ PreprocessAlterSequenceStmt(Node *node, const char *queryString, static Oid SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char depType) { - List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); - Oid citusTableId = InvalidOid; - foreach_oid(citusTableId, citusTableIdList) + Oid relationId; + List *relations = GetDependentRelationsWithSequence(sequenceAddress->objectId, + depType); + foreach_oid(relationId, relations) { - List *seqInfoList = NIL; - GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0, depType); - SequenceInfo *seqInfo = NULL; - foreach_ptr(seqInfo, seqInfoList) + if (IsCitusTable(relationId)) { - /* - * This sequence is used in a distributed table - */ - if (seqInfo->sequenceOid == sequenceAddress->objectId) - { - return citusTableId; - } + return relationId; } } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 511cc45a9..9407573b4 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1626,6 +1626,74 @@ GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, } +/* + * GetDependentDependentRelationsWithSequence returns a list of oids of + * relations that have have a dependency on the given sequence. + * There are three types of dependencies: + * 1. direct auto (owned sequences), created using SERIAL or BIGSERIAL + * 2. indirect auto (through an AttrDef), created using DEFAULT nextval('..') + * 3. internal, created using GENERATED ALWAYS AS IDENTITY + * + * Depending on the passed deptype, we return the relations that have the + * given type(s): + * - DEPENDENCY_AUTO returns both 1 and 2 + * - DEPENDENCY_INTERNAL returns 3 + * + * The returned list can contain duplicates, as the same relation can have + * multiple dependencies on the sequence. + */ +List * +GetDependentRelationsWithSequence(Oid sequenceOid, char depType) +{ + List *relations = NIL; + ScanKeyData key[2]; + HeapTuple tup; + + Relation depRel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(sequenceOid)); + SysScanDesc scan = systable_beginscan(depRel, DependDependerIndexId, true, + NULL, lengthof(key), key); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); + + if ( + deprec->refclassid == RelationRelationId && + deprec->refobjsubid != 0 && + deprec->deptype == depType) + { + relations = lappend_oid(relations, deprec->refobjid); + } + } + + systable_endscan(scan); + + table_close(depRel, AccessShareLock); + + if (depType == DEPENDENCY_AUTO) + { + Oid attrDefOid; + List *attrDefOids = GetAttrDefsFromSequence(sequenceOid); + + foreach_oid(attrDefOid, attrDefOids) + { + ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); + relations = lappend_oid(relations, columnAddress.objectId); + } + } + + return relations; +} + + /* * GetSequencesFromAttrDef returns a list of sequence OIDs that have * dependency with the given attrdefOid in pg_depend diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index 01929b0c5..56b34bc21 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -134,6 +134,7 @@ ObjectAddress GetAttrDefaultColumnAddress(Oid attrdefoid); extern List * GetAttrDefsFromSequence(Oid seqOid); extern void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, AttrNumber attnum, char depType); +extern List * GetDependentRelationsWithSequence(Oid seqId, char depType); extern List * GetDependentFunctionsWithRelation(Oid relationId); extern Oid GetAttributeTypeOid(Oid relationId, AttrNumber attnum); extern void SetLocalEnableMetadataSync(bool state); From 62c32067f13fd27910590dfd867784555808a949 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Mon, 15 Apr 2024 14:42:56 +0200 Subject: [PATCH 019/155] Use an index to get FDWs that depend on extensions (#7574) DESCRIPTION: Fix performance issue when distributing a table that depends on an extension When the database contains many objects this function would show up in profiles because it was doing a sequence scan on pg_depend. And with many objects pg_depend can get very large. This starts using an index scan to only look for rows containing FDWs, of which there are expected to be very few (often even zero). (cherry picked from commit 16604a6601d8445c334d44b32ec6cf825c61839f) --- src/backend/distributed/commands/extension.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/backend/distributed/commands/extension.c b/src/backend/distributed/commands/extension.c index 36267ff66..9c3f830ff 100644 --- a/src/backend/distributed/commands/extension.c +++ b/src/backend/distributed/commands/extension.c @@ -1093,33 +1093,26 @@ List * GetDependentFDWsToExtension(Oid extensionId) { List *extensionFDWs = NIL; - ScanKeyData key[3]; - int scanKeyCount = 3; + ScanKeyData key[1]; HeapTuple tup; Relation pgDepend = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], - Anum_pg_depend_refclassid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(ExtensionRelationId)); - ScanKeyInit(&key[1], - Anum_pg_depend_refobjid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(extensionId)); - ScanKeyInit(&key[2], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ForeignDataWrapperRelationId)); - SysScanDesc scan = systable_beginscan(pgDepend, InvalidOid, false, - NULL, scanKeyCount, key); + SysScanDesc scan = systable_beginscan(pgDepend, DependDependerIndexId, true, + NULL, lengthof(key), key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend pgDependEntry = (Form_pg_depend) GETSTRUCT(tup); - if (pgDependEntry->deptype == DEPENDENCY_EXTENSION) + if (pgDependEntry->deptype == DEPENDENCY_EXTENSION && + pgDependEntry->refclassid == ExtensionRelationId && + pgDependEntry->refobjid == extensionId) { extensionFDWs = lappend_oid(extensionFDWs, pgDependEntry->objid); } From 364e8ece1474cd2e3ef22b18c28afdb74c1365af Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 16 Apr 2024 10:16:40 +0200 Subject: [PATCH 020/155] Speed up GetForeignKeyOids (#7578) DESCRIPTION: Fix performance issue in GetForeignKeyOids on systems with many constraints GetForeignKeyOids was showing up in CPU profiles when distributing schemas on systems with 100k+ constraints. The reason was that this function was doing a sequence scan of pg_constraint to get the foreign keys that referenced the requested table. This fixes that by finding the constraints referencing the table through pg_depend instead of pg_constraint. We're doing this indirection, because pg_constraint doesn't have an index that we can use, but pg_depend does. (cherry picked from commit a263ac6f5f6cdc64d2ed49fa911e3507c73facfe) --- .../distributed/commands/foreign_constraint.c | 187 ++++++++++++------ 1 file changed, 122 insertions(+), 65 deletions(-) diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index c1f2b83b6..2f60c3fb1 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -20,6 +20,7 @@ #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" #include "catalog/pg_type.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -36,6 +37,7 @@ #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/coordinator_protocol.h" +#include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/multi_join_order.h" #include "distributed/namespace_utils.h" @@ -1198,6 +1200,114 @@ TableHasExternalForeignKeys(Oid relationId) } +/* + * ForeignConstraintMatchesFlags is a function with logic that's very specific + * to GetForeignKeyOids. There's no reason to use it in any other context. + */ +static bool +ForeignConstraintMatchesFlags(Form_pg_constraint constraintForm, + int flags) +{ + if (constraintForm->contype != CONSTRAINT_FOREIGN) + { + return false; + } + + bool inheritedConstraint = OidIsValid(constraintForm->conparentid); + if (inheritedConstraint) + { + /* + * We only consider the constraints that are explicitly created on + * the table as we already process the constraints from parent tables + * implicitly when a command is issued + */ + return false; + } + + bool excludeSelfReference = (flags & EXCLUDE_SELF_REFERENCES); + bool isSelfReference = (constraintForm->conrelid == constraintForm->confrelid); + if (excludeSelfReference && isSelfReference) + { + return false; + } + + Oid otherTableId = InvalidOid; + if (flags & INCLUDE_REFERENCING_CONSTRAINTS) + { + otherTableId = constraintForm->confrelid; + } + else + { + otherTableId = constraintForm->conrelid; + } + + return IsTableTypeIncluded(otherTableId, flags); +} + + +/* + * GetForeignKeyOidsForReferencedTable returns a list of foreign key OIDs that + * reference the relationId and match the given flags. + * + * This is separated from GetForeignKeyOids because we need to scan pg_depend + * instead of pg_constraint directly. The reason for this is that there is no + * index on the confrelid of pg_constraint, so searching by that column + * requires a seqscan. + */ +static List * +GetForeignKeyOidsForReferencedTable(Oid relationId, int flags) +{ + HTAB *foreignKeyOidsSet = CreateSimpleHashSetWithName( + Oid, "ReferencingForeignKeyOidsSet"); + List *foreignKeyOidsList = NIL; + ScanKeyData key[2]; + HeapTuple dependTup; + Relation depRel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relationId)); + SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, lengthof(key), key); + while (HeapTupleIsValid(dependTup = systable_getnext(scan))) + { + Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(dependTup); + + if (deprec->classid != ConstraintRelationId || + deprec->deptype != DEPENDENCY_NORMAL || + hash_search(foreignKeyOidsSet, &deprec->objid, HASH_FIND, NULL)) + { + continue; + } + + + HeapTuple constraintTup = SearchSysCache1(CONSTROID, ObjectIdGetDatum( + deprec->objid)); + if (!HeapTupleIsValid(constraintTup)) /* can happen during DROP TABLE */ + { + continue; + } + + Form_pg_constraint constraint = (Form_pg_constraint) GETSTRUCT(constraintTup); + if (constraint->confrelid == relationId && + ForeignConstraintMatchesFlags(constraint, flags)) + { + foreignKeyOidsList = lappend_oid(foreignKeyOidsList, constraint->oid); + hash_search(foreignKeyOidsSet, &constraint->oid, HASH_ENTER, NULL); + } + ReleaseSysCache(constraintTup); + } + systable_endscan(scan); + table_close(depRel, AccessShareLock); + return foreignKeyOidsList; +} + + /* * GetForeignKeyOids takes in a relationId, and returns a list of OIDs for * foreign constraints that the relation with relationId is involved according @@ -1207,9 +1317,8 @@ TableHasExternalForeignKeys(Oid relationId) List * GetForeignKeyOids(Oid relationId, int flags) { - AttrNumber pgConstraintTargetAttrNumber = InvalidAttrNumber; - - bool extractReferencing = (flags & INCLUDE_REFERENCING_CONSTRAINTS); + bool extractReferencing PG_USED_FOR_ASSERTS_ONLY = (flags & + INCLUDE_REFERENCING_CONSTRAINTS); bool extractReferenced = (flags & INCLUDE_REFERENCED_CONSTRAINTS); /* @@ -1220,22 +1329,10 @@ GetForeignKeyOids(Oid relationId, int flags) Assert(!(extractReferencing && extractReferenced)); Assert(extractReferencing || extractReferenced); - bool useIndex = false; - Oid indexOid = InvalidOid; - - if (extractReferencing) + if (extractReferenced) { - pgConstraintTargetAttrNumber = Anum_pg_constraint_conrelid; - - useIndex = true; - indexOid = ConstraintRelidTypidNameIndexId; + return GetForeignKeyOidsForReferencedTable(relationId, flags); } - else if (extractReferenced) - { - pgConstraintTargetAttrNumber = Anum_pg_constraint_confrelid; - } - - bool excludeSelfReference = (flags & EXCLUDE_SELF_REFERENCES); List *foreignKeyOids = NIL; @@ -1243,62 +1340,22 @@ GetForeignKeyOids(Oid relationId, int flags) int scanKeyCount = 1; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); - ScanKeyInit(&scanKey[0], pgConstraintTargetAttrNumber, + ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); - SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, indexOid, useIndex, + + SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, + ConstraintRelidTypidNameIndexId, true, NULL, scanKeyCount, scanKey); - HeapTuple heapTuple = systable_getnext(scanDescriptor); - while (HeapTupleIsValid(heapTuple)) + HeapTuple heapTuple; + while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); - if (constraintForm->contype != CONSTRAINT_FOREIGN) + if (ForeignConstraintMatchesFlags(constraintForm, flags)) { - heapTuple = systable_getnext(scanDescriptor); - continue; + foreignKeyOids = lappend_oid(foreignKeyOids, constraintForm->oid); } - - bool inheritedConstraint = OidIsValid(constraintForm->conparentid); - if (inheritedConstraint) - { - /* - * We only consider the constraints that are explicitly created on - * the table as we already process the constraints from parent tables - * implicitly when a command is issued - */ - heapTuple = systable_getnext(scanDescriptor); - continue; - } - - Oid constraintId = constraintForm->oid; - - bool isSelfReference = (constraintForm->conrelid == constraintForm->confrelid); - if (excludeSelfReference && isSelfReference) - { - heapTuple = systable_getnext(scanDescriptor); - continue; - } - - Oid otherTableId = InvalidOid; - if (extractReferencing) - { - otherTableId = constraintForm->confrelid; - } - else if (extractReferenced) - { - otherTableId = constraintForm->conrelid; - } - - if (!IsTableTypeIncluded(otherTableId, flags)) - { - heapTuple = systable_getnext(scanDescriptor); - continue; - } - - foreignKeyOids = lappend_oid(foreignKeyOids, constraintId); - - heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); From 79616bc7dbc804931f2f06dbc084143c228c6125 Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Wed, 10 Apr 2024 19:08:54 +0300 Subject: [PATCH 021/155] Use expecteddir option when running vanilla tests (#7573) In PostgreSQL 16 a new option expecteddir was introduced to pg_regress. Together with fix in [196eeb6b](https://github.com/postgres/postgres/commit/196eeb6b) it causes check-vanilla failure if expecteddir is not specified. Co-authored-by: Karina Litskevich (cherry picked from commit 41d99249d9a09a00be8e492030f42a39feb28875) --- src/test/regress/pg_regress_multi.pl | 37 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 4cc022198..4e7c66d9e 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -1120,16 +1120,33 @@ sub RunVanillaTests system("mkdir", ("-p", "$pgregressOutputdir/sql")) == 0 or die "Could not create vanilla sql dir."; - $exitcode = system("$plainRegress", - ("--dlpath", $dlpath), - ("--inputdir", $pgregressInputdir), - ("--outputdir", $pgregressOutputdir), - ("--schedule", catfile("$pgregressInputdir", "parallel_schedule")), - ("--use-existing"), - ("--host","$host"), - ("--port","$masterPort"), - ("--user","$user"), - ("--dbname", "$dbName")); + if ($majorversion >= "16") + { + $exitcode = system("$plainRegress", + ("--dlpath", $dlpath), + ("--inputdir", $pgregressInputdir), + ("--outputdir", $pgregressOutputdir), + ("--expecteddir", $pgregressOutputdir), + ("--schedule", catfile("$pgregressInputdir", "parallel_schedule")), + ("--use-existing"), + ("--host","$host"), + ("--port","$masterPort"), + ("--user","$user"), + ("--dbname", "$dbName")); + } + else + { + $exitcode = system("$plainRegress", + ("--dlpath", $dlpath), + ("--inputdir", $pgregressInputdir), + ("--outputdir", $pgregressOutputdir), + ("--schedule", catfile("$pgregressInputdir", "parallel_schedule")), + ("--use-existing"), + ("--host","$host"), + ("--port","$masterPort"), + ("--user","$user"), + ("--dbname", "$dbName")); + } } if ($useMitmproxy) { From 82637f3e13bc187c06f8bea04eb685ad74b118ce Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:44:47 +0300 Subject: [PATCH 022/155] Use expecteddir option in _run_pg_regress() (#7582) Fix check-arbitrary-configs tests failure with current REL_16_STABLE. This is the same problem as described in #7573. I missed pg_regress call in _run_pg_regress() in that PR. Co-authored-by: Karina Litskevich (cherry picked from commit 41e2af8ff5d86c9b327b96cd57f08d26777b7d13) --- src/test/regress/citus_tests/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 907102482..4d04d268c 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -294,6 +294,9 @@ def _run_pg_regress( output_dir, "--use-existing", ] + if PG_MAJOR_VERSION >= 16: + command.append("--expecteddir") + command.append(output_dir) if extra_tests != "": command.append(extra_tests) From f2d102d54b5759b8c813d8962de3538c9ef3171a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emel=20=C5=9Eim=C5=9Fek?= Date: Wed, 20 Mar 2024 11:06:05 +0300 Subject: [PATCH 023/155] Fix crash caused by some form of ALTER TABLE ADD COLUMN statements. (#7522) DESCRIPTION: Fixes a crash caused by some form of ALTER TABLE ADD COLUMN statements. When adding multiple columns, if one of the ADD COLUMN statements contains a FOREIGN constraint ommitting the referenced columns in the statement, a SEGFAULT occurs. For instance, the following statement results in a crash: ``` ALTER TABLE lt ADD COLUMN new_col1 bool, ADD COLUMN new_col2 int references rt; ``` Fixes #7520. (cherry picked from commit fdd658acecced724f275429094d4d381c1b9fe4b) --- src/backend/distributed/commands/table.c | 10 +++++++--- src/backend/distributed/deparser/deparse_table_stmts.c | 2 +- src/include/distributed/deparser.h | 2 ++ src/test/regress/expected/alter_table_add_column.out | 9 +++++++++ src/test/regress/sql/alter_table_add_column.sql | 4 ++++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 074a789ed..30b028b79 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -3053,11 +3053,15 @@ ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *command, else if (constraint->contype == CONSTR_FOREIGN) { RangeVar *referencedTable = constraint->pktable; - char *referencedColumn = strVal(lfirst(list_head(constraint->pk_attrs))); Oid referencedRelationId = RangeVarGetRelid(referencedTable, NoLock, false); - appendStringInfo(errHint, "FOREIGN KEY (%s) REFERENCES %s(%s)", colName, - get_rel_name(referencedRelationId), referencedColumn); + appendStringInfo(errHint, "FOREIGN KEY (%s) REFERENCES %s", colName, + get_rel_name(referencedRelationId)); + + if (list_length(constraint->pk_attrs) > 0) + { + AppendColumnNameList(errHint, constraint->pk_attrs); + } if (constraint->fk_del_action == FKCONSTR_ACTION_SETNULL) { diff --git a/src/backend/distributed/deparser/deparse_table_stmts.c b/src/backend/distributed/deparser/deparse_table_stmts.c index e976b0e2f..5d184fa66 100644 --- a/src/backend/distributed/deparser/deparse_table_stmts.c +++ b/src/backend/distributed/deparser/deparse_table_stmts.c @@ -121,7 +121,7 @@ AppendAlterTableStmt(StringInfo buf, AlterTableStmt *stmt) * AppendColumnNameList converts a list of columns into comma separated string format * (colname_1, colname_2, .., colname_n). */ -static void +void AppendColumnNameList(StringInfo buf, List *columns) { appendStringInfoString(buf, " ("); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 33bc8ac62..04a96e4be 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -121,6 +121,8 @@ extern void AppendGrantedByInGrant(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedPrefix(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedSuffix(StringInfo buf, GrantStmt *stmt); +extern void AppendColumnNameList(StringInfo buf, List *columns); + /* forward declarations for deparse_statistics_stmts.c */ extern char * DeparseCreateStatisticsStmt(Node *node); diff --git a/src/test/regress/expected/alter_table_add_column.out b/src/test/regress/expected/alter_table_add_column.out index 61e7319d9..0408aeeab 100644 --- a/src/test/regress/expected/alter_table_add_column.out +++ b/src/test/regress/expected/alter_table_add_column.out @@ -44,6 +44,15 @@ ERROR: cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and DETAIL: Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names HINT: You can issue each command separately such as ALTER TABLE referencing ADD COLUMN test_8 data_type; ALTER TABLE referencing ADD CONSTRAINT constraint_name CHECK (check_expression); ALTER TABLE referencing ADD COLUMN test_8 integer CONSTRAINT check_test_8 CHECK (test_8 > 0); +-- error out properly even if the REFERENCES does not include the column list of the referenced table +ALTER TABLE referencing ADD COLUMN test_9 bool, ADD COLUMN test_10 int REFERENCES referenced; +ERROR: cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints +DETAIL: Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names +HINT: You can issue each command separately such as ALTER TABLE referencing ADD COLUMN test_10 data_type; ALTER TABLE referencing ADD CONSTRAINT constraint_name FOREIGN KEY (test_10) REFERENCES referenced; +ALTER TABLE referencing ADD COLUMN test_9 bool, ADD COLUMN test_10 int REFERENCES referenced(int_col); +ERROR: cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints +DETAIL: Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names +HINT: You can issue each command separately such as ALTER TABLE referencing ADD COLUMN test_10 data_type; ALTER TABLE referencing ADD CONSTRAINT constraint_name FOREIGN KEY (test_10) REFERENCES referenced (int_col ); -- try to add test_6 again, but with IF NOT EXISTS ALTER TABLE referencing ADD COLUMN IF NOT EXISTS test_6 text; NOTICE: column "test_6" of relation "referencing" already exists, skipping diff --git a/src/test/regress/sql/alter_table_add_column.sql b/src/test/regress/sql/alter_table_add_column.sql index 255e7714f..355667842 100644 --- a/src/test/regress/sql/alter_table_add_column.sql +++ b/src/test/regress/sql/alter_table_add_column.sql @@ -41,6 +41,10 @@ ALTER TABLE referencing ADD COLUMN "test_\'!7" "simple_!\'custom_type"; ALTER TABLE referencing ADD COLUMN test_8 integer CHECK (test_8 > 0); ALTER TABLE referencing ADD COLUMN test_8 integer CONSTRAINT check_test_8 CHECK (test_8 > 0); +-- error out properly even if the REFERENCES does not include the column list of the referenced table +ALTER TABLE referencing ADD COLUMN test_9 bool, ADD COLUMN test_10 int REFERENCES referenced; +ALTER TABLE referencing ADD COLUMN test_9 bool, ADD COLUMN test_10 int REFERENCES referenced(int_col); + -- try to add test_6 again, but with IF NOT EXISTS ALTER TABLE referencing ADD COLUMN IF NOT EXISTS test_6 text; ALTER TABLE referencing ADD COLUMN IF NOT EXISTS test_6 integer; From 2ee43fd00c6adc5cc35181e53c3c1dd4ad3d50b6 Mon Sep 17 00:00:00 2001 From: copetol <40788226+copetol@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:21:42 +0300 Subject: [PATCH 024/155] Fix segfault when using certain DO block in function (#7554) When using a CASE WHEN expression in the body of the function that is used in the DO block, a segmentation fault occured. This fixes that. Fixes #7381 --------- Co-authored-by: Konstantin Morozov (cherry picked from commit 12f56438fc85a676a79b1a2886cc69ab872c9d14) --- .../planner/function_call_delegation.c | 4 +++ .../expected/function_with_case_when.out | 32 +++++++++++++++++++ src/test/regress/multi_schedule | 1 + .../regress/sql/function_with_case_when.sql | 27 ++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 src/test/regress/expected/function_with_case_when.out create mode 100644 src/test/regress/sql/function_with_case_when.sql diff --git a/src/backend/distributed/planner/function_call_delegation.c b/src/backend/distributed/planner/function_call_delegation.c index c72982e2a..43f7d8f7f 100644 --- a/src/backend/distributed/planner/function_call_delegation.c +++ b/src/backend/distributed/planner/function_call_delegation.c @@ -91,6 +91,10 @@ bool InDelegatedFunctionCall = false; static bool contain_param_walker(Node *node, void *context) { + if (node == NULL) + { + return false; + } if (IsA(node, Param)) { Param *paramNode = (Param *) node; diff --git a/src/test/regress/expected/function_with_case_when.out b/src/test/regress/expected/function_with_case_when.out new file mode 100644 index 000000000..18df5be0a --- /dev/null +++ b/src/test/regress/expected/function_with_case_when.out @@ -0,0 +1,32 @@ +CREATE SCHEMA function_with_case; +SET search_path TO function_with_case; +-- create function +CREATE OR REPLACE FUNCTION test_err(v1 text) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ + +begin + return v1 || ' - ok'; +END; +$function$; +do $$ declare + lNewValues text; + val text; +begin + val = 'test'; + lNewValues = test_err(v1 => case when val::text = 'test'::text then 'yes' else 'no' end); + raise notice 'lNewValues= %', lNewValues; +end;$$ ; +NOTICE: lNewValues= yes - ok +CONTEXT: PL/pgSQL function inline_code_block line XX at RAISE +-- call function +SELECT test_err('test'); + test_err +--------------------------------------------------------------------- + test - ok +(1 row) + +DROP SCHEMA function_with_case CASCADE; +NOTICE: drop cascades to function test_err(text) diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 65a272566..2d515e7a2 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -107,6 +107,7 @@ test: object_propagation_debug test: undistribute_table test: run_command_on_all_nodes test: background_task_queue_monitor +test: function_with_case_when # Causal clock test test: clock diff --git a/src/test/regress/sql/function_with_case_when.sql b/src/test/regress/sql/function_with_case_when.sql new file mode 100644 index 000000000..03c6678e4 --- /dev/null +++ b/src/test/regress/sql/function_with_case_when.sql @@ -0,0 +1,27 @@ +CREATE SCHEMA function_with_case; +SET search_path TO function_with_case; + +-- create function +CREATE OR REPLACE FUNCTION test_err(v1 text) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ + +begin + return v1 || ' - ok'; +END; +$function$; +do $$ declare + lNewValues text; + val text; +begin + val = 'test'; + lNewValues = test_err(v1 => case when val::text = 'test'::text then 'yes' else 'no' end); + raise notice 'lNewValues= %', lNewValues; +end;$$ ; + +-- call function +SELECT test_err('test'); + +DROP SCHEMA function_with_case CASCADE; From 9b06d02c43be83f97565f081c7cd36fad3c794b9 Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Wed, 21 Feb 2024 14:35:27 +0300 Subject: [PATCH 025/155] Fix error in master_disable_node/citus_disable_node (#7492) This fixes #7454: master_disable_node() has only two arguments, but calls citus_disable_node() that tries to read three arguments Co-authored-by: Karina Litskevich (cherry picked from commit 683e10ab69d42c752212c4c5080431abb4c35306) --- src/backend/distributed/metadata/node_metadata.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c index de3dbcbdf..bc6f89959 100644 --- a/src/backend/distributed/metadata/node_metadata.c +++ b/src/backend/distributed/metadata/node_metadata.c @@ -507,7 +507,13 @@ citus_disable_node(PG_FUNCTION_ARGS) { text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); - bool synchronousDisableNode = PG_GETARG_BOOL(2); + + bool synchronousDisableNode = 1; + Assert(PG_NARGS() == 2 || PG_NARGS() == 3); + if (PG_NARGS() == 3) + { + synchronousDisableNode = PG_GETARG_BOOL(2); + } char *nodeName = text_to_cstring(nodeNameText); WorkerNode *workerNode = ModifiableWorkerNode(nodeName, nodePort); From 452564c19b1291f8f21052013828beb28b6885d8 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 15 Apr 2024 12:51:11 +0300 Subject: [PATCH 026/155] Allow providing "host" parameter via citus.node_conninfo (#7541) And when that is the case, directly use it as "host" parameter for the connections between nodes and use the "hostname" provided in pg_dist_node / pg_dist_poolinfo as "hostaddr" to avoid host name lookup. This is to avoid allowing dns resolution (and / or setting up DNS names for each host in the cluster). This already works currently when using IPs in the hostname. The only use of setting host is that you can then use sslmode=verify-full and it will validate that the hostname matches the certificate provided by the node you're connecting too. It would be more flexible to make this a per-node setting, but that requires SQL changes. And we'd like to backport this change, and backporting such a sql change would be quite hard while backporting this change would be very easy. And in many setups, a different hostname for TLS validation is actually not needed. The reason for that is query-from-any node: With query-from-any-node all nodes usually have a certificate that is valid for the same "cluster hostname", either using a wildcard cert or a Subject Alternative Name (SAN). Because if you load balance across nodes you don't know which node you're connecting to, but you still want TLS validation to do it's job. So with this change you can use this same "cluster hostname" for TLS validation within the cluster. Obviously this means you don't validate that you're connecting to a particular node, just that you're connecting to one of the nodes in the cluster, but that should be fine from a security perspective (in most cases). Note to self: This change requires updating https://docs.citusdata.com/en/latest/develop/api_guc.html#citus-node-conninfo-text. DESCRIPTION: Allows overwriting host name for all inter-node connections by supporting "host" parameter in citus.node_conninfo (cherry picked from commit 3586aab17a7ff2fef8f336b6528376552a5d4c2c) --- .../connection/connection_configuration.c | 17 +++++- src/backend/distributed/shared_library_init.c | 1 + .../regress/expected/node_conninfo_reload.out | 56 +++++++++++++++++++ src/test/regress/sql/node_conninfo_reload.sql | 26 +++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/connection/connection_configuration.c b/src/backend/distributed/connection/connection_configuration.c index c6a34a9d7..64b8029da 100644 --- a/src/backend/distributed/connection/connection_configuration.c +++ b/src/backend/distributed/connection/connection_configuration.c @@ -267,9 +267,24 @@ GetConnParams(ConnectionHashKey *key, char ***keywords, char ***values, * We allocate everything in the provided context so as to facilitate using * pfree on all runtime parameters when connections using these entries are * invalidated during config reloads. + * + * Also, when "host" is already provided in global parameters, we use hostname + * from the key as "hostaddr" instead of "host" to avoid host name lookup. In + * that case, the value for "host" becomes useful only if the authentication + * method requires it. */ + bool gotHostParamFromGlobalParams = false; + for (Size paramIndex = 0; paramIndex < ConnParams.size; paramIndex++) + { + if (strcmp(ConnParams.keywords[paramIndex], "host") == 0) + { + gotHostParamFromGlobalParams = true; + break; + } + } + const char *runtimeKeywords[] = { - "host", + gotHostParamFromGlobalParams ? "hostaddr" : "host", "port", "dbname", "user", diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 898c276bf..ada4399a2 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -2892,6 +2892,7 @@ NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source) #if defined(ENABLE_GSS) && defined(ENABLE_SSPI) "gsslib", #endif + "host", "keepalives", "keepalives_count", "keepalives_idle", diff --git a/src/test/regress/expected/node_conninfo_reload.out b/src/test/regress/expected/node_conninfo_reload.out index d2e33d950..01fb5c878 100644 --- a/src/test/regress/expected/node_conninfo_reload.out +++ b/src/test/regress/expected/node_conninfo_reload.out @@ -520,5 +520,61 @@ show citus.node_conninfo; -- Should work again ALTER TABLE test ADD COLUMN e INT; +-- show that we allow providing "host" param via citus.node_conninfo +ALTER SYSTEM SET citus.node_conninfo = 'sslmode=require host=nosuchhost'; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + +-- fails due to invalid host +SELECT COUNT(*)>=0 FROM test; +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: could not parse network address "localhost": Name or service not known +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: could not parse network address "localhost": Name or service not known +SELECT array_agg(nodeid) as updated_nodeids from pg_dist_node WHERE nodename = 'localhost' \gset +UPDATE pg_dist_node SET nodename = '127.0.0.1' WHERE nodeid = ANY(:'updated_nodeids'::int[]); +ALTER SYSTEM SET citus.node_conninfo = 'sslmode=require host=localhost'; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + +-- works when hostaddr is specified in pg_dist_node after providing host in citus.node_conninfo +SELECT COUNT(*)>=0 FROM test; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +-- restore original nodenames into pg_dist_node +UPDATE pg_dist_node SET nodename = 'localhost' WHERE nodeid = ANY(:'updated_nodeids'::int[]); +-- reset it +ALTER SYSTEM RESET citus.node_conninfo; +select pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +select pg_sleep(0.1); -- wait for config reload to apply + pg_sleep +--------------------------------------------------------------------- + +(1 row) + DROP SCHEMA node_conninfo_reload CASCADE; NOTICE: drop cascades to table test diff --git a/src/test/regress/sql/node_conninfo_reload.sql b/src/test/regress/sql/node_conninfo_reload.sql index 42ba8c9b1..2faaaeeb1 100644 --- a/src/test/regress/sql/node_conninfo_reload.sql +++ b/src/test/regress/sql/node_conninfo_reload.sql @@ -205,4 +205,30 @@ show citus.node_conninfo; -- Should work again ALTER TABLE test ADD COLUMN e INT; +-- show that we allow providing "host" param via citus.node_conninfo +ALTER SYSTEM SET citus.node_conninfo = 'sslmode=require host=nosuchhost'; +SELECT pg_reload_conf(); +SELECT pg_sleep(0.1); + +-- fails due to invalid host +SELECT COUNT(*)>=0 FROM test; + +SELECT array_agg(nodeid) as updated_nodeids from pg_dist_node WHERE nodename = 'localhost' \gset +UPDATE pg_dist_node SET nodename = '127.0.0.1' WHERE nodeid = ANY(:'updated_nodeids'::int[]); + +ALTER SYSTEM SET citus.node_conninfo = 'sslmode=require host=localhost'; +SELECT pg_reload_conf(); +SELECT pg_sleep(0.1); + +-- works when hostaddr is specified in pg_dist_node after providing host in citus.node_conninfo +SELECT COUNT(*)>=0 FROM test; + +-- restore original nodenames into pg_dist_node +UPDATE pg_dist_node SET nodename = 'localhost' WHERE nodeid = ANY(:'updated_nodeids'::int[]); + +-- reset it +ALTER SYSTEM RESET citus.node_conninfo; +select pg_reload_conf(); +select pg_sleep(0.1); -- wait for config reload to apply + DROP SCHEMA node_conninfo_reload CASCADE; From 812a2b759f3452e94b7d22c8ac52c82001b5888d Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Tue, 30 Jan 2024 18:12:48 +0300 Subject: [PATCH 027/155] Improve error message for recursive CTEs (#7407) Fixes #2870 (cherry picked from commit 5aedec42427443a822076d4a0938c85419212d57) --- .../distributed/planner/recursive_planning.c | 4 ++-- .../regress/expected/multi_mx_router_planner.out | 4 ++-- src/test/regress/expected/multi_router_planner.out | 4 ++-- src/test/regress/expected/pg14.out | 10 +++++----- .../regress/expected/query_single_shard_table.out | 2 +- src/test/regress/expected/subquery_and_cte.out | 2 +- src/test/regress/expected/with_basics.out | 14 +++++++------- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 6c42046ff..9f520fa5f 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1097,8 +1097,8 @@ RecursivelyPlanCTEs(Query *query, RecursivePlanningContext *planningContext) if (query->hasRecursive) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, - "recursive CTEs are not supported in distributed " - "queries", + "recursive CTEs are only supported when they " + "contain a filter on the distribution column", NULL, NULL); } diff --git a/src/test/regress/expected/multi_mx_router_planner.out b/src/test/regress/expected/multi_mx_router_planner.out index e7855a898..5ac6093cb 100644 --- a/src/test/regress/expected/multi_mx_router_planner.out +++ b/src/test/regress/expected/multi_mx_router_planner.out @@ -370,7 +370,7 @@ WITH RECURSIVE hierarchy as ( h.company_id = ce.company_id)) SELECT * FROM hierarchy WHERE LEVEL <= 2; DEBUG: Router planner cannot handle multi-shard select queries -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- logically wrong query, query involves different shards -- from the same table, but still router plannable due to -- shard being placed on the same worker. @@ -386,7 +386,7 @@ WITH RECURSIVE hierarchy as ( ce.company_id = 2)) SELECT * FROM hierarchy WHERE LEVEL <= 2; DEBUG: router planner does not support queries that reference non-colocated distributed tables -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- grouping sets are supported on single shard SELECT id, substring(title, 2, 1) AS subtitle, count(*) diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index c6d46ccc9..14f0bee83 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -436,7 +436,7 @@ WITH RECURSIVE hierarchy as MATERIALIZED ( h.company_id = ce.company_id)) SELECT * FROM hierarchy WHERE LEVEL <= 2; DEBUG: Router planner cannot handle multi-shard select queries -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- logically wrong query, query involves different shards -- from the same table WITH RECURSIVE hierarchy as MATERIALIZED ( @@ -451,7 +451,7 @@ WITH RECURSIVE hierarchy as MATERIALIZED ( ce.company_id = 2)) SELECT * FROM hierarchy WHERE LEVEL <= 2; DEBUG: router planner does not support queries that reference non-colocated distributed tables -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- Test router modifying CTEs WITH new_article AS MATERIALIZED( INSERT INTO articles_hash VALUES (1, 1, 'arsenous', 9) RETURNING * diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 1b1d80df2..8e8fc96f3 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -1168,7 +1168,7 @@ WITH RECURSIVE search_graph(f, t, label) AS ( WHERE g.f = sg.t and g.f = 1 ) SEARCH DEPTH FIRST BY f, t SET seq SELECT * FROM search_graph ORDER BY seq; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column WITH RECURSIVE search_graph(f, t, label) AS ( SELECT * FROM graph0 g WHERE f = 1 UNION ALL @@ -1177,7 +1177,7 @@ WITH RECURSIVE search_graph(f, t, label) AS ( WHERE g.f = sg.t and g.f = 1 ) SEARCH DEPTH FIRST BY f, t SET seq DELETE FROM graph0 WHERE t IN (SELECT t FROM search_graph ORDER BY seq); -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column CREATE TABLE graph1(f INT, t INT, label TEXT); SELECT create_reference_table('graph1'); create_reference_table @@ -1196,7 +1196,7 @@ WITH RECURSIVE search_graph(f, t, label) AS ( WHERE g.f = sg.t and g.f = 1 ) SEARCH DEPTH FIRST BY f, t SET seq SELECT * FROM search_graph ORDER BY seq; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column WITH RECURSIVE search_graph(f, t, label) AS ( SELECT * FROM graph1 g WHERE f = 1 UNION ALL @@ -1205,7 +1205,7 @@ WITH RECURSIVE search_graph(f, t, label) AS ( WHERE g.f = sg.t and g.f = 1 ) SEARCH DEPTH FIRST BY f, t SET seq DELETE FROM graph1 WHERE t IN (SELECT t FROM search_graph ORDER BY seq); -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column SELECT * FROM ( WITH RECURSIVE search_graph(f, t, label) AS ( SELECT * @@ -1217,7 +1217,7 @@ SELECT * FROM ( ) SEARCH DEPTH FIRST BY f, t SET seq SELECT * FROM search_graph ORDER BY seq ) as foo; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- -- https://github.com/citusdata/citus/issues/5258 -- diff --git a/src/test/regress/expected/query_single_shard_table.out b/src/test/regress/expected/query_single_shard_table.out index ad6037b65..5f551a988 100644 --- a/src/test/regress/expected/query_single_shard_table.out +++ b/src/test/regress/expected/query_single_shard_table.out @@ -1529,7 +1529,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Router planner cannot handle multi-shard select queries DEBUG: generating subplan XXX_1 for CTE level_1: WITH RECURSIVE level_2_recursive(x) AS (VALUES (1) UNION ALL SELECT (nullkey_c1_t1.a OPERATOR(pg_catalog.+) 1) FROM (query_single_shard_table.nullkey_c1_t1 JOIN level_2_recursive level_2_recursive_1 ON ((nullkey_c1_t1.a OPERATOR(pg_catalog.=) level_2_recursive_1.x))) WHERE (nullkey_c1_t1.a OPERATOR(pg_catalog.<) 100)) SELECT level_2_recursive.x, distributed_table.a, distributed_table.b FROM (level_2_recursive JOIN query_single_shard_table.distributed_table ON ((level_2_recursive.x OPERATOR(pg_catalog.=) distributed_table.a))) DEBUG: Router planner cannot handle multi-shard select queries -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- grouping set SELECT id, substring(title, 2, 1) AS subtitle, count(*) diff --git a/src/test/regress/expected/subquery_and_cte.out b/src/test/regress/expected/subquery_and_cte.out index c15e9b9d7..896860865 100644 --- a/src/test/regress/expected/subquery_and_cte.out +++ b/src/test/regress/expected/subquery_and_cte.out @@ -527,7 +527,7 @@ FROM ) as bar WHERE foo.user_id = bar.user_id ORDER BY 1 DESC; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column CREATE TABLE ref_table_1 (a int); SELECT create_reference_table('ref_table_1'); create_reference_table diff --git a/src/test/regress/expected/with_basics.out b/src/test/regress/expected/with_basics.out index 4eefb8837..78ed17317 100644 --- a/src/test/regress/expected/with_basics.out +++ b/src/test/regress/expected/with_basics.out @@ -664,14 +664,14 @@ WITH RECURSIVE basic_recursive(x) AS ( SELECT user_id + 1 FROM users_table JOIN basic_recursive ON (user_id = x) WHERE user_id < 100 ) SELECT sum(x) FROM basic_recursive; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column WITH RECURSIVE basic_recursive AS ( SELECT -1 as user_id, '2017-11-22 20:16:16.614779'::timestamp, -1, -1, -1, -1 UNION ALL SELECT basic_recursive.* FROM users_table JOIN basic_recursive USING (user_id) WHERE user_id>1 ) SELECT * FROM basic_recursive ORDER BY user_id LIMIT 1; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- basic_recursive in FROM should error out SELECT * @@ -682,7 +682,7 @@ FROM SELECT basic_recursive.* FROM users_table JOIN basic_recursive USING (user_id) WHERE user_id>1 ) SELECT * FROM basic_recursive ORDER BY user_id LIMIT 1) cte_rec; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- basic_recursive in WHERE with UNION ALL SELECT * @@ -696,7 +696,7 @@ WHERE SELECT basic_recursive.* FROM users_table JOIN basic_recursive USING (user_id) WHERE user_id>1 ) SELECT * FROM basic_recursive ORDER BY user_id LIMIT 1); -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- one recursive one regular CTE should error out WITH RECURSIVE basic_recursive(x) AS( VALUES (1) @@ -707,7 +707,7 @@ basic AS ( SELECT count(user_id) FROM users_table ) SELECT x FROM basic, basic_recursive; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- one recursive one regular which SELECTs from the recursive CTE under a simple SELECT WITH RECURSIVE basic_recursive(x) AS( VALUES (1) @@ -718,7 +718,7 @@ basic AS ( SELECT count(x) FROM basic_recursive ) SELECT * FROM basic; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- recursive CTE in a NESTED manner WITH regular_cte AS ( WITH regular_2 AS ( @@ -732,7 +732,7 @@ WITH regular_cte AS ( SELECT * FROM regular_2 ) SELECT * FROM regular_cte; -ERROR: recursive CTEs are not supported in distributed queries +ERROR: recursive CTEs are only supported when they contain a filter on the distribution column -- CTEs should work with VIEWs as well CREATE VIEW basic_view AS SELECT * FROM users_table; From 94ab1dc2404e15926f202302823339eaaa30cd06 Mon Sep 17 00:00:00 2001 From: Marco Slot Date: Tue, 23 Jan 2024 11:55:03 +0100 Subject: [PATCH 028/155] Replace spurious strdup with pstrdup (#7440) Not sure why we never found this using valgrind, but using strdup will cause memory leaks because the pointer is not tracked in a memory context. (cherry picked from commit 72fbea20c49c6de74f0c5005f77efce9f3a54178) --- src/backend/distributed/planner/query_colocation_checker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index 827a0286c..d9b1aad5a 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -433,7 +433,7 @@ CreateTargetEntryForColumn(Form_pg_attribute attributeTuple, Index rteIndex, attributeTuple->atttypmod, attributeTuple->attcollation, 0); TargetEntry *targetEntry = makeTargetEntry((Expr *) targetColumn, resno, - strdup(attributeTuple->attname.data), false); + pstrdup(attributeTuple->attname.data), false); return targetEntry; } From 146725fc9b76b40e3c840de415e4daf8f467f912 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 23 Jan 2024 13:28:26 +0100 Subject: [PATCH 029/155] Replace more spurious strdups with pstrdups (#7441) DESCRIPTION: Remove a few small memory leaks In #7440 one instance of a strdup was removed. But there were a few more. This removes the ones that are left over, or adds a comment why strdup is on purpose. (cherry picked from commit 9683bef2ecf624b54a42bbd94ed4f4c4e765aa7b) --- src/backend/distributed/commands/extension.c | 8 ++++---- .../distributed/connection/connection_configuration.c | 4 ++++ .../distributed/planner/query_colocation_checker.c | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/commands/extension.c b/src/backend/distributed/commands/extension.c index 9c3f830ff..8d4c6431b 100644 --- a/src/backend/distributed/commands/extension.c +++ b/src/backend/distributed/commands/extension.c @@ -776,7 +776,7 @@ PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree) /*create extension citus version xxx*/ if (newVersionValue) { - char *newVersion = strdup(defGetString(newVersionValue)); + char *newVersion = pstrdup(defGetString(newVersionValue)); versionNumber = GetExtensionVersionNumber(newVersion); } @@ -796,7 +796,7 @@ PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree) Oid citusOid = get_extension_oid("citus", true); if (citusOid != InvalidOid) { - char *curCitusVersion = strdup(get_extension_version(citusOid)); + char *curCitusVersion = pstrdup(get_extension_version(citusOid)); int curCitusVersionNum = GetExtensionVersionNumber(curCitusVersion); if (curCitusVersionNum < 1110) { @@ -891,7 +891,7 @@ PreprocessAlterExtensionCitusStmtForCitusColumnar(Node *parseTree) if (newVersionValue) { char *newVersion = defGetString(newVersionValue); - double newVersionNumber = GetExtensionVersionNumber(strdup(newVersion)); + double newVersionNumber = GetExtensionVersionNumber(pstrdup(newVersion)); /*alter extension citus update to version >= 11.1-1, and no citus_columnar installed */ if (newVersionNumber >= 1110 && citusColumnarOid == InvalidOid) @@ -935,7 +935,7 @@ PostprocessAlterExtensionCitusStmtForCitusColumnar(Node *parseTree) if (newVersionValue) { char *newVersion = defGetString(newVersionValue); - double newVersionNumber = GetExtensionVersionNumber(strdup(newVersion)); + double newVersionNumber = GetExtensionVersionNumber(pstrdup(newVersion)); if (newVersionNumber >= 1110 && citusColumnarOid != InvalidOid) { /*upgrade citus, after "ALTER EXTENSION citus update to xxx" updates citus_columnar Y to version Z. */ diff --git a/src/backend/distributed/connection/connection_configuration.c b/src/backend/distributed/connection/connection_configuration.c index 64b8029da..dd7700d8a 100644 --- a/src/backend/distributed/connection/connection_configuration.c +++ b/src/backend/distributed/connection/connection_configuration.c @@ -123,6 +123,10 @@ AddConnParam(const char *keyword, const char *value) errmsg("ConnParams arrays bound check failed"))); } + /* + * Don't use pstrdup here to avoid being tied to a memory context, we free + * these later using ResetConnParams + */ ConnParams.keywords[ConnParams.size] = strdup(keyword); ConnParams.values[ConnParams.size] = strdup(value); ConnParams.size++; diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index d9b1aad5a..bef91618e 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -449,7 +449,7 @@ CreateTargetEntryForNullCol(Form_pg_attribute attributeTuple, int resno) attributeTuple->attcollation); char *resName = attributeTuple->attname.data; TargetEntry *targetEntry = - makeTargetEntry(nullExpr, resno, strdup(resName), false); + makeTargetEntry(nullExpr, resno, pstrdup(resName), false); return targetEntry; } From db391c0bb79e25961684f20d86e1de305b0bc03f Mon Sep 17 00:00:00 2001 From: eaydingol <60466783+eaydingol@users.noreply.github.com> Date: Sun, 10 Mar 2024 10:20:08 +0300 Subject: [PATCH 030/155] Change the order in which the locks are acquired (#7542) This PR changes the order in which the locks are acquired (for the target and reference tables), when a modify request is initiated from a worker node that is not the "FirstWorkerNode". To prevent concurrent writes, locks are acquired on the first worker node for the replicated tables. When the update statement originates from the first worker node, it acquires the lock on the reference table(s) first, followed by the target table(s). However, if the update statement is initiated in another worker node, the lock requests are sent to the first worker in a different order. This PR unifies the modification order on the first worker node. With the third commit, independent of the node that received the request, the locks are acquired for the modified table and then the reference tables on the first node. The first commit shows a sample output for the test prior to the fix. Fixes #7477 --------- Co-authored-by: Jelte Fennema-Nio (cherry picked from commit 8afa2d0386ff303be29060b709f88994b9648b6c) --- src/backend/distributed/utils/resource_lock.c | 28 ++++++--- src/test/regress/expected/issue_7477.out | 62 +++++++++++++++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/issue_7477.sql | 44 +++++++++++++ 4 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 src/test/regress/expected/issue_7477.out create mode 100644 src/test/regress/sql/issue_7477.sql diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index 13e88a16e..8ac269e43 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -707,13 +707,27 @@ SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode) } List *replicatedShardList = NIL; - if (AnyTableReplicated(shardIntervalList, &replicatedShardList)) - { - if (ClusterHasKnownMetadataWorkers() && !IsFirstWorkerNode()) - { - LockShardListResourcesOnFirstWorker(lockMode, replicatedShardList); - } + bool anyTableReplicated = AnyTableReplicated(shardIntervalList, &replicatedShardList); + /* + * Acquire locks on the modified table. + * If the table is replicated, the locks are first acquired on the first worker node then locally. + * But if we're already on the first worker, acquiring on the first worker node and locally are the same operation. + * So we only acquire locally in that case. + */ + if (anyTableReplicated && ClusterHasKnownMetadataWorkers() && !IsFirstWorkerNode()) + { + LockShardListResourcesOnFirstWorker(lockMode, replicatedShardList); + } + LockShardListResources(shardIntervalList, lockMode); + + /* + * Next, acquire locks on the reference tables that are referenced by a foreign key if there are any. + * Note that LockReferencedReferenceShardResources() first acquires locks on the first worker, + * then locally. + */ + if (anyTableReplicated) + { ShardInterval *firstShardInterval = (ShardInterval *) linitial(replicatedShardList); if (ReferenceTableShardId(firstShardInterval->shardId)) @@ -728,8 +742,6 @@ SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode) LockReferencedReferenceShardResources(firstShardInterval->shardId, lockMode); } } - - LockShardListResources(shardIntervalList, lockMode); } diff --git a/src/test/regress/expected/issue_7477.out b/src/test/regress/expected/issue_7477.out new file mode 100644 index 000000000..224d85c6e --- /dev/null +++ b/src/test/regress/expected/issue_7477.out @@ -0,0 +1,62 @@ +--- Test for updating a table that has a foreign key reference to another reference table. +--- Issue #7477: Distributed deadlock after issuing a simple UPDATE statement +--- https://github.com/citusdata/citus/issues/7477 +CREATE TABLE table1 (id INT PRIMARY KEY); +SELECT create_reference_table('table1'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO table1 VALUES (1); +CREATE TABLE table2 ( + id INT, + info TEXT, + CONSTRAINT table1_id_fk FOREIGN KEY (id) REFERENCES table1 (id) + ); +SELECT create_reference_table('table2'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO table2 VALUES (1, 'test'); +--- Runs the update command in parallel on workers. +--- Due to bug #7477, before the fix, the result is non-deterministic +--- and have several rows of the form: +--- localhost | 57638 | f | ERROR: deadlock detected +--- localhost | 57637 | f | ERROR: deadlock detected +--- localhost | 57637 | f | ERROR: canceling the transaction since it was involved in a distributed deadlock +SELECT * FROM master_run_on_worker( + ARRAY['localhost', 'localhost','localhost', 'localhost','localhost', + 'localhost','localhost', 'localhost','localhost', 'localhost']::text[], + ARRAY[57638, 57637, 57637, 57638, 57637, 57638, 57637, 57638, 57638, 57637]::int[], + ARRAY['UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1' + ]::text[], + true); + node_name | node_port | success | result +--------------------------------------------------------------------- + localhost | 57638 | t | UPDATE 1 + localhost | 57637 | t | UPDATE 1 + localhost | 57637 | t | UPDATE 1 + localhost | 57638 | t | UPDATE 1 + localhost | 57637 | t | UPDATE 1 + localhost | 57638 | t | UPDATE 1 + localhost | 57637 | t | UPDATE 1 + localhost | 57638 | t | UPDATE 1 + localhost | 57638 | t | UPDATE 1 + localhost | 57637 | t | UPDATE 1 +(10 rows) + +--- cleanup +DROP TABLE table2; +DROP TABLE table1; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 2d515e7a2..bc3477ce2 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -102,7 +102,7 @@ test: multi_dropped_column_aliases foreign_key_restriction_enforcement test: binary_protocol test: alter_table_set_access_method test: alter_distributed_table -test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 +test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 test: object_propagation_debug test: undistribute_table test: run_command_on_all_nodes diff --git a/src/test/regress/sql/issue_7477.sql b/src/test/regress/sql/issue_7477.sql new file mode 100644 index 000000000..b9c1578e9 --- /dev/null +++ b/src/test/regress/sql/issue_7477.sql @@ -0,0 +1,44 @@ + +--- Test for updating a table that has a foreign key reference to another reference table. +--- Issue #7477: Distributed deadlock after issuing a simple UPDATE statement +--- https://github.com/citusdata/citus/issues/7477 + +CREATE TABLE table1 (id INT PRIMARY KEY); +SELECT create_reference_table('table1'); +INSERT INTO table1 VALUES (1); + +CREATE TABLE table2 ( + id INT, + info TEXT, + CONSTRAINT table1_id_fk FOREIGN KEY (id) REFERENCES table1 (id) + ); +SELECT create_reference_table('table2'); +INSERT INTO table2 VALUES (1, 'test'); + +--- Runs the update command in parallel on workers. +--- Due to bug #7477, before the fix, the result is non-deterministic +--- and have several rows of the form: +--- localhost | 57638 | f | ERROR: deadlock detected +--- localhost | 57637 | f | ERROR: deadlock detected +--- localhost | 57637 | f | ERROR: canceling the transaction since it was involved in a distributed deadlock + +SELECT * FROM master_run_on_worker( + ARRAY['localhost', 'localhost','localhost', 'localhost','localhost', + 'localhost','localhost', 'localhost','localhost', 'localhost']::text[], + ARRAY[57638, 57637, 57637, 57638, 57637, 57638, 57637, 57638, 57638, 57637]::int[], + ARRAY['UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1', + 'UPDATE table2 SET info = ''test_update'' WHERE id = 1' + ]::text[], + true); + +--- cleanup +DROP TABLE table2; +DROP TABLE table1; From 2a6164d2d9647443a37c3343e6a31f92895e5c47 Mon Sep 17 00:00:00 2001 From: LightDB Enterprise Postgres <88468605+hslightdb@users.noreply.github.com> Date: Wed, 10 Jan 2024 18:49:53 +0800 Subject: [PATCH 031/155] Fix timeout when underlying socket is changed in a MultiConnection (#7377) When there are multiple localhost entries in /etc/hosts like following /etc/hosts: ``` 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 localhost ``` multi_cluster_management check will failed: ``` @@ -857,20 +857,21 @@ ERROR: group 14 already has a primary node -- check that you can add secondaries and unavailable nodes to a group SELECT groupid AS worker_2_group FROM pg_dist_node WHERE nodeport = :worker_2_port \gset SELECT 1 FROM master_add_node('localhost', 9998, groupid => :worker_1_group, noderole => 'secondary'); ?column? ---------- 1 (1 row) SELECT 1 FROM master_add_node('localhost', 9997, groupid => :worker_1_group, noderole => 'unavailable'); +WARNING: could not establish connection after 5000 ms ?column? ---------- 1 (1 row) ``` This actually isn't just a problem in test environments, but could occur as well during actual usage when a hostname in pg_dist_node resolves to multiple IPs and one of those IPs is unreachable. Postgres will then automatically continue with the next IP, but Citus should listen for events on the new socket. Not on the old one. Co-authored-by: chuhx43211 (cherry picked from commit 9a91136a3d320ed7264874b9d24886b5822a1d1b) --- .../distributed/connection/connection_management.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/connection/connection_management.c b/src/backend/distributed/connection/connection_management.c index 64ec1904f..f8e4816ed 100644 --- a/src/backend/distributed/connection/connection_management.c +++ b/src/backend/distributed/connection/connection_management.c @@ -1046,8 +1046,15 @@ FinishConnectionListEstablishment(List *multiConnectionList) continue; } - + bool beforePollSocket = PQsocket(connectionState->connection->pgConn); bool connectionStateChanged = MultiConnectionStatePoll(connectionState); + + if (beforePollSocket != PQsocket(connectionState->connection->pgConn)) + { + /* rebuild the wait events if MultiConnectionStatePoll() changed the socket */ + waitEventSetRebuild = true; + } + if (connectionStateChanged) { if (connectionState->phase != MULTI_CONNECTION_PHASE_CONNECTING) From 5708fca1ea006ab2945e6dcff35830d71fb0515e Mon Sep 17 00:00:00 2001 From: sminux Date: Tue, 5 Mar 2024 10:49:35 +0300 Subject: [PATCH 032/155] fix bad copy-paste rightComparisonLimit (#7547) DESCRIPTION: change for #7543 (cherry picked from commit d59c93bc504ad32621d66583de6b65f936b0ed13) --- src/backend/columnar/columnar_tableam.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 40486d08f..ca3a5f4c4 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -2946,7 +2946,7 @@ MajorVersionsCompatibleColumnar(char *leftVersion, char *rightVersion) } else { - rightComparisionLimit = strlen(leftVersion); + rightComparisionLimit = strlen(rightVersion); } /* we can error out early if hypens are not in the same position */ From f4af59ab4b67c8305dda4af6b2f61853de667705 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Wed, 17 Jan 2024 16:36:26 +0100 Subject: [PATCH 033/155] Support running isolation_update_node in flaky test detection (#7425) I noticed in #7423 that `isolation_update_node` could not be run using flaky test detection. This fixes that. --- src/test/regress/citus_tests/run_test.py | 3 +++ src/test/regress/expected/isolation_update_node.out | 12 ++++++------ src/test/regress/spec/isolation_update_node.spec | 2 ++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 2b71f5e1b..5dbff9193 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -158,6 +158,9 @@ DEPS = { "isolation_extension_commands": TestDeps( None, ["isolation_setup", "isolation_add_remove_node"] ), + "isolation_update_node": TestDeps( + None, ["isolation_setup", "isolation_add_remove_node"] + ), "schema_based_sharding": TestDeps("minimal_schedule"), "multi_sequence_default": TestDeps( None, diff --git a/src/test/regress/expected/isolation_update_node.out b/src/test/regress/expected/isolation_update_node.out index 1a1c65ec8..adb540cdb 100644 --- a/src/test/regress/expected/isolation_update_node.out +++ b/src/test/regress/expected/isolation_update_node.out @@ -93,8 +93,8 @@ nodeid|nodename|nodeport starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes s3-update-node-1-back s3-manually-fix-metadata nodeid|nodename |nodeport --------------------------------------------------------------------- - 25|localhost| 57638 - 24|localhost| 57637 + 23|localhost| 57638 + 22|localhost| 57637 (2 rows) step s1-begin: @@ -139,8 +139,8 @@ step s1-show-nodes: nodeid|nodename |nodeport|isactive --------------------------------------------------------------------- - 25|localhost| 57638|t - 24|localhost| 58637|t + 23|localhost| 57638|t + 22|localhost| 58637|t (2 rows) step s3-update-node-1-back: @@ -178,8 +178,8 @@ nodeid|nodename|nodeport starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s3-manually-fix-metadata nodeid|nodename |nodeport --------------------------------------------------------------------- - 27|localhost| 57638 - 26|localhost| 57637 + 23|localhost| 57638 + 22|localhost| 57637 (2 rows) step s2-create-table: diff --git a/src/test/regress/spec/isolation_update_node.spec b/src/test/regress/spec/isolation_update_node.spec index d6be6bfdc..ccbbbec1b 100644 --- a/src/test/regress/spec/isolation_update_node.spec +++ b/src/test/regress/spec/isolation_update_node.spec @@ -3,6 +3,8 @@ setup -- revert back to pg_isolation_test_session_is_blocked until the tests are fixed SELECT citus_internal.restore_isolation_tester_func(); + ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 22; + SELECT 1 FROM master_add_node('localhost', 57637); SELECT 1 FROM master_add_node('localhost', 57638); From 75130610573a764e6bdb8b2c504a6a5e48e6a7d9 Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:39:07 +0300 Subject: [PATCH 034/155] Make isolation_update_node test system independent (#7423) Test isolation_update_node fails on some systems with the following error: ``` -s2: WARNING: connection to the remote node non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: Name or service not known +s2: WARNING: connection to the remote node non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: Temporary failure in name resolution ``` This slightly modifies an already existing [normalization rule](https://github.com/citusdata/citus/blob/739c6d26df56d38cda7f5420cccca947874d71e6/src/test/regress/bin/normalize.sed#L217-L218) to fix it. Co-authored-by: Karina Litskevich (cherry picked from commit 21464adfecea1f8b1604356909420303f2041bed) --- src/test/regress/bin/normalize.sed | 2 +- src/test/regress/expected/isolation_update_node.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index efa9e310f..5dc5d7ca8 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -215,7 +215,7 @@ s/^(ERROR: The index name \(test_index_creation1_p2020_09_26)_([0-9])+_(tenant_ s/^(DEBUG: the index name on the shards of the partition is too long, switching to sequential and local execution mode to prevent self deadlocks: test_index_creation1_p2020_09_26)_([0-9])+_(tenant_id_timeperiod_idx)/\1_xxxxxx_\3/g # normalize errors for not being able to connect to a non-existing host -s/could not translate host name "foobar" to address: .*$/could not translate host name "foobar" to address: /g +s/could not translate host name "([A-Za-z0-9\.\-]+)" to address: .*$/could not translate host name "\1" to address: /g # ignore PL/pgSQL line numbers that differ on Mac builds s/(CONTEXT: PL\/pgSQL function .* line )([0-9]+)/\1XX/g diff --git a/src/test/regress/expected/isolation_update_node.out b/src/test/regress/expected/isolation_update_node.out index adb540cdb..53d792e61 100644 --- a/src/test/regress/expected/isolation_update_node.out +++ b/src/test/regress/expected/isolation_update_node.out @@ -250,7 +250,7 @@ count step s1-commit-prepared: COMMIT prepared 'label'; -s2: WARNING: connection to the remote node non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: Name or service not known +s2: WARNING: connection to the remote node non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: step s2-execute-prepared: EXECUTE foo; From fc09e1cfdcb4619544c6f356b14a39f766c8b718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Sedl=C3=A1k?= Date: Wed, 24 Jan 2024 12:24:23 +0100 Subject: [PATCH 035/155] Log username in the failed connection message (#7432) This patch includes the username in the reported error message. This makes debugging easier when certain commands open connections as other users than the user that is executing the command. ``` monitora_snapshot=# SELECT citus_move_shard_placement(102030, 'monitora.db-dev-worker-a', 6005, 'monitora.db-dev-worker-a', 6017); ERROR: connection to the remote node monitora_user@monitora.db-dev-worker-a:6017 failed with the following error: fe_sendauth: no password supplied Time: 40,198 ms ``` (cherry picked from commit 8b48d6ab0298d80ded17a6721e187b627f647c91) --- .../distributed/connection/remote_commands.c | 9 +- .../regress/expected/detect_conn_close.out | 2 +- .../failure_connection_establishment.out | 2 +- .../regress/expected/failure_copy_on_hash.out | 4 +- ...ure_create_distributed_table_non_empty.out | 12 +-- .../failure_create_index_concurrently.out | 18 ++-- .../regress/expected/failure_create_table.out | 10 +-- .../regress/expected/failure_cte_subquery.out | 8 +- src/test/regress/expected/failure_ddl.out | 16 ++-- .../expected/failure_distributed_results.out | 2 +- .../failure_failover_to_local_execution.out | 2 +- .../failure_insert_select_pushdown.out | 4 +- .../failure_insert_select_repartition.out | 10 +-- .../regress/expected/failure_multi_dml.out | 6 +- .../expected/failure_multi_row_insert.out | 10 +-- .../failure_multi_shard_update_delete.out | 32 +++---- .../failure_mx_metadata_sync_multi_trans.out | 84 +++++++++---------- .../failure_on_create_subscription.out | 4 +- .../failure_online_move_shard_placement.out | 6 +- .../regress/expected/failure_ref_tables.out | 6 +- .../failure_replicated_partitions.out | 2 +- .../regress/expected/failure_savepoints.out | 2 +- .../regress/expected/failure_single_mod.out | 6 +- .../expected/failure_single_select.out | 10 +-- .../expected/failure_split_cleanup.out | 8 +- .../expected/failure_tenant_isolation.out | 8 +- .../failure_tenant_isolation_nonblocking.out | 6 +- .../regress/expected/failure_truncate.out | 14 ++-- src/test/regress/expected/failure_vacuum.out | 6 +- .../expected/isolation_update_node.out | 2 +- .../expected/local_shard_execution.out | 4 +- .../expected/local_shard_execution_0.out | 4 +- .../regress/expected/multi_citus_tools.out | 2 +- src/test/regress/expected/multi_copy.out | 6 +- .../expected/multi_modifying_xacts.out | 20 ++--- .../regress/expected/multi_multiuser_auth.out | 2 +- .../regress/expected/multi_router_planner.out | 4 +- .../regress/expected/node_conninfo_reload.out | 16 ++-- .../regress/expected/shard_rebalancer.out | 6 +- .../sql/failure_on_create_subscription.sql | 4 +- 40 files changed, 190 insertions(+), 189 deletions(-) diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index f694ff390..4b46e96d2 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -246,6 +246,7 @@ ClearResultsIfReady(MultiConnection *connection) void ReportConnectionError(MultiConnection *connection, int elevel) { + char *userName = connection->user; char *nodeName = connection->hostname; int nodePort = connection->port; PGconn *pgConn = connection->pgConn; @@ -264,15 +265,15 @@ ReportConnectionError(MultiConnection *connection, int elevel) if (messageDetail) { ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("connection to the remote node %s:%d failed with the " - "following error: %s", nodeName, nodePort, + errmsg("connection to the remote node %s@%s:%d failed with the " + "following error: %s", userName, nodeName, nodePort, messageDetail))); } else { ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("connection to the remote node %s:%d failed", - nodeName, nodePort))); + errmsg("connection to the remote node %s@%s:%d failed", + userName, nodeName, nodePort))); } } diff --git a/src/test/regress/expected/detect_conn_close.out b/src/test/regress/expected/detect_conn_close.out index ad758f32e..60973de76 100644 --- a/src/test/regress/expected/detect_conn_close.out +++ b/src/test/regress/expected/detect_conn_close.out @@ -128,7 +128,7 @@ BEGIN; (1 row) SELECT count(*) FROM socket_test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open ROLLBACK; -- repartition joins also can recover SET citus.enable_repartition_joins TO on; diff --git a/src/test/regress/expected/failure_connection_establishment.out b/src/test/regress/expected/failure_connection_establishment.out index d032755dd..f23f11d2b 100644 --- a/src/test/regress/expected/failure_connection_establishment.out +++ b/src/test/regress/expected/failure_connection_establishment.out @@ -84,7 +84,7 @@ SELECT citus.mitmproxy('conn.connect_delay(1400)'); ALTER TABLE products ADD CONSTRAINT p_key PRIMARY KEY(product_no); WARNING: could not establish connection after 900 ms -ERROR: connection to the remote node localhost:xxxxx failed +ERROR: connection to the remote node postgres@localhost:xxxxx failed RESET citus.node_connection_timeout; SELECT citus.mitmproxy('conn.allow()'); mitmproxy diff --git a/src/test/regress/expected/failure_copy_on_hash.out b/src/test/regress/expected/failure_copy_on_hash.out index 24350f707..424ab0da8 100644 --- a/src/test/regress/expected/failure_copy_on_hash.out +++ b/src/test/regress/expected/failure_copy_on_hash.out @@ -36,7 +36,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) \COPY test_table FROM stdin delimiter ','; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open CONTEXT: COPY test_table, line 1: "1,2" SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -271,7 +271,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) \COPY test_table_2 FROM stdin delimiter ','; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_create_distributed_table_non_empty.out b/src/test/regress/expected/failure_create_distributed_table_non_empty.out index 0e4b85701..109d3686f 100644 --- a/src/test/regress/expected/failure_create_distributed_table_non_empty.out +++ b/src/test/regress/expected/failure_create_distributed_table_non_empty.out @@ -28,7 +28,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- @@ -164,7 +164,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- @@ -436,7 +436,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- @@ -519,7 +519,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- @@ -680,7 +680,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- @@ -901,7 +901,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; count --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_create_index_concurrently.out b/src/test/regress/expected/failure_create_index_concurrently.out index a198ddc70..2101ecc80 100644 --- a/src/test/regress/expected/failure_create_index_concurrently.out +++ b/src/test/regress/expected/failure_create_index_concurrently.out @@ -27,8 +27,8 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -60,8 +60,8 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -87,7 +87,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -112,7 +112,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -138,8 +138,8 @@ SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()'); DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -165,7 +165,7 @@ SELECT create_distributed_table('index_test_2', 'a'); INSERT INTO index_test_2 VALUES (1, 1), (1, 2); CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a); WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. + Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. ERROR: could not create unique index "index_test_2_a_idx_1880019" DETAIL: Key (a)=(1) is duplicated. CONTEXT: while executing command on localhost:xxxxx diff --git a/src/test/regress/expected/failure_create_table.out b/src/test/regress/expected/failure_create_table.out index 5d022d678..956bdb2b2 100644 --- a/src/test/regress/expected/failure_create_table.out +++ b/src/test/regress/expected/failure_create_table.out @@ -22,7 +22,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) SELECT create_distributed_table('test_table','id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -116,7 +116,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman (1 row) SELECT create_distributed_table('test_table','id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -147,7 +147,7 @@ BEGIN; (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open COMMIT; SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -215,7 +215,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -484,7 +484,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open ROLLBACK; SELECT citus.mitmproxy('conn.allow()'); mitmproxy diff --git a/src/test/regress/expected/failure_cte_subquery.out b/src/test/regress/expected/failure_cte_subquery.out index 19fb11f37..89e3e1489 100644 --- a/src/test/regress/expected/failure_cte_subquery.out +++ b/src/test/regress/expected/failure_cte_subquery.out @@ -86,7 +86,7 @@ FROM ORDER BY 1 DESC LIMIT 5 ) as foo WHERE foo.user_id = cte.user_id; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- kill at the third copy (pull) SELECT citus.mitmproxy('conn.onQuery(query="SELECT DISTINCT users_table.user").kill()'); mitmproxy @@ -117,7 +117,7 @@ FROM ORDER BY 1 DESC LIMIT 5 ) as foo WHERE foo.user_id = cte.user_id; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancel at the first copy (push) SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')'); mitmproxy @@ -254,7 +254,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()'); WITH cte_delete AS MATERIALIZED (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *) INSERT INTO users_table SELECT * FROM cte_delete; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify contents are the same SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -365,7 +365,7 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode = 'sequential'; WITH cte_delete AS MATERIALIZED (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *) INSERT INTO users_table SELECT * FROM cte_delete; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open END; RESET SEARCH_PATH; SELECT citus.mitmproxy('conn.allow()'); diff --git a/src/test/regress/expected/failure_ddl.out b/src/test/regress/expected/failure_ddl.out index 77b134a72..2f55663a0 100644 --- a/src/test/regress/expected/failure_ddl.out +++ b/src/test/regress/expected/failure_ddl.out @@ -36,7 +36,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -99,7 +99,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- show that we've never commited the changes SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg @@ -300,7 +300,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -361,7 +361,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -661,7 +661,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -722,7 +722,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -1010,7 +1010,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").after(2).kill()'); mitmproxy @@ -1019,7 +1019,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").after(2).cancel(' || pg_backend_pid() || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_distributed_results.out b/src/test/regress/expected/failure_distributed_results.out index fc97c9af6..e56a87760 100644 --- a/src/test/regress/expected/failure_distributed_results.out +++ b/src/test/regress/expected/failure_distributed_results.out @@ -86,7 +86,7 @@ CREATE TABLE distributed_result_info AS SELECT resultId, nodeport, rowcount, targetShardId, targetShardIndex FROM partition_task_list_results('test', $$ SELECT * FROM source_table $$, 'target_table') NATURAL JOIN pg_dist_node; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM distributed_result_info ORDER BY resultId; resultid | nodeport | rowcount | targetshardid | targetshardindex --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_failover_to_local_execution.out b/src/test/regress/expected/failure_failover_to_local_execution.out index 56518141a..20ad2a6df 100644 --- a/src/test/regress/expected/failure_failover_to_local_execution.out +++ b/src/test/regress/expected/failure_failover_to_local_execution.out @@ -101,7 +101,7 @@ NOTICE: issuing SELECT count(*) AS count FROM failure_failover_to_local_executi DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SELECT count(*) AS count FROM failure_failover_to_local_execution.failover_to_local_1980003 failover_to_local WHERE true DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open NOTICE: executing the command locally: SELECT count(*) AS count FROM failure_failover_to_local_execution.failover_to_local_1980000 failover_to_local WHERE true NOTICE: executing the command locally: SELECT count(*) AS count FROM failure_failover_to_local_execution.failover_to_local_1980002 failover_to_local WHERE true count diff --git a/src/test/regress/expected/failure_insert_select_pushdown.out b/src/test/regress/expected/failure_insert_select_pushdown.out index ed461d040..570bf22f9 100644 --- a/src/test/regress/expected/failure_insert_select_pushdown.out +++ b/src/test/regress/expected/failure_insert_select_pushdown.out @@ -44,7 +44,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown" (1 row) INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open --verify nothing is modified SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -95,7 +95,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown" (1 row) INSERT INTO events_table SELECT * FROM events_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open --verify nothing is modified SELECT citus.mitmproxy('conn.allow()'); mitmproxy diff --git a/src/test/regress/expected/failure_insert_select_repartition.out b/src/test/regress/expected/failure_insert_select_repartition.out index d45318208..ca36f7e88 100644 --- a/src/test/regress/expected/failure_insert_select_repartition.out +++ b/src/test/regress/expected/failure_insert_select_repartition.out @@ -55,7 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill (1 row) INSERT INTO target_table SELECT * FROM source_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM target_table ORDER BY a; a | b --------------------------------------------------------------------- @@ -68,7 +68,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill (1 row) INSERT INTO target_table SELECT * FROM replicated_source_table; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM target_table ORDER BY a; a | b --------------------------------------------------------------------- @@ -138,7 +138,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO target_table SELECT * FROM source_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM target_table ORDER BY a; a | b --------------------------------------------------------------------- @@ -151,7 +151,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO target_table SELECT * FROM replicated_source_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM target_table ORDER BY a; a | b --------------------------------------------------------------------- @@ -168,7 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO replicated_target_table SELECT * FROM source_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT * FROM replicated_target_table; a | b --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_multi_dml.out b/src/test/regress/expected/failure_multi_dml.out index bbea2c999..7757f574c 100644 --- a/src/test/regress/expected/failure_multi_dml.out +++ b/src/test/regress/expected/failure_multi_dml.out @@ -33,7 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE").kill()'); BEGIN; DELETE FROM dml_test WHERE id = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open DELETE FROM dml_test WHERE id = 2; ERROR: current transaction is aborted, commands ignored until end of transaction block INSERT INTO dml_test VALUES (5, 'Epsilon'); @@ -93,7 +93,7 @@ BEGIN; DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; INSERT INTO dml_test VALUES (5, 'Epsilon'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open UPDATE dml_test SET name = 'alpha' WHERE id = 1; ERROR: current transaction is aborted, commands ignored until end of transaction block UPDATE dml_test SET name = 'gamma' WHERE id = 3; @@ -148,7 +148,7 @@ DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; INSERT INTO dml_test VALUES (5, 'Epsilon'); UPDATE dml_test SET name = 'alpha' WHERE id = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open UPDATE dml_test SET name = 'gamma' WHERE id = 3; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; diff --git a/src/test/regress/expected/failure_multi_row_insert.out b/src/test/regress/expected/failure_multi_row_insert.out index f3cd4919a..8feffbaeb 100644 --- a/src/test/regress/expected/failure_multi_row_insert.out +++ b/src/test/regress/expected/failure_multi_row_insert.out @@ -43,7 +43,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- this test is broken, see https://github.com/citusdata/citus/issues/2460 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); -- INSERT INTO distributed_table VALUES (1,4), (1,5), (1,6); @@ -55,7 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,7), (5,8); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- this test is broken, see https://github.com/citusdata/citus/issues/2460 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); -- INSERT INTO distributed_table VALUES (1,9), (5,10); @@ -67,7 +67,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,11), (6,12); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- @@ -84,7 +84,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()'); (1 row) INSERT INTO distributed_table VALUES (1,15), (6,16); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- @@ -101,7 +101,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (2,19),(1,20); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_multi_shard_update_delete.out b/src/test/regress/expected/failure_multi_shard_update_delete.out index 24cb895ea..27284ec38 100644 --- a/src/test/regress/expected/failure_multi_shard_update_delete.out +++ b/src/test/regress/expected/failure_multi_shard_update_delete.out @@ -58,7 +58,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); -- issue a multi shard delete DELETE FROM t2 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FROM t2; count @@ -74,7 +74,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005"). (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FROM t2; count @@ -134,7 +134,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); -- issue a multi shard update UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2; b2 | c4 @@ -150,7 +150,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill( (1 row) UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2; b2 | c4 @@ -202,7 +202,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); -- issue a multi shard delete DELETE FROM t2 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FROM t2; count @@ -218,7 +218,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005"). (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FROM t2; count @@ -278,7 +278,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); -- issue a multi shard update UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2; b2 | c4 @@ -294,7 +294,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill( (1 row) UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2; b2 | c4 @@ -364,7 +364,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); (1 row) DELETE FROM r1 WHERE a = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2; b2 @@ -379,7 +379,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is deleted SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2; b2 @@ -459,7 +459,7 @@ UPDATE t3 SET c = q.c FROM ( SELECT b, max(c) as c FROM t2 GROUP BY b) q WHERE t3.b = q.b RETURNING *; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open --- verify nothing is updated SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -515,7 +515,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill( (1 row) UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3; b1 | b2 @@ -547,7 +547,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO -- following will fail UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open END; -- verify everything is rolled back SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2; @@ -563,7 +563,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO (1 row) UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3; b1 | b2 @@ -578,7 +578,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO (1 row) UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- verify nothing is updated SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3; b1 | b2 @@ -610,7 +610,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO -- following will fail UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open END; -- verify everything is rolled back SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2; diff --git a/src/test/regress/expected/failure_mx_metadata_sync_multi_trans.out b/src/test/regress/expected/failure_mx_metadata_sync_multi_trans.out index 541bce5c5..2c4120dbd 100644 --- a/src/test/regress/expected/failure_mx_metadata_sync_multi_trans.out +++ b/src/test/regress/expected/failure_mx_metadata_sync_multi_trans.out @@ -155,7 +155,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE pg_dist_local_group SET group (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to drop node metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").cancel(' || :pid || ')'); mitmproxy @@ -172,7 +172,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").kill()'); (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to send node metadata SELECT citus.mitmproxy('conn.onQuery(query="INSERT INTO pg_dist_node").cancel(' || :pid || ')'); mitmproxy @@ -189,7 +189,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="INSERT INTO pg_dist_node").kill()'); (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to drop sequence dependency for all tables SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_drop_sequence_dependency.*FROM pg_dist_partition").cancel(' || :pid || ')'); mitmproxy @@ -206,7 +206,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_drop_sequen (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to drop shell table SELECT citus.mitmproxy('conn.onQuery(query="CALL pg_catalog.worker_drop_all_shell_tables").cancel(' || :pid || ')'); mitmproxy @@ -223,7 +223,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CALL pg_catalog.worker_drop_all_shel (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to delete all pg_dist_partition metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_partition").cancel(' || :pid || ')'); mitmproxy @@ -240,7 +240,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_partition").kill (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to delete all pg_dist_shard metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_shard").cancel(' || :pid || ')'); mitmproxy @@ -257,7 +257,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_shard").kill()') (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to delete all pg_dist_placement metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_placement").cancel(' || :pid || ')'); mitmproxy @@ -274,7 +274,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_placement").kill (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to delete all pg_dist_object metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_catalog.pg_dist_object").cancel(' || :pid || ')'); mitmproxy @@ -291,7 +291,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_catalog.pg_dist_objec (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to delete all pg_dist_colocation metadata SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_catalog.pg_dist_colocation").cancel(' || :pid || ')'); mitmproxy @@ -308,7 +308,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_catalog.pg_dist_coloc (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to alter or create role SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_alter_role").cancel(' || :pid || ')'); mitmproxy @@ -325,7 +325,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_alter_role") (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to set database owner SELECT citus.mitmproxy('conn.onQuery(query="ALTER DATABASE.*OWNER TO").cancel(' || :pid || ')'); mitmproxy @@ -342,7 +342,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="ALTER DATABASE.*OWNER TO").kill()'); (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create schema SELECT citus.mitmproxy('conn.onQuery(query="CREATE SCHEMA IF NOT EXISTS mx_metadata_sync_multi_trans AUTHORIZATION").cancel(' || :pid || ')'); mitmproxy @@ -359,7 +359,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE SCHEMA IF NOT EXISTS mx_metad (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create collation SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_object.*CREATE COLLATION mx_metadata_sync_multi_trans.german_phonebook").cancel(' || :pid || ')'); mitmproxy @@ -376,7 +376,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_obje (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create function SELECT citus.mitmproxy('conn.onQuery(query="CREATE OR REPLACE FUNCTION mx_metadata_sync_multi_trans.one_as_result").cancel(' || :pid || ')'); mitmproxy @@ -393,7 +393,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE OR REPLACE FUNCTION mx_metada (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create text search dictionary SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_object.*my_german_dict").cancel(' || :pid || ')'); mitmproxy @@ -410,7 +410,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_obje (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create text search config SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_object.*my_ts_config").cancel(' || :pid || ')'); mitmproxy @@ -427,7 +427,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_obje (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create type SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_object.*pair_type").cancel(' || :pid || ')'); mitmproxy @@ -444,7 +444,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_create_or_replace_obje (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create publication SELECT citus.mitmproxy('conn.onQuery(query="CREATE PUBLICATION.*pub_all").cancel(' || :pid || ')'); mitmproxy @@ -461,7 +461,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE PUBLICATION.*pub_all").kill() (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create sequence SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_sequence_command").cancel(' || :pid || ')'); mitmproxy @@ -478,7 +478,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_sequence_command (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to drop sequence dependency for distributed table SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_drop_sequence_dependency.*mx_metadata_sync_multi_trans.dist1").cancel(' || :pid || ')'); mitmproxy @@ -495,7 +495,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_drop_sequen (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to drop distributed table if exists SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS mx_metadata_sync_multi_trans.dist1").cancel(' || :pid || ')'); mitmproxy @@ -512,7 +512,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS mx_metadata_syn (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create distributed table SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_trans.dist1").cancel(' || :pid || ')'); mitmproxy @@ -529,7 +529,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to record sequence dependency for table SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_record_sequence_dependency").cancel(' || :pid || ')'); mitmproxy @@ -546,7 +546,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.worker_record_sequ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create index for table SELECT citus.mitmproxy('conn.onQuery(query="CREATE INDEX dist1_search_phone_idx ON mx_metadata_sync_multi_trans.dist1 USING gin").cancel(' || :pid || ')'); mitmproxy @@ -563,7 +563,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE INDEX dist1_search_phone_idx (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create reference table SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_trans.ref").cancel(' || :pid || ')'); mitmproxy @@ -580,7 +580,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create local table SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_trans.loc1").cancel(' || :pid || ')'); mitmproxy @@ -597,7 +597,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create distributed partitioned table SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_trans.orders").cancel(' || :pid || ')'); mitmproxy @@ -614,7 +614,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to create distributed partition table SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_trans.orders_p2020_01_05").cancel(' || :pid || ')'); mitmproxy @@ -631,7 +631,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE mx_metadata_sync_multi_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to attach partition SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE mx_metadata_sync_multi_trans.orders ATTACH PARTITION mx_metadata_sync_multi_trans.orders_p2020_01_05").cancel(' || :pid || ')'); mitmproxy @@ -648,7 +648,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE mx_metadata_sync_multi_t (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to add partition metadata SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_partition_metadata").cancel(' || :pid || ')'); mitmproxy @@ -665,7 +665,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_partition_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to add shard metadata SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_shard_metadata").cancel(' || :pid || ')'); mitmproxy @@ -682,7 +682,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_shard_meta (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to add placement metadata SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_placement_metadata").cancel(' || :pid || ')'); mitmproxy @@ -699,7 +699,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_placement_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to add colocation metadata SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.citus_internal_add_colocation_metadata").cancel(' || :pid || ')'); mitmproxy @@ -716,7 +716,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT pg_catalog.citus_internal_add (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to add distributed object metadata SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_object_metadata").cancel(' || :pid || ')'); mitmproxy @@ -733,7 +733,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT citus_internal_add_object_met (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark function as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*one_as_result").cancel(' || :pid || ')'); mitmproxy @@ -750,7 +750,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*one_as (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark collation as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*german_phonebook").cancel(' || :pid || ')'); mitmproxy @@ -767,7 +767,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*german (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark text search dictionary as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*my_german_dict").cancel(' || :pid || ')'); mitmproxy @@ -784,7 +784,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*my_ger (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark text search configuration as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*my_ts_config").cancel(' || :pid || ')'); mitmproxy @@ -801,7 +801,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*my_ts_ (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark type as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*pair_type").cancel(' || :pid || ')'); mitmproxy @@ -818,7 +818,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*pair_t (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark sequence as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*seq_owned").cancel(' || :pid || ')'); mitmproxy @@ -835,7 +835,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*seq_ow (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to mark publication as distributed SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*pub_all").cancel(' || :pid || ')'); mitmproxy @@ -852,7 +852,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="WITH distributed_object_data.*pub_al (1 row) SELECT citus_activate_node('localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Failure to set isactive to true SELECT citus.mitmproxy('conn.onQuery(query="UPDATE pg_dist_node SET isactive = TRUE").cancel(' || :pid || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_on_create_subscription.out b/src/test/regress/expected/failure_on_create_subscription.out index 19df82d3e..a42df24d2 100644 --- a/src/test/regress/expected/failure_on_create_subscription.out +++ b/src/test/regress/expected/failure_on_create_subscription.out @@ -43,9 +43,9 @@ SELECT * FROM shards_in_workers; -- Failure on creating the subscription -- Failing exactly on CREATE SUBSCRIPTION is causing flaky test where we fail with either: --- 1) ERROR: connection to the remote node localhost:xxxxx failed with the following error: ERROR: subscription "citus_shard_move_subscription_xxxxxxx" does not exist +-- 1) ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: ERROR: subscription "citus_shard_move_subscription_xxxxxxx" does not exist -- another command is already in progress --- 2) ERROR: connection to the remote node localhost:xxxxx failed with the following error: another command is already in progress +-- 2) ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: another command is already in progress -- Instead fail on the next step (ALTER SUBSCRIPTION) instead which is also required logically as part of uber CREATE SUBSCRIPTION operation. SELECT citus.mitmproxy('conn.onQuery(query="ALTER SUBSCRIPTION").kill()'); mitmproxy diff --git a/src/test/regress/expected/failure_online_move_shard_placement.out b/src/test/regress/expected/failure_online_move_shard_placement.out index cf5890f35..0a881fe42 100644 --- a/src/test/regress/expected/failure_online_move_shard_placement.out +++ b/src/test/regress/expected/failure_online_move_shard_placement.out @@ -407,7 +407,7 @@ SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()'); (1 row) SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cleanup leftovers SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -442,7 +442,7 @@ SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()'); (1 row) SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- failure on parallel create index ALTER SYSTEM RESET citus.max_adaptive_executor_pool_size; SELECT pg_reload_conf(); @@ -458,7 +458,7 @@ SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()'); (1 row) SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- Verify that the shard is not moved and the number of rows are still 100k SELECT citus.mitmproxy('conn.allow()'); mitmproxy diff --git a/src/test/regress/expected/failure_ref_tables.out b/src/test/regress/expected/failure_ref_tables.out index 4984cc1bf..e9a7e4571 100644 --- a/src/test/regress/expected/failure_ref_tables.out +++ b/src/test/regress/expected/failure_ref_tables.out @@ -33,7 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); (1 row) INSERT INTO ref_table VALUES (5, 6); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT COUNT(*) FROM ref_table WHERE key=5; count --------------------------------------------------------------------- @@ -48,7 +48,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); (1 row) UPDATE ref_table SET key=7 RETURNING value; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT COUNT(*) FROM ref_table WHERE key=7; count --------------------------------------------------------------------- @@ -65,7 +65,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); BEGIN; DELETE FROM ref_table WHERE key=5; UPDATE ref_table SET key=value; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open COMMIT; SELECT COUNT(*) FROM ref_table WHERE key=value; count diff --git a/src/test/regress/expected/failure_replicated_partitions.out b/src/test/regress/expected/failure_replicated_partitions.out index 7294df98b..67dda269c 100644 --- a/src/test/regress/expected/failure_replicated_partitions.out +++ b/src/test/regress/expected/failure_replicated_partitions.out @@ -28,7 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); (1 row) INSERT INTO partitioned_table VALUES (0, 0); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- use both placements SET citus.task_assignment_policy TO "round-robin"; -- the results should be the same diff --git a/src/test/regress/expected/failure_savepoints.out b/src/test/regress/expected/failure_savepoints.out index 9b155e90e..ca5cb91f6 100644 --- a/src/test/regress/expected/failure_savepoints.out +++ b/src/test/regress/expected/failure_savepoints.out @@ -312,7 +312,7 @@ SELECT * FROM ref; ROLLBACK TO SAVEPOINT start; SELECT * FROM ref; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open END; -- clean up RESET client_min_messages; diff --git a/src/test/regress/expected/failure_single_mod.out b/src/test/regress/expected/failure_single_mod.out index 2a6ed2d77..aa6c10e66 100644 --- a/src/test/regress/expected/failure_single_mod.out +++ b/src/test/regress/expected/failure_single_mod.out @@ -27,7 +27,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="INSERT").kill()'); (1 row) INSERT INTO mod_test VALUES (2, 6); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT COUNT(*) FROM mod_test WHERE key=2; count --------------------------------------------------------------------- @@ -59,7 +59,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE").kill()'); (1 row) UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT COUNT(*) FROM mod_test WHERE value='ok'; count --------------------------------------------------------------------- @@ -89,7 +89,7 @@ INSERT INTO mod_test VALUES (2, 6); INSERT INTO mod_test VALUES (2, 7); DELETE FROM mod_test WHERE key=2 AND value = '7'; UPDATE mod_test SET value='ok' WHERE key=2; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open COMMIT; SELECT COUNT(*) FROM mod_test WHERE key=2; count diff --git a/src/test/regress/expected/failure_single_select.out b/src/test/regress/expected/failure_single_select.out index 1b60f3125..586dd4756 100644 --- a/src/test/regress/expected/failure_single_select.out +++ b/src/test/regress/expected/failure_single_select.out @@ -30,14 +30,14 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); (1 row) SELECT * FROM select_test WHERE key = 3; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open key | value --------------------------------------------------------------------- 3 | test data (1 row) SELECT * FROM select_test WHERE key = 3; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open key | value --------------------------------------------------------------------- 3 | test data @@ -54,7 +54,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").kill()'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); SELECT * FROM select_test WHERE key = 3; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open COMMIT; SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -142,7 +142,7 @@ SELECT * FROM select_test WHERE key = 3; INSERT INTO select_test VALUES (3, 'even more data'); SELECT * FROM select_test WHERE key = 3; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open COMMIT; SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*pg_prepared_xacts").after(2).kill()'); mitmproxy @@ -186,7 +186,7 @@ SELECT * FROM select_test WHERE key = 1; (1 row) SELECT * FROM select_test WHERE key = 1; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- now the same test with query cancellation SELECT citus.mitmproxy('conn.onQuery(query="SELECT.*select_test").after(1).cancel(' || pg_backend_pid() || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_split_cleanup.out b/src/test/regress/expected/failure_split_cleanup.out index fe646587c..c34fdd771 100644 --- a/src/test/regress/expected/failure_split_cleanup.out +++ b/src/test/regress/expected/failure_split_cleanup.out @@ -627,10 +627,10 @@ WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open -WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open +WARNING: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open ERROR: connection not open CONTEXT: while executing command on localhost:xxxxx SELECT operation_id, object_type, object_name, node_group_id, policy_type diff --git a/src/test/regress/expected/failure_tenant_isolation.out b/src/test/regress/expected/failure_tenant_isolation.out index 6be4580be..b406aa2a3 100644 --- a/src/test/regress/expected/failure_tenant_isolation.out +++ b/src/test/regress/expected/failure_tenant_isolation.out @@ -76,7 +76,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(302").kill()'); (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode => 'block_writes'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on colocated table population SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(302").cancel(' || :pid || ')'); mitmproxy @@ -94,7 +94,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE tenant_isolation.table_2 (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode => 'block_writes'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on colocated table constraints SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE tenant_isolation.table_2 ADD CONSTRAINT").after(2).cancel(' || :pid || ')'); mitmproxy @@ -131,7 +131,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(300").kill()'); (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode => 'block_writes'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on table population SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(300").cancel(' || :pid || ')'); mitmproxy @@ -149,7 +149,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE tenant_isolation.table_1 (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode => 'block_writes'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on table constraints SELECT citus.mitmproxy('conn.onQuery(query="ALTER TABLE tenant_isolation.table_1 ADD CONSTRAINT").after(2).cancel(' || :pid || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_tenant_isolation_nonblocking.out b/src/test/regress/expected/failure_tenant_isolation_nonblocking.out index e40842e2a..aecde71c0 100644 --- a/src/test/regress/expected/failure_tenant_isolation_nonblocking.out +++ b/src/test/regress/expected/failure_tenant_isolation_nonblocking.out @@ -159,7 +159,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SET TRANSACTION SNAPSHOT").kill()'); (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode := 'force_logical'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on setting snapshot SELECT citus.mitmproxy('conn.onQuery(query="SET TRANSACTION SNAPSHOT").cancel(' || :pid || ')'); mitmproxy @@ -177,7 +177,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(300").kill()'); (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode := 'force_logical'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on table population SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(300").cancel(' || :pid || ')'); mitmproxy @@ -195,7 +195,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(302").kill()'); (1 row) SELECT isolate_tenant_to_new_shard('table_1', 5, 'CASCADE', shard_transfer_mode := 'force_logical'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open -- cancellation on colocated table population SELECT citus.mitmproxy('conn.onQuery(query="worker_split_copy\(302").cancel(' || :pid || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_truncate.out b/src/test/regress/expected/failure_truncate.out index 4e332252e..253314ee9 100644 --- a/src/test/regress/expected/failure_truncate.out +++ b/src/test/regress/expected/failure_truncate.out @@ -43,7 +43,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -152,7 +152,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -414,7 +414,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()'); (1 row) TRUNCATE reference_table CASCADE; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -553,7 +553,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -662,7 +662,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -922,7 +922,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -1031,7 +1031,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_vacuum.out b/src/test/regress/expected/failure_vacuum.out index 617d40d3a..b438f413b 100644 --- a/src/test/regress/expected/failure_vacuum.out +++ b/src/test/regress/expected/failure_vacuum.out @@ -30,7 +30,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()'); (1 row) VACUUM vacuum_test; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()'); mitmproxy --------------------------------------------------------------------- @@ -38,7 +38,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()'); (1 row) ANALYZE vacuum_test; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SET client_min_messages TO ERROR; SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); mitmproxy @@ -113,7 +113,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()'); (1 row) VACUUM vacuum_test, other_vacuum_test; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_backend_pid() || ')'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/isolation_update_node.out b/src/test/regress/expected/isolation_update_node.out index 53d792e61..703fcc427 100644 --- a/src/test/regress/expected/isolation_update_node.out +++ b/src/test/regress/expected/isolation_update_node.out @@ -250,7 +250,7 @@ count step s1-commit-prepared: COMMIT prepared 'label'; -s2: WARNING: connection to the remote node non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: +s2: WARNING: connection to the remote node postgres@non-existent:57637 failed with the following error: could not translate host name "non-existent" to address: step s2-execute-prepared: EXECUTE foo; diff --git a/src/test/regress/expected/local_shard_execution.out b/src/test/regress/expected/local_shard_execution.out index f6e4db7ee..58293a2d6 100644 --- a/src/test/regress/expected/local_shard_execution.out +++ b/src/test/regress/expected/local_shard_execution.out @@ -3281,9 +3281,9 @@ SELECT pg_sleep(0.1); -- wait to make sure the config has changed before running SET citus.enable_local_execution TO false; -- force a connection to the dummy placements -- run queries that use dummy placements for local execution SELECT * FROM event_responses WHERE FALSE; -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: WITH cte_1 AS (SELECT * FROM event_responses LIMIT 1) SELECT count(*) FROM cte_1; -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: ALTER SYSTEM RESET citus.local_hostname; SELECT pg_reload_conf(); pg_reload_conf diff --git a/src/test/regress/expected/local_shard_execution_0.out b/src/test/regress/expected/local_shard_execution_0.out index 8c4fbfd74..948941aad 100644 --- a/src/test/regress/expected/local_shard_execution_0.out +++ b/src/test/regress/expected/local_shard_execution_0.out @@ -3281,9 +3281,9 @@ SELECT pg_sleep(0.1); -- wait to make sure the config has changed before running SET citus.enable_local_execution TO false; -- force a connection to the dummy placements -- run queries that use dummy placements for local execution SELECT * FROM event_responses WHERE FALSE; -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: WITH cte_1 AS (SELECT * FROM event_responses LIMIT 1) SELECT count(*) FROM cte_1; -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: ALTER SYSTEM RESET citus.local_hostname; SELECT pg_reload_conf(); pg_reload_conf diff --git a/src/test/regress/expected/multi_citus_tools.out b/src/test/regress/expected/multi_citus_tools.out index eef7a98ca..b47763686 100644 --- a/src/test/regress/expected/multi_citus_tools.out +++ b/src/test/regress/expected/multi_citus_tools.out @@ -587,7 +587,7 @@ SET client_min_messages TO DEBUG; -- verify that we can create connections only with users with login privileges. SET ROLE role_without_login; SELECT citus_check_connection_to_node('localhost', :worker_1_port); -WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "role_without_login" is not permitted to log in +WARNING: connection to the remote node role_without_login@localhost:xxxxx failed with the following error: FATAL: role "role_without_login" is not permitted to log in citus_check_connection_to_node --------------------------------------------------------------------- f diff --git a/src/test/regress/expected/multi_copy.out b/src/test/regress/expected/multi_copy.out index abd58eb1d..ff4cbdd2c 100644 --- a/src/test/regress/expected/multi_copy.out +++ b/src/test/regress/expected/multi_copy.out @@ -730,7 +730,7 @@ ALTER USER test_user WITH nologin; \c - test_user - :master_port -- reissue copy, and it should fail COPY numbers_hash FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in -- verify shards in the none of the workers as marked invalid SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) @@ -749,7 +749,7 @@ SELECT shardid, shardstate, nodename, nodeport -- try to insert into a reference table copy should fail COPY numbers_reference FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in -- verify shards for reference table are still valid SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) @@ -765,7 +765,7 @@ SELECT shardid, shardstate, nodename, nodeport -- since it can not insert into either copies of a shard. shards are expected to -- stay valid since the operation is rolled back. COPY numbers_hash_other FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in -- verify shards for numbers_hash_other are still valid -- since copy has failed altogether SELECT shardid, shardstate, nodename, nodeport diff --git a/src/test/regress/expected/multi_modifying_xacts.out b/src/test/regress/expected/multi_modifying_xacts.out index 5eba6e21d..29254d59e 100644 --- a/src/test/regress/expected/multi_modifying_xacts.out +++ b/src/test/regress/expected/multi_modifying_xacts.out @@ -1207,15 +1207,15 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes SET search_path TO multi_modifying_xacts; -- should fail since the worker doesn't have test_user anymore INSERT INTO reference_failure_test VALUES (1, '1'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- the same as the above, but wrapped within a transaction BEGIN; INSERT INTO reference_failure_test VALUES (1, '1'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist COMMIT; BEGIN; COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist COMMIT; -- show that no data go through the table and shard states are good SET client_min_messages to 'ERROR'; @@ -1241,7 +1241,7 @@ ORDER BY s.logicalrelid, sp.shardstate; -- any failure rollbacks the transaction BEGIN; COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist ABORT; -- none of placements are invalid after abort SELECT shardid, shardstate, nodename, nodeport @@ -1262,8 +1262,8 @@ ORDER BY shardid, nodeport; -- verify nothing is inserted SELECT count(*) FROM numbers_hash_failure_test; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist count --------------------------------------------------------------------- 0 @@ -1289,7 +1289,7 @@ ORDER BY shardid, nodeport; -- all failures roll back the transaction BEGIN; COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist COMMIT; -- expect none of the placements to be market invalid after commit SELECT shardid, shardstate, nodename, nodeport @@ -1310,8 +1310,8 @@ ORDER BY shardid, nodeport; -- verify no data is inserted SELECT count(*) FROM numbers_hash_failure_test; -WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist count --------------------------------------------------------------------- 0 @@ -1326,7 +1326,7 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes SET search_path TO multi_modifying_xacts; -- fails on all shard placements INSERT INTO numbers_hash_failure_test VALUES (2,2); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +ERROR: connection to the remote node test_user@localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- connect back to the master with the proper user to continue the tests \c - :default_user - :master_port SET search_path TO multi_modifying_xacts; diff --git a/src/test/regress/expected/multi_multiuser_auth.out b/src/test/regress/expected/multi_multiuser_auth.out index 8dd9b8ba7..92df674b9 100644 --- a/src/test/regress/expected/multi_multiuser_auth.out +++ b/src/test/regress/expected/multi_multiuser_auth.out @@ -82,7 +82,7 @@ GRANT ALL ON TABLE lineitem, orders, lineitem, customer, nation, part, supplier \c :alice_conninfo -- router query (should break because of bad password) INSERT INTO customer VALUES (12345, 'name', NULL, 5, 'phone', 123.45, 'segment', 'comment'); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: password authentication failed for user "alice" +ERROR: connection to the remote node alice@localhost:xxxxx failed with the following error: FATAL: password authentication failed for user "alice" -- fix alice's worker1 password ... UPDATE pg_dist_authinfo SET authinfo = ('password=' || :'alice_worker_1_pw') diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index 14f0bee83..fee821a7d 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -2703,10 +2703,10 @@ SET search_path TO multi_router_planner; -- still, we never mark placements inactive. Instead, fail the transaction BEGIN; INSERT INTO failure_test VALUES (1, 1); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist +ERROR: connection to the remote node router_user@localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist ROLLBACK; INSERT INTO failure_test VALUES (2, 1); -ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist +ERROR: connection to the remote node router_user@localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement WHERE shardid IN ( SELECT shardid FROM pg_dist_shard diff --git a/src/test/regress/expected/node_conninfo_reload.out b/src/test/regress/expected/node_conninfo_reload.out index 01fb5c878..3b33c54b2 100644 --- a/src/test/regress/expected/node_conninfo_reload.out +++ b/src/test/regress/expected/node_conninfo_reload.out @@ -47,7 +47,7 @@ show citus.node_conninfo; -- Should give a connection error because of bad sslmode select count(*) from test where a = 0; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -118,7 +118,7 @@ select count(*) from test where a = 0; COMMIT; -- Should fail now with connection error, when transaction is finished select count(*) from test where a = 0; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -181,7 +181,7 @@ COMMIT; -- Should fail now, when transaction is finished SET client_min_messages TO ERROR; select count(*) from test where a = 0; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" RESET client_min_messages; -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; @@ -235,11 +235,11 @@ show citus.node_conninfo; -- Should fail since a different shard is accessed and thus a new connection -- will to be created. select count(*) from test where a = 0; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" COMMIT; -- Should still fail now, when transaction is finished select count(*) from test where a = 0; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -301,7 +301,7 @@ COMMIT; -- Should fail now, when transaction is finished SET client_min_messages TO ERROR; select count(*) from test; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" RESET client_min_messages; -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; @@ -359,7 +359,7 @@ ROLLBACK; -- Should fail now, when transaction is finished SET client_min_messages TO ERROR; select count(*) from test; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" RESET client_min_messages; -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; @@ -497,7 +497,7 @@ ALTER TABLE test ADD COLUMN c INT; COMMIT; -- Should fail now, when transaction is finished ALTER TABLE test ADD COLUMN d INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); diff --git a/src/test/regress/expected/shard_rebalancer.out b/src/test/regress/expected/shard_rebalancer.out index 7997b5e28..bae42b558 100644 --- a/src/test/regress/expected/shard_rebalancer.out +++ b/src/test/regress/expected/shard_rebalancer.out @@ -127,7 +127,7 @@ SELECT pg_sleep(.1); -- wait to make sure the config has changed before running (1 row) SELECT master_drain_node('localhost', :master_port); -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: CALL citus_cleanup_orphaned_resources(); ALTER SYSTEM RESET citus.local_hostname; SELECT pg_reload_conf(); @@ -197,7 +197,7 @@ SELECT pg_sleep(.1); -- wait to make sure the config has changed before running (1 row) SELECT replicate_table_shards('dist_table_test_2', max_shard_copies := 4, shard_transfer_mode:='block_writes'); -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: ALTER SYSTEM RESET citus.local_hostname; SELECT pg_reload_conf(); pg_reload_conf @@ -681,7 +681,7 @@ FROM ( FROM pg_dist_shard WHERE logicalrelid = 'rebalance_test_table'::regclass ) T; -ERROR: connection to the remote node foobar:57636 failed with the following error: could not translate host name "foobar" to address: +ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: CALL citus_cleanup_orphaned_resources(); ALTER SYSTEM RESET citus.local_hostname; SELECT pg_reload_conf(); diff --git a/src/test/regress/sql/failure_on_create_subscription.sql b/src/test/regress/sql/failure_on_create_subscription.sql index 3a0ae3b5e..60af71e47 100644 --- a/src/test/regress/sql/failure_on_create_subscription.sql +++ b/src/test/regress/sql/failure_on_create_subscription.sql @@ -34,9 +34,9 @@ SELECT * FROM shards_in_workers; -- Failure on creating the subscription -- Failing exactly on CREATE SUBSCRIPTION is causing flaky test where we fail with either: --- 1) ERROR: connection to the remote node localhost:xxxxx failed with the following error: ERROR: subscription "citus_shard_move_subscription_xxxxxxx" does not exist +-- 1) ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: ERROR: subscription "citus_shard_move_subscription_xxxxxxx" does not exist -- another command is already in progress --- 2) ERROR: connection to the remote node localhost:xxxxx failed with the following error: another command is already in progress +-- 2) ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: another command is already in progress -- Instead fail on the next step (ALTER SUBSCRIPTION) instead which is also required logically as part of uber CREATE SUBSCRIPTION operation. SELECT citus.mitmproxy('conn.onQuery(query="ALTER SUBSCRIPTION").kill()'); From 38967491efee6701e8f5585f76b09ab09964183d Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Tue, 16 Apr 2024 21:29:14 +0800 Subject: [PATCH 036/155] Add missing volatile qualifier. (#7570) Variables being modified in the PG_TRY block and read in the PG_CATCH block should be qualified with volatile. The variable waitEventSet is modified in the PG_TRY block (line 1085) and read in the PG_CATCH block (line 1095). The variable relation is modified in the PG_TRY block (line 500) and read in the PG_CATCH block (line 515). Besides, the variable objectAddress doesn't need the volatile qualifier. Ref: C99 7.13.2.1[^1], > All accessible objects have values, and all other components of the abstract machine have state, as of the time the longjmp function was called, except that the values of objects of automatic storage duration that are local to the function containing the invocation of the corresponding setjmp macro that do not have volatile-qualified type and have been changed between the setjmp invocation and longjmp call are indeterminate. [^1]: https://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf DESCRIPTION: Correctly mark some variables as volatile --------- Co-authored-by: Hong Yi (cherry picked from commit ada3ba25072cc5be055b3bbdedfa2fe936443b0d) --- .../distributed/connection/remote_commands.c | 2 +- .../distributed/utils/citus_depended_object.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index 4b46e96d2..cbd74ff51 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -883,7 +883,7 @@ WaitForAllConnections(List *connectionList, bool raiseInterrupts) palloc(totalConnectionCount * sizeof(MultiConnection *)); WaitEvent *events = palloc(totalConnectionCount * sizeof(WaitEvent)); bool *connectionReady = palloc(totalConnectionCount * sizeof(bool)); - WaitEventSet *waitEventSet = NULL; + WaitEventSet *volatile waitEventSet = NULL; /* convert connection list to an array such that we can move items around */ MultiConnection *connectionItem = NULL; diff --git a/src/backend/distributed/utils/citus_depended_object.c b/src/backend/distributed/utils/citus_depended_object.c index a160fcd56..7588f8594 100644 --- a/src/backend/distributed/utils/citus_depended_object.c +++ b/src/backend/distributed/utils/citus_depended_object.c @@ -465,8 +465,8 @@ static bool AnyObjectViolatesOwnership(DropStmt *dropStmt) { bool hasOwnershipViolation = false; - volatile ObjectAddress objectAddress = { 0 }; - Relation relation = NULL; + ObjectAddress objectAddress = { 0 }; + volatile Relation relation = NULL; ObjectType objectType = dropStmt->removeType; bool missingOk = dropStmt->missing_ok; @@ -480,8 +480,17 @@ AnyObjectViolatesOwnership(DropStmt *dropStmt) Node *object = NULL; foreach_ptr(object, dropStmt->objects) { + Relation rel = NULL; objectAddress = get_object_address(objectType, object, - &relation, AccessShareLock, missingOk); + &rel, AccessShareLock, missingOk); + + /* + * The object relation is qualified with volatile and its value is obtained from + * get_object_address(). Unless we can qualify the corresponding parameter of + * get_object_address() with volatile (this is a function defined in PostgreSQL), + * we cannot get rid of this assignment. + */ + relation = rel; if (OidIsValid(objectAddress.objectId)) { From bac95cc523c3f42f262b3e83b43214506d395e19 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 16 Apr 2024 17:26:12 +0200 Subject: [PATCH 037/155] Greatly speed up "\d tablename" on servers with many tables (#7577) DESCRIPTION: Fix performance issue when using "\d tablename" on a server with many tables We introduce a filter to every query on pg_class to automatically remove shards. This is useful to make sure \d and PgAdmin are not cluttered with shards. However, the way we were introducing this filter was using `securityQuals` which can have negative impact on query performance. On clusters with 100k+ tables this could cause a simple "\d tablename" command to take multiple seconds, because a skipped optimization by Postgres causes a full table scan. This changes the code to introduce this filter in the regular `quals` list instead of in `securityQuals`. Which causes Postgres to use the intended optimization again. For reference, this was initially reported as a Postgres issue by me: https://www.postgresql.org/message-id/flat/4189982.1712785863%40sss.pgh.pa.us#b87421293b362d581ea8677e3bfea920 (cherry picked from commit a0151aa31d7584187a50dfbc19b28bccc76cb616) --- .../worker/worker_shard_visibility.c | 65 ++++++++++++++++--- .../expected/multi_mx_hide_shard_names.out | 46 +++++++++++++ .../regress/sql/multi_mx_hide_shard_names.sql | 18 +++++ 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index 49131ef6d..ccd1a897c 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -54,6 +54,7 @@ static bool ShouldHideShardsInternal(void); static bool IsPgBgWorker(void); static bool FilterShardsFromPgclass(Node *node, void *context); static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno); +static bool HasRangeTableRef(Node *node, int *varno); PG_FUNCTION_INFO_V1(citus_table_is_visible); PG_FUNCTION_INFO_V1(relation_is_a_known_shard); @@ -421,8 +422,8 @@ IsPgBgWorker(void) /* - * FilterShardsFromPgclass adds a NOT relation_is_a_known_shard(oid) filter - * to the security quals of pg_class RTEs. + * FilterShardsFromPgclass adds a "relation_is_a_known_shard(oid) IS NOT TRUE" + * filter to the quals of queries that query pg_class. */ static bool FilterShardsFromPgclass(Node *node, void *context) @@ -456,12 +457,35 @@ FilterShardsFromPgclass(Node *node, void *context) continue; } + /* + * Skip if pg_class is not actually queried. This is possible on + * INSERT statements that insert into pg_class. + */ + if (!expression_tree_walker((Node *) query->jointree->fromlist, + HasRangeTableRef, &varno)) + { + /* the query references pg_class */ + continue; + } + /* make sure the expression is in the right memory context */ MemoryContext originalContext = MemoryContextSwitchTo(queryContext); - /* add NOT relation_is_a_known_shard(oid) to the security quals of the RTE */ - rangeTableEntry->securityQuals = - list_make1(CreateRelationIsAKnownShardFilter(varno)); + + /* add relation_is_a_known_shard(oid) IS NOT TRUE to the quals of the query */ + Node *newQual = CreateRelationIsAKnownShardFilter(varno); + Node *oldQuals = query->jointree->quals; + if (oldQuals) + { + query->jointree->quals = (Node *) makeBoolExpr( + AND_EXPR, + list_make2(oldQuals, newQual), + -1); + } + else + { + query->jointree->quals = newQual; + } MemoryContextSwitchTo(originalContext); } @@ -473,9 +497,32 @@ FilterShardsFromPgclass(Node *node, void *context) } +/* + * HasRangeTableRef passed to expression_tree_walker to check if a node is a + * RangeTblRef of the given varno is present in a fromlist. + */ +static bool +HasRangeTableRef(Node *node, int *varno) +{ + if (IsA(node, RangeTblRef)) + { + RangeTblRef *rangeTblRef = (RangeTblRef *) node; + return rangeTblRef->rtindex == *varno; + } + + return expression_tree_walker(node, HasRangeTableRef, varno); +} + + /* * CreateRelationIsAKnownShardFilter constructs an expression of the form: - * NOT pg_catalog.relation_is_a_known_shard(oid) + * pg_catalog.relation_is_a_known_shard(oid) IS NOT TRUE + * + * The difference between "NOT pg_catalog.relation_is_a_known_shard(oid)" and + * "pg_catalog.relation_is_a_known_shard(oid) IS NOT TRUE" is that the former + * will return FALSE if the function returns NULL, while the second will return + * TRUE. This difference is important in the case of outer joins, because this + * filter might be applied on an oid that is then NULL. */ static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno) @@ -496,9 +543,9 @@ CreateRelationIsAKnownShardFilter(int pgClassVarno) funcExpr->location = -1; funcExpr->args = list_make1(oidVar); - BoolExpr *notExpr = makeNode(BoolExpr); - notExpr->boolop = NOT_EXPR; - notExpr->args = list_make1(funcExpr); + BooleanTest *notExpr = makeNode(BooleanTest); + notExpr->booltesttype = IS_NOT_TRUE; + notExpr->arg = (Expr *) funcExpr; notExpr->location = -1; return (Node *) notExpr; diff --git a/src/test/regress/expected/multi_mx_hide_shard_names.out b/src/test/regress/expected/multi_mx_hide_shard_names.out index 116269a4e..762c6a30b 100644 --- a/src/test/regress/expected/multi_mx_hide_shard_names.out +++ b/src/test/regress/expected/multi_mx_hide_shard_names.out @@ -83,6 +83,52 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name test_table (1 row) +-- Even when using subquery and having no existing quals on pg_clcass +SELECT relname FROM (SELECT relname, relnamespace FROM pg_catalog.pg_class) AS q WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + relname +--------------------------------------------------------------------- + test_table +(1 row) + +-- Check that inserts into pg_class don't add the filter +EXPLAIN (COSTS OFF) INSERT INTO pg_class VALUES (1); + QUERY PLAN +--------------------------------------------------------------------- + Insert on pg_class + -> Result +(2 rows) + +-- Unless it's an INSERT SELECT that queries from pg_class; +EXPLAIN (COSTS OFF) INSERT INTO pg_class SELECT * FROM pg_class; + QUERY PLAN +--------------------------------------------------------------------- + Insert on pg_class + -> Seq Scan on pg_class pg_class_1 + Filter: (relation_is_a_known_shard(oid) IS NOT TRUE) +(3 rows) + +-- Check that query that psql "\d test_table" does gets optimized to an index +-- scan +EXPLAIN (COSTS OFF) SELECT c.oid, + n.nspname, + c.relname +FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE c.relname OPERATOR(pg_catalog.~) '^(test_table)$' COLLATE pg_catalog.default + AND pg_catalog.pg_table_is_visible(c.oid) +ORDER BY 2, 3; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: n.nspname, c.relname + -> Nested Loop Left Join + Join Filter: (n.oid = c.relnamespace) + -> Index Scan using pg_class_relname_nsp_index on pg_class c + Index Cond: (relname = 'test_table'::text) + Filter: ((relname ~ '^(test_table)$'::text) AND (relation_is_a_known_shard(oid) IS NOT TRUE) AND pg_table_is_visible(oid)) + -> Seq Scan on pg_namespace n +(8 rows) + commit prepared 'take-aggressive-lock'; -- now create an index \c - - - :master_port diff --git a/src/test/regress/sql/multi_mx_hide_shard_names.sql b/src/test/regress/sql/multi_mx_hide_shard_names.sql index e5213a41b..addc7f90e 100644 --- a/src/test/regress/sql/multi_mx_hide_shard_names.sql +++ b/src/test/regress/sql/multi_mx_hide_shard_names.sql @@ -50,6 +50,24 @@ prepare transaction 'take-aggressive-lock'; -- shards are hidden when using psql as application_name SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +-- Even when using subquery and having no existing quals on pg_clcass +SELECT relname FROM (SELECT relname, relnamespace FROM pg_catalog.pg_class) AS q WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; + +-- Check that inserts into pg_class don't add the filter +EXPLAIN (COSTS OFF) INSERT INTO pg_class VALUES (1); +-- Unless it's an INSERT SELECT that queries from pg_class; +EXPLAIN (COSTS OFF) INSERT INTO pg_class SELECT * FROM pg_class; + +-- Check that query that psql "\d test_table" does gets optimized to an index +-- scan +EXPLAIN (COSTS OFF) SELECT c.oid, + n.nspname, + c.relname +FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE c.relname OPERATOR(pg_catalog.~) '^(test_table)$' COLLATE pg_catalog.default + AND pg_catalog.pg_table_is_visible(c.oid) +ORDER BY 2, 3; commit prepared 'take-aggressive-lock'; From 40e9e2614d1297a889eb603bca37527e6f36d48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Tue, 19 Sep 2023 14:37:35 +0300 Subject: [PATCH 038/155] Removes centos 7 for PG 16 in packaging pipelines (#7205) centos 7 and oracle 7 is not being supported for newer releases by Postgres. Therefore, getting package download errors in packaging pipelines. This PR removes el/7 and ol/7 Postgres 16 pipelines (cherry picked from commit b0e982d0b5e9b3f3cb1ca526ae8388b2576e6734) --- .github/workflows/packaging-test-pipelines.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 23ed39bba..885ad4def 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -46,12 +46,22 @@ jobs: # For this reason, we need to use a "matrix" to generate names of # rpm images, e.g. citus/packaging:centos-7-pg12 packaging_docker_image: - - oraclelinux-7 - oraclelinux-8 - - centos-7 - almalinux-8 - almalinux-9 POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }} + # Postgres removed support for CentOS 7 in PG 16. Below block is needed to + # keep the build for CentOS 7 working for PG 14 and PG 15. + # Once dependent systems drop support for Centos 7, we can remove this block. + include: + - packaging_docker_image: centos-7 + POSTGRES_VERSION: 14 + - packaging_docker_image: centos-7 + POSTGRES_VERSION: 15 + - packaging_docker_image: oraclelinux-7 + POSTGRES_VERSION: 14 + - packaging_docker_image: oraclelinux-7 + POSTGRES_VERSION: 15 container: image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }} From 75ff237340398d70e08d271ac6cdcc2ac29223e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 17 Nov 2023 08:51:56 +0300 Subject: [PATCH 039/155] Removes unnecessary package installations in packaging pipelines (#7341) With the recent changes in packaging images, linux package installations to execute validate_output is unnecessary now. In this PR, I removed them to make the pipeline more effective. - [x] Remove the test warning before merge (cherry picked from commit 32b0fc23f5b5f439f62defbd80cb09b5294e97ac) --- .github/packaging/validate_build_output.sh | 5 ++++- .github/workflows/packaging-test-pipelines.yml | 10 ---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/packaging/validate_build_output.sh b/.github/packaging/validate_build_output.sh index 64098811e..dab301aa5 100755 --- a/.github/packaging/validate_build_output.sh +++ b/.github/packaging/validate_build_output.sh @@ -32,7 +32,10 @@ python3 -m pip install -r tools/packaging_automation/requirements.txt echo "Package type: ${package_type}" echo "OS version: $(get_rpm_os_version)" - # if os version is centos 7 or oracle linux 7, then remove urllib3 with pip uninstall and install urllib3<2.0.0 with pip install + # For RHEL 7, we need to install urllib3<2 due to below execution error + # ImportError: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' + # module is compiled with 'OpenSSL 1.0.2k-fips 26 Jan 2017'. + # See: https://github.com/urllib3/urllib3/issues/2168 if [[ ${package_type} == "rpm" && $(get_rpm_os_version) == 7* ]]; then python3 -m pip uninstall -y urllib3 python3 -m pip install 'urllib3<2' diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 885ad4def..fec0fefa9 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -111,11 +111,6 @@ jobs: PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }} run: | echo "Postgres version: ${POSTGRES_VERSION}" - - ## Install required packages to execute packaging tools for rpm based distros - yum install python3-pip python3-devel postgresql-devel -y || true - python3 -m pip install wheel - ./.github/packaging/validate_build_output.sh "rpm" deb_build_tests: @@ -191,9 +186,4 @@ jobs: PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }} run: | echo "Postgres version: ${POSTGRES_VERSION}" - - apt-get update -y - ## Install required packages to execute packaging tools for deb based distros - apt-get install python3-dev python3-pip -y - apt-get purge -y python3-yaml ./.github/packaging/validate_build_output.sh "deb" From 035aa6eada5a1d35454d008711b9b75341d942da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 24 Apr 2024 11:15:04 +0300 Subject: [PATCH 040/155] Bump Citus version to 12.1.3 (#7588) --- CHANGELOG.md | 40 +++++++++++++++++++ configure | 18 ++++----- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b982af3..b5a0c548f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,43 @@ +### citus v12.1.3 (April 18, 2024) ### + +* Allows overwriting host name for all inter-node connections by + supporting "host" parameter in citus.node_conninfo (#7541) + +* Changes the order in which the locks are acquired for the target and + reference tables, when a modify request is initiated from a worker + node that is not the "FirstWorkerNode" (#7542) + +* Fixes a performance issue when distributing a table that depends on an + extension (#7574) + +* Fixes a performance issue when using "\d tablename" on a server with + many tables (#7577) + +* Fixes a crash caused by some form of ALTER TABLE ADD COLUMN + statements. When adding multiple columns, if one of the ADD COLUMN + statements contains a FOREIGN constraint omitting the referenced + columns in the statement, a SEGFAULT was occurring. (#7522) + +* Fixes a performance issue when creating distributed tables if many + already exist (#7575, #7579) + +* Fixes a bug when hostname in pg_dist_node resolves to multiple IPs + (#7377) + +* Fixes performance issue when tracking foreign key constraints on + systems with many constraints (#7578) + +* Fixes segmentation fault when using CASE WHEN in DO block within + functions. (#7554) + +* Fixes undefined behavior in master_disable_node due to argument + mismatch (#7492) + +* Fixes some potential bugs by correctly marking some variables as + volatile (#7570) + +* Logs username in the failed connection message (#7432) + ### citus v12.1.2 (February 12, 2024) ### * Fixes the incorrect column count after ALTER TABLE (#7379) diff --git a/configure b/configure index 615ca97c5..06eb870e8 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.2. +# Generated by GNU Autoconf 2.69 for Citus 12.1.3. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.2' -PACKAGE_STRING='Citus 12.1.2' +PACKAGE_VERSION='12.1.3' +PACKAGE_STRING='Citus 12.1.3' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.2 to adapt to many kinds of systems. +\`configure' configures Citus 12.1.3 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.2:";; + short | recursive ) echo "Configuration of Citus 12.1.3:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.2 +Citus configure 12.1.3 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.2, which was +It was created by Citus $as_me 12.1.3, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.2, which was +This file was extended by Citus $as_me 12.1.3, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.2 +Citus config.status 12.1.3 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 0a198b007..e635fc3ee 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.2]) +AC_INIT([Citus], [12.1.3]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 108eebf36..555b4d85b 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.2 + 12.1.3 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 4e838a471a0ca49e1c2075361332e2b67f1e426c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Tue, 28 May 2024 08:54:40 +0300 Subject: [PATCH 041/155] Adds null check for node in HasRangeTableRef (#7604) DESCRIPTION: Adds null check for node in HasRangeTableRef to prevent errors When executing the query below, users encountered an error due to a null Node object. This PR adds a null check to handle this error. Query: ```sql select ct.conname as constraint_name, a.attname as column_name, fc.relname as foreign_table_name, fns.nspname as foreign_table_schema, fa.attname as foreign_column_name from (SELECT ct.conname, ct.conrelid, ct.confrelid, ct.conkey, ct.contype, ct.confkey, generate_subscripts(ct.conkey, 1) AS s FROM pg_constraint ct ) AS ct inner join pg_class c on c.oid=ct.conrelid inner join pg_namespace ns on c.relnamespace=ns.oid inner join pg_attribute a on a.attrelid=ct.conrelid and a.attnum = ct.conkey[ct.s] left join pg_class fc on fc.oid=ct.confrelid left join pg_namespace fns on fc.relnamespace=fns.oid left join pg_attribute fa on fa.attrelid=ct.confrelid and fa.attnum = ct.confkey[ct.s] where ct.contype='f' and c.relname='table1' and ns.nspname='schemauser' order by fns.nspname, fc.relname, a.attnum ; ``` Error: ``` #0 HasRangeTableRef (node=0x0, varno=varno@entry=0x7ffe18cc3674) at worker/worker_shard_visibility.c:507 507 if (IsA(node, RangeTblRef)) #0 HasRangeTableRef (node=0x0, varno=varno@entry=0x7ffe18cc3674) at worker/worker_shard_visibility.c:507 #1 0x0000561b0aae390e in expression_tree_walker_impl (node=0x561b0d19cc78, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2091 #2 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #3 0x0000561b0aae3e09 in expression_tree_walker_impl (node=0x561b0d19cd68, walker=walker@entry=0x7f2a73249f0a , context=context@entry=0x7ffe18cc3674) at nodeFuncs.c:2405 #4 0x0000561b0aae3945 in expression_tree_walker_impl (node=0x561b0d19d0f8, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2111 #5 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #6 0x0000561b0aae3e09 in expression_tree_walker_impl (node=0x561b0d19cb38, walker=walker@entry=0x7f2a73249f0a , context=context@entry=0x7ffe18cc3674) at nodeFuncs.c:2405 #7 0x0000561b0aae396d in expression_tree_walker_impl (node=0x561b0d19d198, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2127 #8 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #9 0x0000561b0aae3ef7 in expression_tree_walker_impl (node=0x561b0d183e88, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2464 #10 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #11 0x0000561b0aae3ed3 in expression_tree_walker_impl (node=0x561b0d184278, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2460 #12 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #13 0x0000561b0aae3ed3 in expression_tree_walker_impl (node=0x561b0d184668, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2460 #14 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #15 0x0000561b0aae3ed3 in expression_tree_walker_impl (node=0x561b0d184f68, walker=walker@entry=0x7f2a73249f0a , context=0x7ffe18cc3674) at nodeFuncs.c:2460 #16 0x00007f2a73249f26 in HasRangeTableRef (node=, varno=) at worker/worker_shard_visibility.c:513 #17 0x0000561b0aae3e09 in expression_tree_walker_impl (node=0x7f2a68010148, walker=walker@entry=0x7f2a73249f0a , context=context@entry=0x7ffe18cc3674) at nodeFuncs.c:2405 #18 0x00007f2a7324a0eb in FilterShardsFromPgclass (node=node@entry=0x561b0d185de8, context=context@entry=0x0) at worker/worker_shard_visibility.c:464 #19 0x00007f2a7324a5ff in HideShardsFromSomeApplications (query=query@entry=0x561b0d185de8) at worker/worker_shard_visibility.c:294 #20 0x00007f2a731ed7ac in distributed_planner (parse=0x561b0d185de8, query_string=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"..., cursorOptions=, boundParams=0x0) at planner/distributed_planner.c:237 #21 0x00007f2a7311a52a in pgss_planner (parse=0x561b0d185de8, query_string=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"..., cursorOptions=2048, boundParams=0x0) at pg_stat_statements.c:953 #22 0x0000561b0ab65465 in planner (parse=parse@entry=0x561b0d185de8, query_string=query_string@entry=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"..., cursorOptions=cursorOptions@entry=2048, boundParams=boundParams@entry=0x0) at planner.c:279 #23 0x0000561b0ac53aa3 in pg_plan_query (querytree=querytree@entry=0x561b0d185de8, query_string=query_string@entry=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"..., cursorOptions=cursorOptions@entry=2048, boundParams=boundParams@entry=0x0) at postgres.c:904 #24 0x0000561b0ac53b71 in pg_plan_queries (querytrees=0x7f2a68012878, query_string=query_string@entry=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"..., cursorOptions=cursorOptions@entry=2048, boundParams=boundParams@entry=0x0) at postgres.c:996 #25 0x0000561b0ac5408e in exec_simple_query ( query_string=query_string@entry=0x561b0d009478 "select\n ct.conname as constraint_name,\n a.attname as column_name,\n fc.relname as foreign_table_name,\n fns.nspname as foreign_table_schema,\n fa.attname as foreign_column_name\nfrom\n (S"...) at postgres.c:1193 #26 0x0000561b0ac56116 in PostgresMain (dbname=, username=) at postgres.c:4637 #27 0x0000561b0abab7a7 in BackendRun (port=port@entry=0x561b0d0caf50) at postmaster.c:4464 #28 0x0000561b0abae969 in BackendStartup (port=port@entry=0x561b0d0caf50) at postmaster.c:4192 #29 0x0000561b0abaeaa6 in ServerLoop () at postmaster.c:1782 ``` Fixes #7603 --- .../worker/worker_shard_visibility.c | 5 +++ src/test/regress/expected/system_queries.out | 33 +++++++++++++++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/system_queries.sql | 27 +++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/system_queries.out create mode 100644 src/test/regress/sql/system_queries.sql diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index ccd1a897c..3725800c3 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -504,6 +504,11 @@ FilterShardsFromPgclass(Node *node, void *context) static bool HasRangeTableRef(Node *node, int *varno) { + if (node == NULL) + { + return false; + } + if (IsA(node, RangeTblRef)) { RangeTblRef *rangeTblRef = (RangeTblRef *) node; diff --git a/src/test/regress/expected/system_queries.out b/src/test/regress/expected/system_queries.out new file mode 100644 index 000000000..cd2aef4d2 --- /dev/null +++ b/src/test/regress/expected/system_queries.out @@ -0,0 +1,33 @@ +-- The following query retrieves the foreign key constraints of the table "pg_dist_background_job" +-- along with their details. This modification includes a fix for a null pointer exception that occurred +-- in the "HasRangeTableRef" method of "worker_shard_visibility". The issue was resolved with PR #7604. +select + ct.conname as constraint_name, + a.attname as column_name, + fc.relname as foreign_table_name, + fns.nspname as foreign_table_schema +from + (SELECT ct.conname, ct.conrelid, ct.confrelid, ct.conkey, ct.contype, +ct.confkey, generate_subscripts(ct.conkey, 1) AS s + FROM pg_constraint ct + ) AS ct + inner join pg_class c on c.oid=ct.conrelid + inner join pg_namespace ns on c.relnamespace=ns.oid + inner join pg_attribute a on a.attrelid=ct.conrelid and a.attnum = +ct.conkey[ct.s] + left join pg_class fc on fc.oid=ct.confrelid + left join pg_namespace fns on fc.relnamespace=fns.oid + left join pg_attribute fa on fa.attrelid=ct.confrelid and fa.attnum = +ct.confkey[ct.s] +where + ct.contype='f' + and fc.relname='pg_dist_background_job' + and ns.nspname='pg_catalog' +order by + fns.nspname, fc.relname, a.attnum; + constraint_name | column_name | foreign_table_name | foreign_table_schema +--------------------------------------------------------------------- + pg_dist_background_task_job_id_fkey | job_id | pg_dist_background_job | pg_catalog + pg_dist_background_task_depend_job_id_fkey | job_id | pg_dist_background_job | pg_catalog +(2 rows) + diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index bc3477ce2..54d65686c 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -79,7 +79,7 @@ test: multi_basic_queries cross_join multi_complex_expressions multi_subquery mu test: multi_subquery_complex_reference_clause multi_subquery_window_functions multi_view multi_sql_function multi_prepare_sql test: sql_procedure multi_function_in_join row_types materialized_view test: multi_subquery_in_where_reference_clause adaptive_executor propagate_set_commands geqo -test: forcedelegation_functions +test: forcedelegation_functions system_queries # this should be run alone as it gets too many clients test: join_pushdown test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc statement_cancel_error_message diff --git a/src/test/regress/sql/system_queries.sql b/src/test/regress/sql/system_queries.sql new file mode 100644 index 000000000..1e1d86876 --- /dev/null +++ b/src/test/regress/sql/system_queries.sql @@ -0,0 +1,27 @@ +-- The following query retrieves the foreign key constraints of the table "pg_dist_background_job" +-- along with their details. This modification includes a fix for a null pointer exception that occurred +-- in the "HasRangeTableRef" method of "worker_shard_visibility". The issue was resolved with PR #7604. +select + ct.conname as constraint_name, + a.attname as column_name, + fc.relname as foreign_table_name, + fns.nspname as foreign_table_schema +from + (SELECT ct.conname, ct.conrelid, ct.confrelid, ct.conkey, ct.contype, +ct.confkey, generate_subscripts(ct.conkey, 1) AS s + FROM pg_constraint ct + ) AS ct + inner join pg_class c on c.oid=ct.conrelid + inner join pg_namespace ns on c.relnamespace=ns.oid + inner join pg_attribute a on a.attrelid=ct.conrelid and a.attnum = +ct.conkey[ct.s] + left join pg_class fc on fc.oid=ct.confrelid + left join pg_namespace fns on fc.relnamespace=fns.oid + left join pg_attribute fa on fa.attrelid=ct.confrelid and fa.attnum = +ct.confkey[ct.s] +where + ct.contype='f' + and fc.relname='pg_dist_background_job' + and ns.nspname='pg_catalog' +order by + fns.nspname, fc.relname, a.attnum; From 7e0dc18b22b71b16a3b418d7c2c2d86d65bff8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 29 May 2024 11:35:08 +0300 Subject: [PATCH 042/155] Bump Citus version to 12.1.4 (#7610) --- CHANGELOG.md | 4 ++++ configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a0c548f..edf338f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### citus v12.1.4 (May 28, 2024) ### + +* Adds null check for node in HasRangeTableRef (#7604) + ### citus v12.1.3 (April 18, 2024) ### * Allows overwriting host name for all inter-node connections by diff --git a/configure b/configure index 06eb870e8..1a20dcded 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.3. +# Generated by GNU Autoconf 2.69 for Citus 12.1.4. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.3' -PACKAGE_STRING='Citus 12.1.3' +PACKAGE_VERSION='12.1.4' +PACKAGE_STRING='Citus 12.1.4' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.3 to adapt to many kinds of systems. +\`configure' configures Citus 12.1.4 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.3:";; + short | recursive ) echo "Configuration of Citus 12.1.4:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.3 +Citus configure 12.1.4 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.3, which was +It was created by Citus $as_me 12.1.4, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.3, which was +This file was extended by Citus $as_me 12.1.4, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.3 +Citus config.status 12.1.4 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e635fc3ee..79ff5e430 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.3]) +AC_INIT([Citus], [12.1.4]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 555b4d85b..8131ad7cb 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.3 + 12.1.4 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 3594bd7ac011a1a9ec337eafb8abe0199d486905 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Fri, 14 Jun 2024 16:20:23 +0200 Subject: [PATCH 043/155] Fix CI issues after Github Actions networking changes (#7624) For some reason using localhost in our hba file doesn't have the intended effect anymore in our Github Actions runners. Probably because of some networking change (IPv6 maybe) or some change in the `/etc/hosts` file. Replacing localhost with the equivalent loopback IPv4 and IPv6 addresses resolved this issue. (cherry picked from commit 8c9de08b76332308deb9fd082d0d00f4afba8cd3) --- src/test/regress/pg_regress_multi.pl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 4e7c66d9e..adcb431e1 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -297,10 +297,12 @@ sub generate_hba open(my $fh, ">", catfile($TMP_CHECKDIR, $nodename, "data", "pg_hba.conf")) or die "could not open pg_hba.conf"; - print $fh "host all alice,bob localhost md5\n"; + print $fh "host all alice,bob 127.0.0.1/32 md5\n"; + print $fh "host all alice,bob ::1/128 md5\n"; print $fh "host all all 127.0.0.1/32 trust\n"; print $fh "host all all ::1/128 trust\n"; - print $fh "host replication postgres localhost trust\n"; + print $fh "host replication postgres 127.0.0.1/32 trust\n"; + print $fh "host replication postgres ::1/128 trust\n"; close $fh; } From 4f0053ed6d1453fb77286211f38e959f8c9ed4f4 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Mon, 17 Jun 2024 16:07:25 +0200 Subject: [PATCH 044/155] Redo #7620: Fix merge command when insert value does not have source distributed column (#7627) Related to issue #7619, #7620 Merge command fails when source query is single sharded and source and target are co-located and insert is not using distribution key of source. Example ``` CREATE TABLE source (id integer); CREATE TABLE target (id integer ); -- let's distribute both table on id field SELECT create_distributed_table('source', 'id'); SELECT create_distributed_table('target', 'id'); MERGE INTO target t USING ( SELECT 1 AS somekey FROM source WHERE source.id = 1) s ON t.id = s.somekey WHEN NOT MATCHED THEN INSERT (id) VALUES (s.somekey) ERROR: MERGE INSERT must use the source table distribution column value HINT: MERGE INSERT must use the source table distribution column value ``` Author's Opinion: If join is not between source and target distributed column, we should not force user to use source distributed column while inserting value of target distributed column. Fix: If user is not using distributed key of source for insertion let's not push down query to workers and don't force user to use source distributed column if it is not part of join. This reverts commit fa4fc0b372e4068e069946e3fdf454137736bcc7. Co-authored-by: paragjain (cherry picked from commit aaaf637a6babebc9d9fa181e3a94b68825e2816f) --- .../distributed/planner/merge_planner.c | 29 +- src/test/regress/expected/merge.out | 337 ++++++++++++++++-- src/test/regress/multi_schedule | 3 +- src/test/regress/sql/merge.sql | 133 +++++++ 4 files changed, 470 insertions(+), 32 deletions(-) diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 4d64b8f56..09d2d90ac 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -182,14 +182,6 @@ CreateRouterMergePlan(Oid targetRelationId, Query *originalQuery, Query *query, return distributedPlan; } - Var *insertVar = - FetchAndValidateInsertVarIfExists(targetRelationId, originalQuery); - if (insertVar && - !IsDistributionColumnInMergeSource((Expr *) insertVar, originalQuery, true)) - { - ereport(ERROR, (errmsg("MERGE INSERT must use the source table " - "distribution column value"))); - } Job *job = RouterJob(originalQuery, plannerRestrictionContext, &distributedPlan->planningError); @@ -1124,6 +1116,27 @@ DeferErrorIfRoutableMergeNotSupported(Query *query, List *rangeTableList, "repartitioning"))); return deferredError; } + + + /* + * If execution has reached this point, it indicates that the query can be delegated to the worker. + * However, before proceeding with this delegation, we need to confirm that the user is utilizing + * the distribution column of the source table in the Insert variable. + * If this is not the case, we should refrain from pushing down the query. + * This is just a deffered error which will be handle by caller. + */ + + Var *insertVar = + FetchAndValidateInsertVarIfExists(targetRelationId, query); + if (insertVar && + !IsDistributionColumnInMergeSource((Expr *) insertVar, query, true)) + { + ereport(DEBUG1, (errmsg( + "MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied"))); + return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, + "MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied", + NULL, NULL); + } return NULL; } diff --git a/src/test/regress/expected/merge.out b/src/test/regress/expected/merge.out index a73467e81..5056ba543 100644 --- a/src/test/regress/expected/merge.out +++ b/src/test/regress/expected/merge.out @@ -1128,7 +1128,7 @@ DO NOTHING WHEN NOT MATCHED THEN INSERT VALUES(rs_source.id); DEBUG: Creating MERGE router plan -DEBUG: +DEBUG: RESET client_min_messages; SELECT * INTO rs_local FROM rs_target ORDER BY 1 ; -- Should be equal @@ -1259,7 +1259,7 @@ DO NOTHING WHEN NOT MATCHED THEN INSERT VALUES(fn_source.id, fn_source.source); DEBUG: Creating MERGE router plan -DEBUG: +DEBUG: RESET client_min_messages; SELECT * INTO fn_local FROM fn_target ORDER BY 1 ; -- Should be equal @@ -1552,7 +1552,7 @@ BEGIN; SET citus.log_remote_commands to true; SET client_min_messages TO DEBUG1; EXECUTE merge_prepare(2); -DEBUG: +DEBUG: DEBUG: Creating MERGE router plan NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx @@ -1782,13 +1782,13 @@ NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) +NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) +NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) +NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) +NOTICE: issuing MERGE INTO merge_schema.citus_target_xxxxxxx t USING merge_schema.citus_source_xxxxxxx s ON (t.id OPERATOR(pg_catalog.=) s.id) WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED AND (s.id OPERATOR(pg_catalog.<) 100) THEN INSERT (id, val) VALUES (s.id, s.val) DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands to false; SELECT compare_tables(); @@ -1842,6 +1842,297 @@ SELECT compare_tables(); (1 row) ROLLBACK; +-- let's create source and target table +ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 13000; +CREATE TABLE source_pushdowntest (id integer); +CREATE TABLE target_pushdowntest (id integer ); +-- let's distribute both table on id field +SELECT create_distributed_table('source_pushdowntest', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target_pushdowntest', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- we are doing this operation on single node setup let's figure out colocation id of both tables +-- both has same colocation id so both are colocated. +WITH colocations AS ( + SELECT colocationid + FROM pg_dist_partition + WHERE logicalrelid = 'source_pushdowntest'::regclass + OR logicalrelid = 'target_pushdowntest'::regclass +) +SELECT + CASE + WHEN COUNT(DISTINCT colocationid) = 1 THEN 'Same' + ELSE 'Different' + END AS colocation_status +FROM colocations; + colocation_status +--------------------------------------------------------------------- + Same +(1 row) + +SET client_min_messages TO DEBUG1; +-- Test 1 : tables are colocated AND query is multisharded AND Join On distributed column : should push down to workers. +EXPLAIN (costs off, timing off, summary off) +MERGE INTO target_pushdowntest t +USING source_pushdowntest s +ON t.id = s.id +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.id); +DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: Creating MERGE router plan + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Merge on target_pushdowntest_4000068 t + -> Merge Left Join + Merge Cond: (s.id = t.id) + -> Sort + Sort Key: s.id + -> Seq Scan on source_pushdowntest_4000064 s + -> Sort + Sort Key: t.id + -> Seq Scan on target_pushdowntest_4000068 t + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Merge on target_pushdowntest_4000069 t + -> Merge Left Join + Merge Cond: (s.id = t.id) + -> Sort + Sort Key: s.id + -> Seq Scan on source_pushdowntest_4000065 s + -> Sort + Sort Key: t.id + -> Seq Scan on target_pushdowntest_4000069 t + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Merge on target_pushdowntest_4000070 t + -> Merge Left Join + Merge Cond: (s.id = t.id) + -> Sort + Sort Key: s.id + -> Seq Scan on source_pushdowntest_4000066 s + -> Sort + Sort Key: t.id + -> Seq Scan on target_pushdowntest_4000070 t + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Merge on target_pushdowntest_4000071 t + -> Merge Left Join + Merge Cond: (s.id = t.id) + -> Sort + Sort Key: s.id + -> Seq Scan on source_pushdowntest_4000067 s + -> Sort + Sort Key: t.id + -> Seq Scan on target_pushdowntest_4000071 t +(47 rows) + +-- Test 2 : tables are colocated AND source query is not multisharded : should push down to worker. +-- DEBUG LOGS show that query is getting pushed down +MERGE INTO target_pushdowntest t +USING (SELECT * from source_pushdowntest where id = 1) s +on t.id = s.id +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.id); +DEBUG: +DEBUG: Creating MERGE router plan +-- Test 3 : tables are colocated source query is single sharded but not using source distributed column in insertion. let's not pushdown. +INSERT INTO source_pushdowntest (id) VALUES (3); +EXPLAIN (costs off, timing off, summary off) +MERGE INTO target_pushdowntest t +USING (SELECT 1 as somekey, id from source_pushdowntest where id = 1) s +on t.id = s.somekey +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.somekey); +DEBUG: MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied +DEBUG: MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied +DEBUG: Creating MERGE repartition plan +DEBUG: Using column - index:0 from the source list to redistribute + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target_pushdowntest method: pull to coordinator + -> Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_pushdowntest_4000064 source_pushdowntest + Filter: (id = 1) +(9 rows) + +-- let's verify if we use some other column from source for value of distributed column in target. +-- it should be inserted to correct shard of target. +CREATE TABLE source_withdata (id integer, some_number integer); +CREATE TABLE target_table (id integer, name text); +SELECT create_distributed_table('source_withdata', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target_table', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source_withdata (id, some_number) VALUES (1, 3); +-- we will use some_number column from source_withdata to insert into distributed column of target. +-- value of some_number is 3 let's verify what shard it should go to. +select worker_hash(3); + worker_hash +--------------------------------------------------------------------- + -28094569 +(1 row) + +-- it should go to second shard of target as target has 4 shard and hash "-28094569" comes in range of second shard. +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN NOT MATCHED THEN + INSERT (id, name) + VALUES (s.some_number, 'parag'); +DEBUG: Sub-query is not pushable, try repartitioning +DEBUG: MERGE command is only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: Creating MERGE repartition plan +DEBUG: Using column - index:1 from the source list to redistribute +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: Execute MERGE task list +-- let's verify if data inserted to second shard of target. +EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM target_table; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 4 + Tuple data received from nodes: 9 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on target_table_4000076 target_table (actual rows=0 loops=1) + -> Task + Tuple data received from node: 9 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on target_table_4000077 target_table (actual rows=1 loops=1) + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on target_table_4000078 target_table (actual rows=0 loops=1) + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on target_table_4000079 target_table (actual rows=0 loops=1) +(20 rows) + +-- let's verify target data too. +SELECT * FROM target_table; + id | name +--------------------------------------------------------------------- + 3 | parag +(1 row) + +-- test UPDATE : when source is single sharded and table are colocated +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + UPDATE SET name = 'parag jain'; +DEBUG: Sub-query is not pushable, try repartitioning +DEBUG: MERGE command is only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: Creating MERGE repartition plan +DEBUG: Using column - index:1 from the source list to redistribute +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: Execute MERGE task list +-- let's verify if data updated properly. +SELECT * FROM target_table; + id | name +--------------------------------------------------------------------- + 3 | parag jain +(1 row) + +-- let's see what happend when we try to update distributed key of target table +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + UPDATE SET id = 1500; +ERROR: updating the distribution column is not allowed in MERGE actions +SELECT * FROM target_table; + id | name +--------------------------------------------------------------------- + 3 | parag jain +(1 row) + +-- test DELETE : when source is single sharded and table are colocated +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + DELETE; +DEBUG: Sub-query is not pushable, try repartitioning +DEBUG: MERGE command is only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: Creating MERGE repartition plan +DEBUG: Using column - index:1 from the source list to redistribute +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: Execute MERGE task list +-- let's verify if data deleted properly. +SELECT * FROM target_table; + id | name +--------------------------------------------------------------------- +(0 rows) + +-- +DELETE FROM source_withdata; +DELETE FROM target_table; +INSERT INTO source VALUES (1,1); +merge into target_table sda +using source_withdata sdn +on sda.id = sdn.id AND sda.id = 1 +when not matched then + insert (id) + values (10000); +ERROR: MERGE INSERT is using unsupported expression type for distribution column +DETAIL: Inserting arbitrary values that don't correspond to the joined column values can lead to unpredictable outcomes where rows are incorrectly distributed among different shards +SELECT * FROM target_table WHERE id = 10000; + id | name +--------------------------------------------------------------------- +(0 rows) + +RESET client_min_messages; -- This will prune shards with restriction information as NOT MATCHED is void BEGIN; SET citus.log_remote_commands to true; @@ -2898,14 +3189,14 @@ WHEN NOT MATCHED THEN -> Limit -> Sort Sort Key: id2 - -> Seq Scan on demo_source_table_4000135 demo_source_table + -> Seq Scan on demo_source_table_4000151 demo_source_table -> Distributed Subplan XXX_2 -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on demo_source_table_4000135 demo_source_table + -> Seq Scan on demo_source_table_4000151 demo_source_table Task Count: 1 Tasks Shown: All -> Task @@ -3119,10 +3410,10 @@ DEBUG: Creating MERGE repartition plan DEBUG: Using column - index:0 from the source list to redistribute DEBUG: Collect source query results on coordinator DEBUG: Create a MERGE task list that needs to be routed -DEBUG: -DEBUG: -DEBUG: -DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: DEBUG: Execute MERGE task list RESET client_min_messages; SELECT * FROM target_6785 ORDER BY 1; @@ -3240,7 +3531,7 @@ USING s1 s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES(s.val); -ERROR: MERGE INSERT must use the source table distribution column value +ERROR: MERGE INSERT must use the source's joining column for target's distribution column MERGE INTO t1 t USING s1 s ON t.id = s.id @@ -3966,7 +4257,7 @@ CONTEXT: SQL statement "SELECT citus_drop_all_shards(v_obj.objid, v_obj.schema_ PL/pgSQL function citus_drop_trigger() line XX at PERFORM DROP FUNCTION merge_when_and_write(); DROP SCHEMA merge_schema CASCADE; -NOTICE: drop cascades to 103 other objects +NOTICE: drop cascades to 107 other objects DETAIL: drop cascades to function insert_data() drop cascades to table local_local drop cascades to table target @@ -4026,6 +4317,10 @@ drop cascades to table pg_source drop cascades to table citus_target drop cascades to table citus_source drop cascades to function compare_tables() +drop cascades to table source_pushdowntest +drop cascades to table target_pushdowntest +drop cascades to table source_withdata +drop cascades to table target_table drop cascades to view pg_source_view drop cascades to view citus_source_view drop cascades to table pg_pa_target @@ -4042,7 +4337,7 @@ drop cascades to table target_set drop cascades to table source_set drop cascades to table refsource_ref drop cascades to table pg_result -drop cascades to table refsource_ref_4000112 +drop cascades to table refsource_ref_4000128 drop cascades to table pg_ref drop cascades to table local_ref drop cascades to table reftarget_local @@ -4060,11 +4355,7 @@ drop cascades to table source_6785 drop cascades to table target_6785 drop cascades to function add_s(integer,integer) drop cascades to table pg -drop cascades to table t1_4000174 -drop cascades to table s1_4000175 +drop cascades to table t1_4000190 +drop cascades to table s1_4000191 drop cascades to table t1 -drop cascades to table s1 -drop cascades to table dist_target -drop cascades to table dist_source -drop cascades to view show_tables -and 3 other objects (see server log for list) +and 7 other objects (see server log for list) diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 54d65686c..a56469993 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -113,7 +113,8 @@ test: function_with_case_when test: clock # MERGE tests -test: merge pgmerge merge_repartition2 +test: merge pgmerge +test: merge_repartition2 test: merge_repartition1 merge_schema_sharding test: merge_partition_tables diff --git a/src/test/regress/sql/merge.sql b/src/test/regress/sql/merge.sql index a41e80841..5316b5233 100644 --- a/src/test/regress/sql/merge.sql +++ b/src/test/regress/sql/merge.sql @@ -1206,6 +1206,139 @@ SET citus.log_remote_commands to false; SELECT compare_tables(); ROLLBACK; + +-- let's create source and target table +ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 13000; +CREATE TABLE source_pushdowntest (id integer); +CREATE TABLE target_pushdowntest (id integer ); + +-- let's distribute both table on id field +SELECT create_distributed_table('source_pushdowntest', 'id'); +SELECT create_distributed_table('target_pushdowntest', 'id'); + +-- we are doing this operation on single node setup let's figure out colocation id of both tables +-- both has same colocation id so both are colocated. +WITH colocations AS ( + SELECT colocationid + FROM pg_dist_partition + WHERE logicalrelid = 'source_pushdowntest'::regclass + OR logicalrelid = 'target_pushdowntest'::regclass +) +SELECT + CASE + WHEN COUNT(DISTINCT colocationid) = 1 THEN 'Same' + ELSE 'Different' + END AS colocation_status +FROM colocations; + +SET client_min_messages TO DEBUG1; +-- Test 1 : tables are colocated AND query is multisharded AND Join On distributed column : should push down to workers. + +EXPLAIN (costs off, timing off, summary off) +MERGE INTO target_pushdowntest t +USING source_pushdowntest s +ON t.id = s.id +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.id); + +-- Test 2 : tables are colocated AND source query is not multisharded : should push down to worker. +-- DEBUG LOGS show that query is getting pushed down +MERGE INTO target_pushdowntest t +USING (SELECT * from source_pushdowntest where id = 1) s +on t.id = s.id +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.id); + + +-- Test 3 : tables are colocated source query is single sharded but not using source distributed column in insertion. let's not pushdown. +INSERT INTO source_pushdowntest (id) VALUES (3); + +EXPLAIN (costs off, timing off, summary off) +MERGE INTO target_pushdowntest t +USING (SELECT 1 as somekey, id from source_pushdowntest where id = 1) s +on t.id = s.somekey +WHEN NOT MATCHED THEN + INSERT (id) + VALUES (s.somekey); + + +-- let's verify if we use some other column from source for value of distributed column in target. +-- it should be inserted to correct shard of target. +CREATE TABLE source_withdata (id integer, some_number integer); +CREATE TABLE target_table (id integer, name text); +SELECT create_distributed_table('source_withdata', 'id'); +SELECT create_distributed_table('target_table', 'id'); + +INSERT INTO source_withdata (id, some_number) VALUES (1, 3); + +-- we will use some_number column from source_withdata to insert into distributed column of target. +-- value of some_number is 3 let's verify what shard it should go to. +select worker_hash(3); + +-- it should go to second shard of target as target has 4 shard and hash "-28094569" comes in range of second shard. +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN NOT MATCHED THEN + INSERT (id, name) + VALUES (s.some_number, 'parag'); + +-- let's verify if data inserted to second shard of target. +EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM target_table; + +-- let's verify target data too. +SELECT * FROM target_table; + + +-- test UPDATE : when source is single sharded and table are colocated +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + UPDATE SET name = 'parag jain'; + +-- let's verify if data updated properly. +SELECT * FROM target_table; + +-- let's see what happend when we try to update distributed key of target table +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + UPDATE SET id = 1500; + +SELECT * FROM target_table; + +-- test DELETE : when source is single sharded and table are colocated +MERGE INTO target_table t +USING (SELECT id, some_number from source_withdata where id = 1) s +on t.id = s.some_number +WHEN MATCHED THEN + DELETE; + +-- let's verify if data deleted properly. +SELECT * FROM target_table; + +-- +DELETE FROM source_withdata; +DELETE FROM target_table; +INSERT INTO source VALUES (1,1); + +merge into target_table sda +using source_withdata sdn +on sda.id = sdn.id AND sda.id = 1 +when not matched then + insert (id) + values (10000); + +SELECT * FROM target_table WHERE id = 10000; + +RESET client_min_messages; + + + -- This will prune shards with restriction information as NOT MATCHED is void BEGIN; SET citus.log_remote_commands to true; From d9635609f487d1a46f9dbedbdcb0dedfad3733e5 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Thu, 2 Nov 2023 14:02:34 +0300 Subject: [PATCH 045/155] Fix flaky multi_mx_node_metadata.sql test (#7317) Fixes the flaky test that results in following diff: ```diff --- /__w/citus/citus/src/test/regress/expected/multi_mx_node_metadata.out.modified 2023-11-01 14:22:12.890476575 +0000 +++ /__w/citus/citus/src/test/regress/results/multi_mx_node_metadata.out.modified 2023-11-01 14:22:12.914476657 +0000 @@ -840,24 +840,26 @@ (1 row) \c :datname - - :master_port SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; datname ------------ db_to_drop (1 row) DROP DATABASE db_to_drop; +ERROR: database "db_to_drop" is being accessed by other users SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; datname ------------ -(0 rows) + db_to_drop +(1 row) -- cleanup DROP SEQUENCE sequence CASCADE; NOTICE: drop cascades to default value for column a of table reference_table ``` (cherry picked from commit 9867c5b949d5a2b9dd7183a09df722336cc73db6) --- src/test/regress/citus_tests/run_test.py | 9 ++++++++ .../expected/multi_mx_node_metadata.out | 21 ++++++++++++++++--- .../regress/sql/multi_mx_node_metadata.sql | 21 ++++++++++++++++--- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 5dbff9193..67993cecc 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -172,6 +172,15 @@ DEPS = { ), "grant_on_schema_propagation": TestDeps("minimal_schedule"), "propagate_extension_commands": TestDeps("minimal_schedule"), + "multi_size_queries": TestDeps("base_schedule", ["multi_copy"]), + "multi_mx_node_metadata": TestDeps( + None, + [ + "multi_extension", + "multi_test_helpers", + "multi_test_helpers_superuser", + ], + ), } diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out index 707dcc472..6a152b515 100644 --- a/src/test/regress/expected/multi_mx_node_metadata.out +++ b/src/test/regress/expected/multi_mx_node_metadata.out @@ -9,7 +9,7 @@ SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; \set VERBOSITY terse -- Simulates a readonly node by setting default_transaction_read_only. -CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) +CREATE OR REPLACE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT LANGUAGE sql AS $$ @@ -27,7 +27,7 @@ CREATE OR REPLACE FUNCTION raise_error_in_metadata_sync() RETURNS void LANGUAGE C STRICT AS 'citus'; -CREATE PROCEDURE wait_until_process_count(appname text, target_count int) AS $$ +CREATE OR REPLACE PROCEDURE wait_until_process_count(appname text, target_count int) AS $$ declare counter integer := -1; begin @@ -846,7 +846,22 @@ SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; db_to_drop (1 row) -DROP DATABASE db_to_drop; +DO $$ +DECLARE + i int := 0; +BEGIN + WHILE NOT (SELECT bool_and(success) from run_command_on_all_nodes('DROP DATABASE IF EXISTS db_to_drop')) + LOOP + BEGIN + i := i + 1; + IF i > 5 THEN + RAISE EXCEPTION 'DROP DATABASE timed out'; + END IF; + PERFORM pg_sleep(1); + END; + END LOOP; +END; +$$; SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; datname --------------------------------------------------------------------- diff --git a/src/test/regress/sql/multi_mx_node_metadata.sql b/src/test/regress/sql/multi_mx_node_metadata.sql index 45b4edae1..e0d765a20 100644 --- a/src/test/regress/sql/multi_mx_node_metadata.sql +++ b/src/test/regress/sql/multi_mx_node_metadata.sql @@ -14,7 +14,7 @@ SET citus.shard_replication_factor TO 1; \set VERBOSITY terse -- Simulates a readonly node by setting default_transaction_read_only. -CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) +CREATE OR REPLACE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT LANGUAGE sql AS $$ @@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION raise_error_in_metadata_sync() LANGUAGE C STRICT AS 'citus'; -CREATE PROCEDURE wait_until_process_count(appname text, target_count int) AS $$ +CREATE OR REPLACE PROCEDURE wait_until_process_count(appname text, target_count int) AS $$ declare counter integer := -1; begin @@ -378,7 +378,22 @@ SELECT trigger_metadata_sync(); SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; -DROP DATABASE db_to_drop; +DO $$ +DECLARE + i int := 0; +BEGIN + WHILE NOT (SELECT bool_and(success) from run_command_on_all_nodes('DROP DATABASE IF EXISTS db_to_drop')) + LOOP + BEGIN + i := i + 1; + IF i > 5 THEN + RAISE EXCEPTION 'DROP DATABASE timed out'; + END IF; + PERFORM pg_sleep(1); + END; + END LOOP; +END; +$$; SELECT datname FROM pg_stat_activity WHERE application_name LIKE 'Citus Met%'; From caee20ad7c521af2128c5b8e55844f575113746e Mon Sep 17 00:00:00 2001 From: paragjain Date: Tue, 18 Jun 2024 08:58:00 +0000 Subject: [PATCH 046/155] fixing expected file of multi_move_mx test --- src/test/regress/expected/multi_move_mx.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/expected/multi_move_mx.out b/src/test/regress/expected/multi_move_mx.out index b5aeec8ca..21ae5685c 100644 --- a/src/test/regress/expected/multi_move_mx.out +++ b/src/test/regress/expected/multi_move_mx.out @@ -147,7 +147,7 @@ SELECT pg_reload_conf(); -- only with ommitting user. CREATE SUBSCRIPTION subs_01 CONNECTION 'host=''localhost'' port=57637' PUBLICATION pub_01 WITH (citus_use_authinfo=true); -ERROR: could not connect to the publisher: root certificate file "/non/existing/certificate.crt" does not exist +ERROR: could not connect to the publisher: connection to server at "localhost" (::1), port 57637 failed: root certificate file "/non/existing/certificate.crt" does not exist Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable server certificate verification. ALTER SYSTEM RESET citus.node_conninfo; SELECT pg_reload_conf(); From 9dcfcb92ff5793583b6915793c777b5f347da4ec Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Fri, 12 Jul 2024 10:26:38 +0200 Subject: [PATCH 047/155] CI: move to github container registry (#7652) We move the CI images to the github container registry. Given we mostly (if not solely) run these containers on github actions infra it makes sense to have them hosted closer to where they are needed. Image changes: https://github.com/citusdata/the-process/pull/157 (cherry picked from commit e776a7ebbb25e8827280b3a4708ea1116419953e) --- .github/workflows/build_and_test.yml | 25 +++++++++++++------------ src/test/regress/Pipfile | 2 +- src/test/regress/Pipfile.lock | 8 +++++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 840814f7f..672a073f9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -19,18 +19,19 @@ jobs: runs-on: ubuntu-latest name: Initialize parameters outputs: - build_image_name: "citus/extbuilder" - test_image_name: "citus/exttester" - citusupgrade_image_name: "citus/citusupgradetester" - fail_test_image_name: "citus/failtester" - pgupgrade_image_name: "citus/pgupgradetester" - style_checker_image_name: "citus/stylechecker" + build_image_name: "ghcr.io/citusdata/extbuilder" + test_image_name: "ghcr.io/citusdata/exttester" + citusupgrade_image_name: "ghcr.io/citusdata/citusupgradetester" + fail_test_image_name: "ghcr.io/citusdata/failtester" + pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" + style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.18" - image_suffix: "-v9d71045" - pg14_version: '{ "major": "14", "full": "14.9" }' - pg15_version: '{ "major": "15", "full": "15.4" }' - pg16_version: '{ "major": "16", "full": "16.0" }' - upgrade_pg_versions: "14.9-15.4-16.0" + sql_snapshot_pg_version: "16.2" + image_suffix: "-v7693016" + pg14_version: '{ "major": "14", "full": "14.11" }' + pg15_version: '{ "major": "15", "full": "15.6" }' + pg16_version: '{ "major": "16", "full": "16.2" }' + upgrade_pg_versions: "14.11-15.6-16.2" steps: # Since GHA jobs needs at least one step we use a noop step here. - name: Set up parameters @@ -39,7 +40,7 @@ jobs: needs: params runs-on: ubuntu-20.04 container: - image: ${{ needs.params.outputs.build_image_name }}:latest + image: ${{ needs.params.outputs.build_image_name }}:${{ needs.params.outputs.sql_snapshot_pg_version }}${{ needs.params.outputs.image_suffix }} options: --user root steps: - uses: actions/checkout@v3.5.0 diff --git a/src/test/regress/Pipfile b/src/test/regress/Pipfile index 5bce63004..097ab1517 100644 --- a/src/test/regress/Pipfile +++ b/src/test/regress/Pipfile @@ -5,7 +5,7 @@ verify_ssl = true [packages] mitmproxy = {editable = true, ref = "main", git = "https://github.com/citusdata/mitmproxy.git"} -construct = "==2.9.45" +construct = "*" docopt = "==0.6.2" cryptography = ">=39.0.1" pytest = "*" diff --git a/src/test/regress/Pipfile.lock b/src/test/regress/Pipfile.lock index e6717e5fc..1b87dba5e 100644 --- a/src/test/regress/Pipfile.lock +++ b/src/test/regress/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9568b1f3e4d4fd408e5e263f6346b0a4f479ac88e02f64bb79a9d482096e6a03" + "sha256": "f8db86383082539f626f1402e720f5f2e3f9718b44a8f26110cf9f52e7ca46bc" }, "pipfile-spec": 6, "requires": { @@ -201,10 +201,12 @@ }, "construct": { "hashes": [ - "sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c" + "sha256:4d2472f9684731e58cc9c56c463be63baa1447d674e0d66aeb5627b22f512c29", + "sha256:c80be81ef595a1a821ec69dc16099550ed22197615f4320b57cc9ce2a672cb30" ], "index": "pypi", - "version": "==2.9.45" + "markers": "python_version >= '3.6'", + "version": "==2.10.70" }, "cryptography": { "hashes": [ From f0ea07a813d39951f38f30663ce154b706238c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 12 Jul 2024 12:25:12 +0300 Subject: [PATCH 048/155] Removes el/7 and ol/7 as runners (#7650) Removes el/7 and ol/7 as runners and update checkout action to v4 We use EL/7 and OL/7 runners to test packaging for these distributions. However, for the past two weeks, we've encountered errors during the checkout step in the pipelines. The error message is as follows: ``` /__e/node20/bin/node: /lib64/libm.so.6: version `GLIBC_2.27' not found (required by /__e/node20/bin/node) /__e/node20/bin/node: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.20' not found (required by /__e/node20/bin/node) /__e/node20/bin/node: /lib64/libstdc++.so.6: version `CXXABI_1.3.9' not found (required by /__e/node20/bin/node) /__e/node20/bin/node: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.21' not found (required by /__e/node20/bin/node) /__e/node20/bin/node: /lib64/libc.so.6: version `GLIBC_2.28' not found (required by /__e/node20/bin/node) /__e/node20/bin/node: /lib64/libc.so.6: version `GLIBC_2.25' not found (required by /__e/node20/bin/node) ``` The GCC version within the EL/7 and OL/7 Docker images is 2.17, and we cannot upgrade it. Therefore, we need to remove these images from the packaging test pipelines. Consequently, we will no longer verify if the code builds for EL/7 and OL/7. However, we are not using these packaging images as runners within the packaging infrastructure, so we can continue to use these images for packaging. Additional Info: I learned that Marlin team fully dropped the el/7 support so we will drop in further releases as well (cherry picked from commit c603c3ed7446b2618f48537111a0bd5bf70fbc2b) --- .github/workflows/packaging-test-pipelines.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index fec0fefa9..6c3672fed 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -18,7 +18,7 @@ jobs: pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 2 - name: Get Postgres Versions @@ -50,18 +50,6 @@ jobs: - almalinux-8 - almalinux-9 POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }} - # Postgres removed support for CentOS 7 in PG 16. Below block is needed to - # keep the build for CentOS 7 working for PG 14 and PG 15. - # Once dependent systems drop support for Centos 7, we can remove this block. - include: - - packaging_docker_image: centos-7 - POSTGRES_VERSION: 14 - - packaging_docker_image: centos-7 - POSTGRES_VERSION: 15 - - packaging_docker_image: oraclelinux-7 - POSTGRES_VERSION: 14 - - packaging_docker_image: oraclelinux-7 - POSTGRES_VERSION: 15 container: image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }} @@ -69,7 +57,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set Postgres and python parameters for rpm based distros run: | From f60c4cbd1965b9ad3d303db886df767e67b70720 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Fri, 12 Jul 2024 17:26:23 +0200 Subject: [PATCH 049/155] bump postgres versions in CI and dev (#7655) Upgrade postgres versions to: - 14.12 - 15.7 - 16.3 Depends on https://github.com/citusdata/the-process/pull/158 (cherry picked from commit accb7d09f7a1b8064d9cca90f1abd17d0490b603) --- .github/workflows/build_and_test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 672a073f9..54721c0e0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,12 +26,12 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.18" - sql_snapshot_pg_version: "16.2" - image_suffix: "-v7693016" - pg14_version: '{ "major": "14", "full": "14.11" }' - pg15_version: '{ "major": "15", "full": "15.6" }' - pg16_version: '{ "major": "16", "full": "16.2" }' - upgrade_pg_versions: "14.11-15.6-16.2" + sql_snapshot_pg_version: "16.3" + image_suffix: "-v13fd57c" + pg14_version: '{ "major": "14", "full": "14.12" }' + pg15_version: '{ "major": "15", "full": "15.7" }' + pg16_version: '{ "major": "16", "full": "16.3" }' + upgrade_pg_versions: "14.12-15.7-16.3" steps: # Since GHA jobs needs at least one step we use a noop step here. - name: Set up parameters From 6349f2d52d59efeb7f18a622298ffc17d7626bea Mon Sep 17 00:00:00 2001 From: Parag Jain <40451840+paragikjain@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:38:44 +0530 Subject: [PATCH 050/155] Support MERGE command for single_shard_distributed Target (#7643) This PR has following changes : 1. Enable MERGE command for single_shard_distributed targets. (cherry picked from commit 3c467e6e02f630643bf3120210c0462bd51af2dd) --- src/backend/distributed/commands/multi_copy.c | 2 +- .../distributed/planner/merge_planner.c | 42 +- src/test/regress/bin/normalize.sed | 1 + .../expected/merge_schema_sharding.out | 53 +- src/test/regress/expected/merge_vcore.out | 481 ++++++++++++++++++ src/test/regress/expected/merge_vcore_0.out | 6 + src/test/regress/multi_schedule | 1 + src/test/regress/sql/merge_vcore.sql | 319 ++++++++++++ 8 files changed, 879 insertions(+), 26 deletions(-) create mode 100644 src/test/regress/expected/merge_vcore.out create mode 100644 src/test/regress/expected/merge_vcore_0.out create mode 100644 src/test/regress/sql/merge_vcore.sql diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index aaff24a10..1517c1d5a 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -2572,7 +2572,7 @@ ShardIdForTuple(CitusCopyDestReceiver *copyDest, Datum *columnValues, bool *colu * Find the shard interval and id for the partition column value for * non-reference tables. * - * For reference table, this function blindly returns the tables single + * For reference table, and single shard distributed table this function blindly returns the tables single * shard. */ ShardInterval *shardInterval = FindShardInterval(partitionColumnValue, cacheEntry); diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 09d2d90ac..1f9d17c43 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -243,14 +243,27 @@ CreateNonPushableMergePlan(Oid targetRelationId, uint64 planId, Query *originalQ CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(targetRelationId); - /* - * Get the index of the column in the source query that will be utilized - * to repartition the source rows, ensuring colocation with the target - */ - distributedPlan->sourceResultRepartitionColumnIndex = - SourceResultPartitionColumnIndex(mergeQuery, - sourceQuery->targetList, - targetRelation); + + if (IsCitusTableType(targetRelation->relationId, SINGLE_SHARD_DISTRIBUTED)) + { + /* + * if target table is SINGLE_SHARD_DISTRIBUTED let's set this to invalid -1 + * so later in execution phase we don't rely on this value and try to find single shard of target instead. + */ + distributedPlan->sourceResultRepartitionColumnIndex = -1; + } + else + { + /* + * Get the index of the column in the source query that will be utilized + * to repartition the source rows, ensuring colocation with the target + */ + + distributedPlan->sourceResultRepartitionColumnIndex = + SourceResultPartitionColumnIndex(mergeQuery, + sourceQuery->targetList, + targetRelation); + } /* * Make a copy of the source query, since following code scribbles it @@ -262,11 +275,11 @@ CreateNonPushableMergePlan(Oid targetRelationId, uint64 planId, Query *originalQ int cursorOptions = CURSOR_OPT_PARALLEL_OK; PlannedStmt *sourceRowsPlan = pg_plan_query(sourceQueryCopy, NULL, cursorOptions, boundParams); - bool repartitioned = IsRedistributablePlan(sourceRowsPlan->planTree) && - IsSupportedRedistributionTarget(targetRelationId); + bool isRepartitionAllowed = IsRedistributablePlan(sourceRowsPlan->planTree) && + IsSupportedRedistributionTarget(targetRelationId); /* If plan is distributed, no work at the coordinator */ - if (repartitioned) + if (isRepartitionAllowed) { distributedPlan->modifyWithSelectMethod = MODIFY_WITH_SELECT_REPARTITION; } @@ -1273,13 +1286,6 @@ static int SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, CitusTableCacheEntry *targetRelation) { - if (IsCitusTableType(targetRelation->relationId, SINGLE_SHARD_DISTRIBUTED)) - { - ereport(ERROR, (errmsg("MERGE operation across distributed schemas " - "or with a row-based distributed table is " - "not yet supported"))); - } - /* Get all the Join conditions from the ON clause */ List *mergeJoinConditionList = WhereClauseList(mergeQuery->jointree); Var *targetColumn = targetRelation->partitionColumn; diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 5dc5d7ca8..62e9de697 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -32,6 +32,7 @@ s/"t2_[0-9]+"/"t2_xxxxxxx"/g # shard table names for MERGE tests s/merge_schema\.([_a-z0-9]+)_40[0-9]+ /merge_schema.\1_xxxxxxx /g s/pgmerge_schema\.([_a-z0-9]+)_40[0-9]+ /pgmerge_schema.\1_xxxxxxx /g +s/merge_vcore_schema\.([_a-z0-9]+)_40[0-9]+ /pgmerge_schema.\1_xxxxxxx /g # shard table names for multi_subquery s/ keyval(1|2|ref)_[0-9]+ / keyval\1_xxxxxxx /g diff --git a/src/test/regress/expected/merge_schema_sharding.out b/src/test/regress/expected/merge_schema_sharding.out index 8a9ba89dd..17f6f6adb 100644 --- a/src/test/regress/expected/merge_schema_sharding.out +++ b/src/test/regress/expected/merge_schema_sharding.out @@ -98,14 +98,26 @@ WHEN MATCHED THEN UPDATE SET b = nullkey_c2_t1.b; DEBUG: Distributed tables are not co-located, try repartitioning DEBUG: For MERGE command, all the distributed tables must be colocated DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Distributed planning for a fast-path router query +DEBUG: Creating router plan +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) nullkey_c2_t1 ON (citus_table_alias.a OPERATOR(pg_catalog.=) nullkey_c2_t1.a) WHEN MATCHED THEN UPDATE SET b = nullkey_c2_t1.b +DEBUG: Execute MERGE task list MERGE INTO schema_shard_table1.nullkey_c1_t1 USING nullkey_c2_t1 ON (schema_shard_table1.nullkey_c1_t1.a = nullkey_c2_t1.a) WHEN MATCHED THEN UPDATE SET b = nullkey_c2_t1.b WHEN NOT MATCHED THEN INSERT VALUES (nullkey_c2_t1.a, nullkey_c2_t1.b); DEBUG: Distributed tables are not co-located, try repartitioning DEBUG: For MERGE command, all the distributed tables must be colocated DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Distributed planning for a fast-path router query +DEBUG: Creating router plan +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) nullkey_c2_t1 ON (citus_table_alias.a OPERATOR(pg_catalog.=) nullkey_c2_t1.a) WHEN MATCHED THEN UPDATE SET b = nullkey_c2_t1.b WHEN NOT MATCHED THEN INSERT (a, b) VALUES (nullkey_c2_t1.a, nullkey_c2_t1.b) +DEBUG: Execute MERGE task list -- with a distributed table SET search_path TO schema_shard_table1; MERGE INTO nullkey_c1_t1 USING schema_shard_table.distributed_table ON (nullkey_c1_t1.a = schema_shard_table.distributed_table.a) @@ -114,7 +126,12 @@ WHEN NOT MATCHED THEN INSERT VALUES (schema_shard_table.distributed_table.a, sch DEBUG: Distributed tables are not co-located, try repartitioning DEBUG: For MERGE command, all the distributed tables must be colocated DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) distributed_table ON (citus_table_alias.a OPERATOR(pg_catalog.=) distributed_table.a) WHEN MATCHED THEN UPDATE SET b = distributed_table.b WHEN NOT MATCHED THEN INSERT (a, b) VALUES (distributed_table.a, distributed_table.b) +DEBUG: Execute MERGE task list MERGE INTO schema_shard_table.distributed_table USING nullkey_c1_t1 ON (nullkey_c1_t1.a = schema_shard_table.distributed_table.a) WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (nullkey_c1_t1.a, nullkey_c1_t1.b); @@ -163,7 +180,13 @@ WHEN MATCHED THEN UPDATE SET b = schema_shard_table.reference_table.b; DEBUG: A mix of distributed and reference table, try repartitioning DEBUG: A mix of distributed and reference table, routable query is not possible DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Distributed planning for a fast-path router query +DEBUG: Creating router plan +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) reference_table ON (citus_table_alias.a OPERATOR(pg_catalog.=) reference_table.a) WHEN MATCHED THEN UPDATE SET b = reference_table.b +DEBUG: Execute MERGE task list MERGE INTO schema_shard_table.reference_table USING nullkey_c1_t1 ON (nullkey_c1_t1.a = schema_shard_table.reference_table.a) WHEN MATCHED THEN UPDATE SET b = nullkey_c1_t1.b WHEN NOT MATCHED THEN INSERT VALUES (nullkey_c1_t1.a, nullkey_c1_t1.b); @@ -174,7 +197,13 @@ WHEN MATCHED THEN UPDATE SET b = schema_shard_table.citus_local_table.b; DEBUG: A mix of distributed and local table, try repartitioning DEBUG: A mix of distributed and citus-local table, routable query is not possible DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Distributed planning for a fast-path router query +DEBUG: Creating router plan +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) citus_local_table ON (citus_table_alias.a OPERATOR(pg_catalog.=) citus_local_table.a) WHEN MATCHED THEN UPDATE SET b = citus_local_table.b +DEBUG: Execute MERGE task list MERGE INTO schema_shard_table.citus_local_table USING nullkey_c1_t1 ON (nullkey_c1_t1.a = schema_shard_table.citus_local_table.a) WHEN MATCHED THEN DELETE; DEBUG: A mix of distributed and local table, try repartitioning @@ -210,7 +239,12 @@ WHEN MATCHED THEN UPDATE SET b = cte.b; DEBUG: Distributed tables are not co-located, try repartitioning DEBUG: For MERGE command, all the distributed tables must be colocated DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte ON (citus_table_alias.a OPERATOR(pg_catalog.=) cte.a) WHEN MATCHED THEN UPDATE SET b = cte.b +DEBUG: Execute MERGE task list WITH cte AS materialized ( SELECT * FROM schema_shard_table.distributed_table ) @@ -219,7 +253,12 @@ WHEN MATCHED THEN UPDATE SET b = cte.b; DEBUG: Distributed tables are not co-located, try repartitioning DEBUG: For MERGE command, all the distributed tables must be colocated DEBUG: Creating MERGE repartition plan -ERROR: MERGE operation across distributed schemas or with a row-based distributed table is not yet supported +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Collect source query results on coordinator +DEBUG: Create a MERGE task list that needs to be routed +DEBUG: +DEBUG: distributed statement: MERGE INTO schema_shard_table1.nullkey_c1_t1_4005006 citus_table_alias USING (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('merge_into_XXX_4005006'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte ON (citus_table_alias.a OPERATOR(pg_catalog.=) cte.a) WHEN MATCHED THEN UPDATE SET b = cte.b +DEBUG: Execute MERGE task list SET client_min_messages TO WARNING; DROP SCHEMA schema_shard_table1 CASCADE; DROP SCHEMA schema_shard_table2 CASCADE; diff --git a/src/test/regress/expected/merge_vcore.out b/src/test/regress/expected/merge_vcore.out new file mode 100644 index 000000000..03f6f8820 --- /dev/null +++ b/src/test/regress/expected/merge_vcore.out @@ -0,0 +1,481 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif +-- MERGE command performs a join from data_source to target_table_name +DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; +NOTICE: schema "merge_vcore_schema" does not exist, skipping +--MERGE INTO target +--USING source +--WHEN NOT MATCHED +--WHEN MATCHED AND +--WHEN MATCHED +CREATE SCHEMA merge_vcore_schema; +SET search_path TO merge_vcore_schema; +SET citus.shard_count TO 4; +SET citus.next_shard_id TO 4000000; +SET citus.explain_all_tasks TO true; +SET citus.shard_replication_factor TO 1; +SET citus.max_adaptive_executor_pool_size TO 1; +SET client_min_messages = warning; +SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +RESET client_min_messages; +-- ****************************************** CASE 1 : Both are singleSharded*************************************** +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', null, colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', null, colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target method: pull to coordinator + -> Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000000 source +(8 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- *************** CASE 2 : source is single sharded and target is distributed ******************************* +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', null, colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target method: pull to coordinator + -> Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000002 source +(8 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- *************** CASE 3 : source is distributed and target is single sharded ******************************* +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', null); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target method: pull to coordinator + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000007 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000008 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000009 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000010 source +(17 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- *************** CASE 4 : both are distributed ******************************* +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target method: repartition + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000012 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000013 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000014 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000015 source +(17 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- *************** CASE 5 : both are distributed & colocated ******************************* +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id', colocate_with=>'source'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus MERGE INTO ...) + MERGE INTO target method: repartition + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000020 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000021 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000022 source + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on source_4000023 source +(17 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- *************** CASE 6 : both are singlesharded & colocated ******************************* +CREATE TABLE source ( + id bigint, + doc text +); +CREATE TABLE target ( + id bigint, + doc text +); +SELECT create_distributed_table('source', null); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', null, colocate_with=>'source'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"a" : 1} + 2 | {"a" : 2} +(2 rows) + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); +SELECT * FROM target; + id | doc +--------------------------------------------------------------------- + 2 | {"b" : 1} + 2 | {"b" : 1} +(2 rows) + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Merge on target_4000029 target + -> Nested Loop + -> Seq Scan on source_4000028 source + -> Materialize + -> Seq Scan on target_4000029 target + Filter: ('2'::bigint = id) +(11 rows) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; diff --git a/src/test/regress/expected/merge_vcore_0.out b/src/test/regress/expected/merge_vcore_0.out new file mode 100644 index 000000000..a7e3fbf20 --- /dev/null +++ b/src/test/regress/expected/merge_vcore_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index a56469993..d32757e48 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -117,6 +117,7 @@ test: merge pgmerge test: merge_repartition2 test: merge_repartition1 merge_schema_sharding test: merge_partition_tables +test: merge_vcore # --------- # test that no tests leaked intermediate results. This should always be last diff --git a/src/test/regress/sql/merge_vcore.sql b/src/test/regress/sql/merge_vcore.sql new file mode 100644 index 000000000..472bbfe91 --- /dev/null +++ b/src/test/regress/sql/merge_vcore.sql @@ -0,0 +1,319 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif + +-- MERGE command performs a join from data_source to target_table_name +DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; +--MERGE INTO target +--USING source +--WHEN NOT MATCHED +--WHEN MATCHED AND +--WHEN MATCHED + +CREATE SCHEMA merge_vcore_schema; +SET search_path TO merge_vcore_schema; +SET citus.shard_count TO 4; +SET citus.next_shard_id TO 4000000; +SET citus.explain_all_tasks TO true; +SET citus.shard_replication_factor TO 1; +SET citus.max_adaptive_executor_pool_size TO 1; +SET client_min_messages = warning; +SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0); +RESET client_min_messages; + + +-- ****************************************** CASE 1 : Both are singleSharded*************************************** +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', null, colocate_with=>'none'); +SELECT create_distributed_table('target', null, colocate_with=>'none'); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- *************** CASE 2 : source is single sharded and target is distributed ******************************* +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', null, colocate_with=>'none'); +SELECT create_distributed_table('target', 'id'); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + + + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- *************** CASE 3 : source is distributed and target is single sharded ******************************* +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', 'id'); +SELECT create_distributed_table('target', null); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- *************** CASE 4 : both are distributed ******************************* +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', 'id'); +SELECT create_distributed_table('target', 'id'); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- *************** CASE 5 : both are distributed & colocated ******************************* + +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', 'id'); +SELECT create_distributed_table('target', 'id', colocate_with=>'source'); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- *************** CASE 6 : both are singlesharded & colocated ******************************* + +CREATE TABLE source ( + id bigint, + doc text +); + +CREATE TABLE target ( + id bigint, + doc text +); + +SELECT create_distributed_table('source', null); +SELECT create_distributed_table('target', null, colocate_with=>'source'); + +INSERT INTO source (id, doc) VALUES (1, '{"a" : 1}'), (1, '{"a" : 2}'); + +-- insert +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- update +MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) AND src.doc = target.doc +WHEN MATCHED THEN +UPDATE SET doc = '{"b" : 1}' +WHEN NOT MATCHED THEN +INSERT (id, doc) +VALUES (src.t_id, doc); + +SELECT * FROM target; + +-- Explain +EXPLAIN (costs off, timing off, summary off) MERGE INTO ONLY target USING (SELECT 2::bigint AS t_id, doc FROM source) src +ON (src.t_id = target.id) +WHEN MATCHED THEN DO NOTHING; + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + +DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; + + + + From 5c2ef8e2d848fa2661f5f8bb858938e52656b4f3 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Wed, 10 Jul 2024 14:10:39 +0300 Subject: [PATCH 051/155] Add changelog entries for 12.1.5 (cherry picked from commit 5c097860aad25df26a1cb7f6e4a916e00a306c76) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index edf338f49..a226fe2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +### citus v12.1.5 (July 17, 2024) ### + +* Adds support for MERGE commands with single shard distributed target tables + (#7643) + +* Fixes an error with MERGE commands when insert value does not have source + distribution column (#7627) + ### citus v12.1.4 (May 28, 2024) ### * Adds null check for node in HasRangeTableRef (#7604) From 15ecc37ecd766af4393c9bb222aa1efe038dc5f3 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Wed, 10 Jul 2024 14:25:47 +0300 Subject: [PATCH 052/155] Bump Citus to 12.1.5 --- configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 1a20dcded..468c8be10 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.4. +# Generated by GNU Autoconf 2.69 for Citus 12.1.5. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.4' -PACKAGE_STRING='Citus 12.1.4' +PACKAGE_VERSION='12.1.5' +PACKAGE_STRING='Citus 12.1.5' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.4 to adapt to many kinds of systems. +\`configure' configures Citus 12.1.5 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.4:";; + short | recursive ) echo "Configuration of Citus 12.1.5:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.4 +Citus configure 12.1.5 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.4, which was +It was created by Citus $as_me 12.1.5, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.4, which was +This file was extended by Citus $as_me 12.1.5, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.4 +Citus config.status 12.1.5 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 79ff5e430..e3063ee70 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.4]) +AC_INIT([Citus], [12.1.5]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 8131ad7cb..b2badd878 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1413,7 +1413,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.4 + 12.1.5 (1 row) -- ensure no unexpected objects were created outside pg_catalog From 9d364332ac1b30920847e39ab55a7f6fa6941ff9 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:01:39 +0300 Subject: [PATCH 053/155] Rename foreach_ macros to foreach_declared_ macros (#7700) This is prep work for successful compilation with PG17 PG17added foreach_ptr, foreach_int and foreach_oid macros Relevant PG commit 14dd0f27d7cd56ffae9ecdbe324965073d01a9ff https://github.com/postgres/postgres/commit/14dd0f27d7cd56ffae9ecdbe324965073d01a9ff We already have these macros, but they are different with the PG17 ones because our macros take a DECLARED variable, whereas the PG16 macros declare a locally-scoped loop variable themselves. Hence I am renaming our macros to foreach_declared_ I am separating this into its own PR since it touches many files. The main compilation PR is https://github.com/citusdata/citus/pull/7699 --- src/backend/columnar/columnar_customscan.c | 10 +-- src/backend/columnar/columnar_metadata.c | 2 +- src/backend/columnar/columnar_reader.c | 4 +- src/backend/columnar/columnar_tableam.c | 2 +- src/backend/distributed/clock/causal_clock.c | 4 +- .../distributed/commands/alter_table.c | 36 +++++----- src/backend/distributed/commands/begin.c | 2 +- ..._table_operation_for_connected_relations.c | 24 +++---- .../citus_add_local_table_to_metadata.c | 28 ++++---- src/backend/distributed/commands/cluster.c | 2 +- src/backend/distributed/commands/common.c | 8 +-- .../commands/create_distributed_table.c | 12 ++-- .../distributed/commands/dependencies.c | 22 +++--- src/backend/distributed/commands/domain.c | 2 +- src/backend/distributed/commands/extension.c | 18 ++--- .../distributed/commands/foreign_constraint.c | 12 ++-- .../commands/foreign_data_wrapper.c | 2 +- .../distributed/commands/foreign_server.c | 4 +- src/backend/distributed/commands/function.c | 22 +++--- src/backend/distributed/commands/index.c | 14 ++-- src/backend/distributed/commands/multi_copy.c | 4 +- src/backend/distributed/commands/policy.c | 6 +- .../distributed/commands/publication.c | 6 +- src/backend/distributed/commands/role.c | 18 ++--- src/backend/distributed/commands/schema.c | 8 +-- .../commands/schema_based_sharding.c | 16 ++--- src/backend/distributed/commands/sequence.c | 22 +++--- src/backend/distributed/commands/statistics.c | 8 +-- src/backend/distributed/commands/table.c | 66 ++++++++--------- src/backend/distributed/commands/trigger.c | 4 +- src/backend/distributed/commands/truncate.c | 8 +-- .../distributed/commands/utility_hook.c | 16 ++--- src/backend/distributed/commands/vacuum.c | 16 ++--- src/backend/distributed/commands/view.c | 6 +- .../connection/connection_management.c | 6 +- .../locally_reserved_shared_connections.c | 2 +- .../connection/placement_connection.c | 4 +- .../distributed/connection/remote_commands.c | 8 +-- .../distributed/deparser/citus_ruleutils.c | 4 +- src/backend/distributed/deparser/deparse.c | 2 +- .../deparser/deparse_domain_stmts.c | 4 +- .../deparser/deparse_extension_stmts.c | 6 +- .../deparser/deparse_foreign_server_stmts.c | 4 +- .../deparser/deparse_publication_stmts.c | 8 +-- .../distributed/deparser/deparse_role_stmts.c | 4 +- .../deparser/deparse_schema_stmts.c | 2 +- .../deparser/deparse_statistics_stmts.c | 4 +- .../deparser/deparse_text_search.c | 10 +-- .../distributed/deparser/deparse_view_stmts.c | 2 +- .../deparser/qualify_collation_stmt.c | 2 +- .../distributed/deparser/qualify_domain.c | 4 +- .../deparser/qualify_publication_stmt.c | 4 +- .../deparser/qualify_sequence_stmt.c | 4 +- .../deparser/qualify_statistics_stmt.c | 2 +- .../deparser/qualify_text_search_stmts.c | 6 +- .../distributed/deparser/qualify_view_stmt.c | 2 +- .../distributed/executor/adaptive_executor.c | 44 ++++++------ .../distributed/executor/citus_custom_scan.c | 2 +- .../directed_acyclic_graph_execution.c | 8 +-- .../executor/distributed_execution_locks.c | 10 +-- .../distributed_intermediate_results.c | 12 ++-- .../executor/executor_util_tasks.c | 6 +- .../executor/insert_select_executor.c | 12 ++-- .../executor/intermediate_results.c | 10 +-- .../distributed/executor/local_executor.c | 20 +++--- .../distributed/executor/merge_executor.c | 2 +- .../distributed/executor/multi_executor.c | 4 +- .../distributed/executor/placement_access.c | 2 +- .../executor/repartition_join_execution.c | 2 +- .../distributed/executor/subplan_execution.c | 2 +- src/backend/distributed/metadata/dependency.c | 32 ++++----- src/backend/distributed/metadata/distobject.c | 2 +- .../distributed/metadata/metadata_cache.c | 12 ++-- .../distributed/metadata/metadata_sync.c | 50 ++++++------- .../distributed/metadata/metadata_utility.c | 28 ++++---- .../distributed/metadata/node_metadata.c | 22 +++--- .../operations/citus_create_restore_point.c | 6 +- .../distributed/operations/create_shards.c | 8 +-- .../distributed/operations/delete_protocol.c | 6 +- .../distributed/operations/health_check.c | 6 +- .../distributed/operations/node_protocol.c | 4 +- .../replicate_none_dist_table_shard.c | 4 +- .../distributed/operations/shard_cleaner.c | 4 +- .../distributed/operations/shard_rebalancer.c | 52 +++++++------- .../distributed/operations/shard_split.c | 30 ++++---- .../distributed/operations/shard_transfer.c | 46 ++++++------ .../distributed/operations/stage_protocol.c | 12 ++-- .../operations/worker_node_manager.c | 2 +- .../operations/worker_split_copy_udf.c | 6 +- ...worker_split_shard_replication_setup_udf.c | 2 +- .../planner/combine_query_planner.c | 6 +- .../distributed/planner/deparse_shard_query.c | 4 +- .../distributed/planner/distributed_planner.c | 16 ++--- .../planner/insert_select_planner.c | 8 +-- .../planner/intermediate_result_pruning.c | 4 +- .../planner/local_distributed_join_planner.c | 12 ++-- .../distributed/planner/local_plan_cache.c | 2 +- .../distributed/planner/merge_planner.c | 14 ++-- .../distributed/planner/multi_explain.c | 8 +-- .../distributed/planner/multi_join_order.c | 14 ++-- .../planner/multi_logical_optimizer.c | 72 +++++++++---------- .../planner/multi_logical_planner.c | 2 +- .../planner/multi_physical_planner.c | 24 +++---- .../planner/multi_router_planner.c | 16 ++--- .../planner/query_pushdown_planning.c | 8 +-- .../distributed/planner/recursive_planning.c | 8 +-- .../relation_restriction_equivalence.c | 12 ++-- .../distributed/planner/shard_pruning.c | 2 +- .../distributed/progress/multi_progress.c | 2 +- .../distributed/relay/relay_event_utility.c | 12 ++-- .../replication/multi_logical_replication.c | 50 ++++++------- .../shardsplit/shardsplit_decoder.c | 2 +- .../shardsplit_logical_replication.c | 6 +- src/backend/distributed/shared_library_init.c | 2 +- .../distributed/test/colocation_utils.c | 2 +- src/backend/distributed/test/create_shards.c | 2 +- .../distributed/test/deparse_shard_query.c | 4 +- src/backend/distributed/test/dependency.c | 4 +- .../test/distributed_intermediate_results.c | 4 +- .../distributed/test/distribution_metadata.c | 8 +-- .../test/foreign_key_relationship_query.c | 2 +- src/backend/distributed/test/metadata_sync.c | 4 +- .../distributed/test/partitioning_utils.c | 2 +- src/backend/distributed/test/progress_utils.c | 2 +- .../distributed/test/prune_shard_list.c | 2 +- .../distributed/test/shard_rebalancer.c | 16 ++--- .../distributed/transaction/backend_data.c | 6 +- .../distributed_deadlock_detection.c | 6 +- .../distributed/transaction/lock_graph.c | 6 +- .../transaction/relation_access_tracking.c | 13 ++-- .../transaction/remote_transaction.c | 10 +-- .../transaction/transaction_management.c | 6 +- .../transaction/transaction_recovery.c | 2 +- .../transaction/worker_transaction.c | 34 ++++----- .../distributed/utils/background_jobs.c | 12 ++-- .../distributed/utils/citus_copyfuncs.c | 2 +- .../distributed/utils/citus_depended_object.c | 6 +- .../distributed/utils/colocation_utils.c | 6 +- .../utils/distribution_column_map.c | 2 +- .../utils/foreign_key_relationship.c | 8 +-- src/backend/distributed/utils/listutils.c | 12 ++-- .../utils/multi_partitioning_utils.c | 24 +++---- .../distributed/utils/reference_table_utils.c | 18 ++--- src/backend/distributed/utils/resource_lock.c | 32 ++++----- .../distributed/utils/shardinterval_utils.c | 2 +- .../distributed/utils/statistics_collection.c | 2 +- .../worker/worker_create_or_replace.c | 4 +- .../worker/worker_data_fetch_protocol.c | 4 +- .../distributed/worker/worker_drop_protocol.c | 12 ++-- .../worker/worker_shard_visibility.c | 4 +- src/include/distributed/listutils.h | 16 ++--- 151 files changed, 803 insertions(+), 802 deletions(-) diff --git a/src/backend/columnar/columnar_customscan.c b/src/backend/columnar/columnar_customscan.c index ebf815d20..698734bc9 100644 --- a/src/backend/columnar/columnar_customscan.c +++ b/src/backend/columnar/columnar_customscan.c @@ -363,7 +363,7 @@ ColumnarGetRelationInfoHook(PlannerInfo *root, Oid relationObjectId, /* disable index-only scan */ IndexOptInfo *indexOptInfo = NULL; - foreach_ptr(indexOptInfo, rel->indexlist) + foreach_declared_ptr(indexOptInfo, rel->indexlist) { memset(indexOptInfo->canreturn, false, indexOptInfo->ncolumns * sizeof(bool)); } @@ -381,7 +381,7 @@ RemovePathsByPredicate(RelOptInfo *rel, PathPredicate removePathPredicate) List *filteredPathList = NIL; Path *path = NULL; - foreach_ptr(path, rel->pathlist) + foreach_declared_ptr(path, rel->pathlist) { if (!removePathPredicate(path)) { @@ -428,7 +428,7 @@ static void CostColumnarPaths(PlannerInfo *root, RelOptInfo *rel, Oid relationId) { Path *path = NULL; - foreach_ptr(path, rel->pathlist) + foreach_declared_ptr(path, rel->pathlist) { if (IsA(path, IndexPath)) { @@ -783,7 +783,7 @@ ExtractPushdownClause(PlannerInfo *root, RelOptInfo *rel, Node *node) List *pushdownableArgs = NIL; Node *boolExprArg = NULL; - foreach_ptr(boolExprArg, boolExpr->args) + foreach_declared_ptr(boolExprArg, boolExpr->args) { Expr *pushdownableArg = ExtractPushdownClause(root, rel, (Node *) boolExprArg); @@ -1550,7 +1550,7 @@ ColumnarPerStripeScanCost(RelOptInfo *rel, Oid relationId, int numberOfColumnsRe uint32 maxColumnCount = 0; uint64 totalStripeSize = 0; StripeMetadata *stripeMetadata = NULL; - foreach_ptr(stripeMetadata, stripeList) + foreach_declared_ptr(stripeMetadata, stripeList) { totalStripeSize += stripeMetadata->dataLength; maxColumnCount = Max(maxColumnCount, stripeMetadata->columnCount); diff --git a/src/backend/columnar/columnar_metadata.c b/src/backend/columnar/columnar_metadata.c index b756a7607..f3035ba8d 100644 --- a/src/backend/columnar/columnar_metadata.c +++ b/src/backend/columnar/columnar_metadata.c @@ -2042,7 +2042,7 @@ GetHighestUsedRowNumber(uint64 storageId) List *stripeMetadataList = ReadDataFileStripeList(storageId, GetTransactionSnapshot()); StripeMetadata *stripeMetadata = NULL; - foreach_ptr(stripeMetadata, stripeMetadataList) + foreach_declared_ptr(stripeMetadata, stripeMetadataList) { highestRowNumber = Max(highestRowNumber, StripeGetHighestRowNumber(stripeMetadata)); diff --git a/src/backend/columnar/columnar_reader.c b/src/backend/columnar/columnar_reader.c index 7ef0d15d7..65ef27617 100644 --- a/src/backend/columnar/columnar_reader.c +++ b/src/backend/columnar/columnar_reader.c @@ -880,7 +880,7 @@ ReadChunkGroupNextRow(ChunkGroupReadState *chunkGroupReadState, Datum *columnVal memset(columnNulls, true, sizeof(bool) * chunkGroupReadState->columnCount); int attno; - foreach_int(attno, chunkGroupReadState->projectedColumnList) + foreach_declared_int(attno, chunkGroupReadState->projectedColumnList) { const ChunkData *chunkGroupData = chunkGroupReadState->chunkGroupData; const int rowIndex = chunkGroupReadState->currentRow; @@ -1489,7 +1489,7 @@ ProjectedColumnMask(uint32 columnCount, List *projectedColumnList) bool *projectedColumnMask = palloc0(columnCount * sizeof(bool)); int attno; - foreach_int(attno, projectedColumnList) + foreach_declared_int(attno, projectedColumnList) { /* attno is 1-indexed; projectedColumnMask is 0-indexed */ int columnIndex = attno - 1; diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index ca3a5f4c4..6d31c2779 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -3083,7 +3083,7 @@ DefElem * GetExtensionOption(List *extensionOptions, const char *defname) { DefElem *defElement = NULL; - foreach_ptr(defElement, extensionOptions) + foreach_declared_ptr(defElement, extensionOptions) { if (IsA(defElement, DefElem) && strncmp(defElement->defname, defname, NAMEDATALEN) == 0) diff --git a/src/backend/distributed/clock/causal_clock.c b/src/backend/distributed/clock/causal_clock.c index 3d64757e3..230d4e733 100644 --- a/src/backend/distributed/clock/causal_clock.c +++ b/src/backend/distributed/clock/causal_clock.c @@ -328,7 +328,7 @@ GetHighestClockInTransaction(List *nodeConnectionList) { MultiConnection *connection = NULL; - foreach_ptr(connection, nodeConnectionList) + foreach_declared_ptr(connection, nodeConnectionList) { int querySent = SendRemoteCommand(connection, "SELECT citus_get_node_clock();"); @@ -349,7 +349,7 @@ GetHighestClockInTransaction(List *nodeConnectionList) globalClockValue->counter))); /* fetch the results and pick the highest clock value of all the nodes */ - foreach_ptr(connection, nodeConnectionList) + foreach_declared_ptr(connection, nodeConnectionList) { bool raiseInterrupts = true; diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index a81f23ad6..5e830fa47 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -417,7 +417,7 @@ UndistributeTables(List *relationIdList) */ List *originalForeignKeyRecreationCommands = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { List *fkeyCommandsForRelation = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, @@ -803,7 +803,7 @@ ConvertTableInternal(TableConversionState *con) List *partitionList = PartitionList(con->relationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { char *tableQualifiedName = generate_qualified_relation_name( partitionRelationId); @@ -876,7 +876,7 @@ ConvertTableInternal(TableConversionState *con) } TableDDLCommand *tableCreationCommand = NULL; - foreach_ptr(tableCreationCommand, preLoadCommands) + foreach_declared_ptr(tableCreationCommand, preLoadCommands) { Assert(CitusIsA(tableCreationCommand, TableDDLCommand)); @@ -950,7 +950,7 @@ ConvertTableInternal(TableConversionState *con) con->suppressNoticeMessages); TableDDLCommand *tableConstructionCommand = NULL; - foreach_ptr(tableConstructionCommand, postLoadCommands) + foreach_declared_ptr(tableConstructionCommand, postLoadCommands) { Assert(CitusIsA(tableConstructionCommand, TableDDLCommand)); char *tableConstructionSQL = GetTableDDLCommand(tableConstructionCommand); @@ -968,7 +968,7 @@ ConvertTableInternal(TableConversionState *con) MemoryContext oldContext = MemoryContextSwitchTo(citusPerPartitionContext); char *attachPartitionCommand = NULL; - foreach_ptr(attachPartitionCommand, attachPartitionCommands) + foreach_declared_ptr(attachPartitionCommand, attachPartitionCommands) { MemoryContextReset(citusPerPartitionContext); @@ -993,7 +993,7 @@ ConvertTableInternal(TableConversionState *con) /* For now we only support cascade to colocation for alter_distributed_table UDF */ Assert(con->conversionType == ALTER_DISTRIBUTED_TABLE); - foreach_oid(colocatedTableId, con->colocatedTableList) + foreach_declared_oid(colocatedTableId, con->colocatedTableList) { if (colocatedTableId == con->relationId) { @@ -1023,7 +1023,7 @@ ConvertTableInternal(TableConversionState *con) if (con->cascadeToColocated != CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) { char *foreignKeyCommand = NULL; - foreach_ptr(foreignKeyCommand, foreignKeyCommands) + foreach_declared_ptr(foreignKeyCommand, foreignKeyCommands) { ExecuteQueryViaSPI(foreignKeyCommand, SPI_OK_UTILITY); } @@ -1059,7 +1059,7 @@ CopyTableConversionReturnIntoCurrentContext(TableConversionReturn *tableConversi tableConversionReturnCopy = palloc0(sizeof(TableConversionReturn)); List *copyForeignKeyCommands = NIL; char *foreignKeyCommand = NULL; - foreach_ptr(foreignKeyCommand, tableConversionReturn->foreignKeyCommands) + foreach_declared_ptr(foreignKeyCommand, tableConversionReturn->foreignKeyCommands) { char *copyForeignKeyCommand = MemoryContextStrdup(CurrentMemoryContext, foreignKeyCommand); @@ -1134,7 +1134,7 @@ DropIndexesNotSupportedByColumnar(Oid relationId, bool suppressNoticeMessages) RelationClose(columnarRelation); Oid indexId = InvalidOid; - foreach_oid(indexId, indexIdList) + foreach_declared_oid(indexId, indexIdList) { char *indexAmName = GetIndexAccessMethodName(indexId); if (extern_ColumnarSupportsIndexAM(indexAmName)) @@ -1394,7 +1394,7 @@ CreateTableConversion(TableConversionParameters *params) * since they will be handled separately. */ Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { if (PartitionTable(colocatedTableId)) { @@ -1610,7 +1610,7 @@ DoesCascadeDropUnsupportedObject(Oid classId, Oid objectId, HTAB *nodeMap) targetObjectId); HeapTuple depTup = NULL; - foreach_ptr(depTup, dependencyTupleList) + foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); @@ -1650,7 +1650,7 @@ GetViewCreationCommandsOfTable(Oid relationId) List *commands = NIL; Oid viewOid = InvalidOid; - foreach_oid(viewOid, views) + foreach_declared_oid(viewOid, views) { StringInfo query = makeStringInfo(); @@ -1688,7 +1688,7 @@ WrapTableDDLCommands(List *commandStrings) List *tableDDLCommands = NIL; char *command = NULL; - foreach_ptr(command, commandStrings) + foreach_declared_ptr(command, commandStrings) { tableDDLCommands = lappend(tableDDLCommands, makeTableDDLCommandString(command)); } @@ -1851,7 +1851,7 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, */ List *ownedSequences = getOwnedSequences_internal(sourceId, 0, DEPENDENCY_AUTO); Oid sequenceOid = InvalidOid; - foreach_oid(sequenceOid, ownedSequences) + foreach_declared_oid(sequenceOid, ownedSequences) { changeDependencyFor(RelationRelationId, sequenceOid, RelationRelationId, sourceId, targetId); @@ -1887,7 +1887,7 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, } char *justBeforeDropCommand = NULL; - foreach_ptr(justBeforeDropCommand, justBeforeDropCommands) + foreach_declared_ptr(justBeforeDropCommand, justBeforeDropCommands) { ExecuteQueryViaSPI(justBeforeDropCommand, SPI_OK_UTILITY); } @@ -2003,7 +2003,7 @@ CheckAlterDistributedTableConversionParameters(TableConversionState *con) Oid colocatedTableOid = InvalidOid; text *colocateWithText = cstring_to_text(con->colocateWith); Oid colocateWithTableOid = ResolveRelationId(colocateWithText, false); - foreach_oid(colocatedTableOid, con->colocatedTableList) + foreach_declared_oid(colocatedTableOid, con->colocatedTableList) { if (colocateWithTableOid == colocatedTableOid) { @@ -2235,7 +2235,7 @@ WillRecreateForeignKeyToReferenceTable(Oid relationId, { List *colocatedTableList = ColocatedTableList(relationId); Oid colocatedTableOid = InvalidOid; - foreach_oid(colocatedTableOid, colocatedTableList) + foreach_declared_oid(colocatedTableOid, colocatedTableList) { if (HasForeignKeyToReferenceTable(colocatedTableOid)) { @@ -2263,7 +2263,7 @@ WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId) List *foreignKeys = list_concat(referencingForeingKeys, referencedForeignKeys); Oid foreignKeyOid = InvalidOid; - foreach_oid(foreignKeyOid, foreignKeys) + foreach_declared_oid(foreignKeyOid, foreignKeys) { ereport(WARNING, (errmsg("foreign key %s will be dropped", get_constraint_name(foreignKeyOid)))); diff --git a/src/backend/distributed/commands/begin.c b/src/backend/distributed/commands/begin.c index b19b04484..3b5728868 100644 --- a/src/backend/distributed/commands/begin.c +++ b/src/backend/distributed/commands/begin.c @@ -33,7 +33,7 @@ SaveBeginCommandProperties(TransactionStmt *transactionStmt) * * While BEGIN can be quite frequent it will rarely have options set. */ - foreach_ptr(item, transactionStmt->options) + foreach_declared_ptr(item, transactionStmt->options) { A_Const *constant = (A_Const *) item->arg; diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index c88367462..02b175960 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -168,7 +168,7 @@ GetPartitionRelationIds(List *relationIdList) List *partitionRelationIdList = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (PartitionTable(relationId)) { @@ -189,7 +189,7 @@ LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode) { Oid relationId; relationIdList = SortList(relationIdList, CompareOids); - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { LockRelationOid(relationId, lockMode); } @@ -207,7 +207,7 @@ static void ErrorIfConvertingMultiLevelPartitionedTable(List *relationIdList) { Oid relationId; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (PartitionedTable(relationId) && PartitionTable(relationId)) { @@ -236,7 +236,7 @@ void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (!PartitionTable(relationId)) { @@ -300,7 +300,7 @@ bool RelationIdListHasReferenceTable(List *relationIdList) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (IsCitusTableType(relationId, REFERENCE_TABLE)) { @@ -322,7 +322,7 @@ GetFKeyCreationCommandsForRelationIdList(List *relationIdList) List *fKeyCreationCommands = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { List *relationFKeyCreationCommands = GetReferencingForeignConstaintCommands(relationId); @@ -342,7 +342,7 @@ static void DropRelationIdListForeignKeys(List *relationIdList, int fKeyFlags) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { DropRelationForeignKeys(relationId, fKeyFlags); } @@ -399,7 +399,7 @@ GetRelationDropFkeyCommands(Oid relationId, int fKeyFlags) List *relationFKeyIdList = GetForeignKeyOids(relationId, fKeyFlags); Oid foreignKeyId; - foreach_oid(foreignKeyId, relationFKeyIdList) + foreach_declared_oid(foreignKeyId, relationFKeyIdList) { char *dropFkeyCascadeCommand = GetDropFkeyCascadeCommand(foreignKeyId); dropFkeyCascadeCommandList = lappend(dropFkeyCascadeCommandList, @@ -450,7 +450,7 @@ ExecuteCascadeOperationForRelationIdList(List *relationIdList, cascadeOperationType) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { /* * The reason behind skipping certain table types in below loop is @@ -531,7 +531,7 @@ ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(List *utilityCommandL PG_TRY(); { char *utilityCommand = NULL; - foreach_ptr(utilityCommand, utilityCommandList) + foreach_declared_ptr(utilityCommand, utilityCommandList) { /* * CREATE MATERIALIZED VIEW commands need to be parsed/transformed, @@ -569,7 +569,7 @@ void ExecuteAndLogUtilityCommandList(List *utilityCommandList) { char *utilityCommand = NULL; - foreach_ptr(utilityCommand, utilityCommandList) + foreach_declared_ptr(utilityCommand, utilityCommandList) { ExecuteAndLogUtilityCommand(utilityCommand); } @@ -597,7 +597,7 @@ void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, bool skip_validation) { char *ddlCommand = NULL; - foreach_ptr(ddlCommand, ddlCommandList) + foreach_declared_ptr(ddlCommand, ddlCommandList) { ExecuteForeignKeyCreateCommand(ddlCommand, skip_validation); } diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index d95cdd353..e2baa1025 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -588,7 +588,7 @@ ErrorIfOptionListHasNoTableName(List *optionList) { char *table_nameString = "table_name"; DefElem *option = NULL; - foreach_ptr(option, optionList) + foreach_declared_ptr(option, optionList) { char *optionName = option->defname; if (strcmp(optionName, table_nameString) == 0) @@ -613,7 +613,7 @@ ForeignTableDropsTableNameOption(List *optionList) { char *table_nameString = "table_name"; DefElem *option = NULL; - foreach_ptr(option, optionList) + foreach_declared_ptr(option, optionList) { char *optionName = option->defname; DefElemAction optionAction = option->defaction; @@ -732,7 +732,7 @@ UpdateAutoConvertedForConnectedRelations(List *relationIds, bool autoConverted) List *relationIdList = NIL; Oid relid = InvalidOid; - foreach_oid(relid, relationIds) + foreach_declared_oid(relid, relationIds) { List *connectedRelations = GetForeignKeyConnectedRelationIdList(relid); relationIdList = list_concat_unique_oid(relationIdList, connectedRelations); @@ -740,7 +740,7 @@ UpdateAutoConvertedForConnectedRelations(List *relationIds, bool autoConverted) relationIdList = SortList(relationIdList, CompareOids); - foreach_oid(relid, relationIdList) + foreach_declared_oid(relid, relationIdList) { UpdatePgDistPartitionAutoConverted(relid, autoConverted); } @@ -776,7 +776,7 @@ GetShellTableDDLEventsForCitusLocalTable(Oid relationId) List *shellTableDDLEvents = NIL; TableDDLCommand *tableDDLCommand = NULL; - foreach_ptr(tableDDLCommand, tableDDLCommands) + foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); shellTableDDLEvents = lappend(shellTableDDLEvents, @@ -863,7 +863,7 @@ RenameShardRelationConstraints(Oid shardRelationId, uint64 shardId) List *constraintNameList = GetConstraintNameList(shardRelationId); char *constraintName = NULL; - foreach_ptr(constraintName, constraintNameList) + foreach_declared_ptr(constraintName, constraintNameList) { const char *commandString = GetRenameShardConstraintCommand(shardRelationId, constraintName, shardId); @@ -958,7 +958,7 @@ RenameShardRelationIndexes(Oid shardRelationId, uint64 shardId) List *indexOidList = GetExplicitIndexOidList(shardRelationId); Oid indexOid = InvalidOid; - foreach_oid(indexOid, indexOidList) + foreach_declared_oid(indexOid, indexOidList) { const char *commandString = GetRenameShardIndexCommand(indexOid, shardId); ExecuteAndLogUtilityCommand(commandString); @@ -1008,7 +1008,7 @@ RenameShardRelationStatistics(Oid shardRelationId, uint64 shardId) List *statsCommandList = GetRenameStatsCommandList(statsOidList, shardId); char *command = NULL; - foreach_ptr(command, statsCommandList) + foreach_declared_ptr(command, statsCommandList) { ExecuteAndLogUtilityCommand(command); } @@ -1044,7 +1044,7 @@ RenameShardRelationNonTruncateTriggers(Oid shardRelationId, uint64 shardId) List *triggerIdList = GetExplicitTriggerIdList(shardRelationId); Oid triggerId = InvalidOid; - foreach_oid(triggerId, triggerIdList) + foreach_declared_oid(triggerId, triggerIdList) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); @@ -1097,7 +1097,7 @@ DropRelationTruncateTriggers(Oid relationId) List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; - foreach_oid(triggerId, triggerIdList) + foreach_declared_oid(triggerId, triggerIdList) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); @@ -1177,7 +1177,7 @@ DropIdentitiesOnTable(Oid relationId) relation_close(relation, NoLock); char *dropCommand = NULL; - foreach_ptr(dropCommand, dropCommandList) + foreach_declared_ptr(dropCommand, dropCommandList) { /* * We need to disable/enable ddl propagation for this command, to prevent @@ -1220,7 +1220,7 @@ DropViewsOnTable(Oid relationId) List *reverseOrderedViews = ReversedOidList(views); Oid viewId = InvalidOid; - foreach_oid(viewId, reverseOrderedViews) + foreach_declared_oid(viewId, reverseOrderedViews) { char *viewName = get_rel_name(viewId); char *schemaName = get_namespace_name(get_rel_namespace(viewId)); @@ -1245,7 +1245,7 @@ ReversedOidList(List *oidList) { List *reversed = NIL; Oid oid = InvalidOid; - foreach_oid(oid, oidList) + foreach_declared_oid(oid, oidList) { reversed = lcons_oid(oid, reversed); } @@ -1297,7 +1297,7 @@ GetRenameStatsCommandList(List *statsOidList, uint64 shardId) { List *statsCommandList = NIL; Oid statsOid; - foreach_oid(statsOid, statsOidList) + foreach_declared_oid(statsOid, statsOidList) { HeapTuple tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); diff --git a/src/backend/distributed/commands/cluster.c b/src/backend/distributed/commands/cluster.c index 7a1dac302..44a1b6109 100644 --- a/src/backend/distributed/commands/cluster.c +++ b/src/backend/distributed/commands/cluster.c @@ -115,7 +115,7 @@ static bool IsClusterStmtVerbose_compat(ClusterStmt *clusterStmt) { DefElem *opt = NULL; - foreach_ptr(opt, clusterStmt->params) + foreach_declared_ptr(opt, clusterStmt->params) { if (strcmp(opt->defname, "verbose") == 0) { diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index 347a99e8a..de05efe45 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -235,7 +235,7 @@ PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, List *distributedObjects = NIL; List *distributedObjectAddresses = NIL; Node *object = NULL; - foreach_ptr(object, stmt->objects) + foreach_declared_ptr(object, stmt->objects) { /* TODO understand if the lock should be sth else */ Relation rel = NULL; /* not used, but required to pass to get_object_address */ @@ -267,7 +267,7 @@ PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, * remove the entries for the distributed objects on dropping */ ObjectAddress *address = NULL; - foreach_ptr(address, distributedObjectAddresses) + foreach_declared_ptr(address, distributedObjectAddresses) { UnmarkObjectDistributed(address); } @@ -303,7 +303,7 @@ DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess) List *objectAddresses = NIL; List *objNameList = NIL; - foreach_ptr(objNameList, stmt->objects) + foreach_declared_ptr(objNameList, stmt->objects) { Oid tsdictOid = get_ts_dict_oid(objNameList, missing_ok); @@ -328,7 +328,7 @@ DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostproces List *objectAddresses = NIL; List *objNameList = NIL; - foreach_ptr(objNameList, stmt->objects) + foreach_declared_ptr(objNameList, stmt->objects) { Oid tsconfigOid = get_ts_config_oid(objNameList, missing_ok); diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index fced0fe3a..2cba7e1a5 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -834,7 +834,7 @@ HashSplitPointsForShardList(List *shardList) List *splitPointList = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { int32 shardMaxValue = DatumGetInt32(shardInterval->maxValue); @@ -890,7 +890,7 @@ WorkerNodesForShardList(List *shardList) List *nodeIdList = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { WorkerNode *workerNode = ActiveShardPlacementWorkerNode(shardInterval->shardId); nodeIdList = lappend_int(nodeIdList, workerNode->nodeId); @@ -1340,7 +1340,7 @@ CreateCitusTable(Oid relationId, CitusTableType tableType, ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(citusPartitionContext); - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { MemoryContextReset(citusPartitionContext); @@ -1554,7 +1554,7 @@ ConvertCitusLocalTableToTableType(Oid relationId, CitusTableType tableType, MemoryContext oldContext = MemoryContextSwitchTo(citusPartitionContext); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { MemoryContextReset(citusPartitionContext); @@ -1704,7 +1704,7 @@ EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid ownerRelationId Oid attrDefOid; List *attrDefOids = GetAttrDefsFromSequence(seqOid); - foreach_oid(attrDefOid, attrDefOids) + foreach_declared_oid(attrDefOid, attrDefOids) { ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); @@ -1786,7 +1786,7 @@ static void EnsureDistributedSequencesHaveOneType(Oid relationId, List *seqInfoList) { SequenceInfo *seqInfo = NULL; - foreach_ptr(seqInfo, seqInfoList) + foreach_declared_ptr(seqInfo, seqInfoList) { if (!seqInfo->isNextValDefault) { diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 735fb16a9..cc8938ccb 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -69,7 +69,7 @@ EnsureDependenciesExistOnAllNodes(const ObjectAddress *target) /* collect all dependencies in creation order and get their ddl commands */ List *dependencies = GetDependenciesForObject(target); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { List *dependencyCommands = GetDependencyCreateDDLCommands(dependency); ddlCommands = list_concat(ddlCommands, dependencyCommands); @@ -108,7 +108,7 @@ EnsureDependenciesExistOnAllNodes(const ObjectAddress *target) */ List *addressSortedDependencies = SortList(dependenciesWithCommands, ObjectAddressComparator); - foreach_ptr(dependency, addressSortedDependencies) + foreach_declared_ptr(dependency, addressSortedDependencies) { LockDatabaseObject(dependency->classId, dependency->objectId, dependency->objectSubId, ExclusiveLock); @@ -134,7 +134,7 @@ EnsureDependenciesExistOnAllNodes(const ObjectAddress *target) else { WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; uint32 nodePort = workerNode->workerPort; @@ -150,7 +150,7 @@ EnsureDependenciesExistOnAllNodes(const ObjectAddress *target) * that objects have been created on worker nodes before marking them * distributed, so MarkObjectDistributed wouldn't fail. */ - foreach_ptr(dependency, dependenciesWithCommands) + foreach_declared_ptr(dependency, dependenciesWithCommands) { /* * pg_dist_object entries must be propagated with the super user, since @@ -173,7 +173,7 @@ void EnsureAllObjectDependenciesExistOnAllNodes(const List *targets) { ObjectAddress *target = NULL; - foreach_ptr(target, targets) + foreach_declared_ptr(target, targets) { EnsureDependenciesExistOnAllNodes(target); } @@ -230,7 +230,7 @@ DeferErrorIfCircularDependencyExists(const ObjectAddress *objectAddress) List *dependencies = GetAllDependenciesForObject(objectAddress); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { if (dependency->classId == objectAddress->classId && dependency->objectId == objectAddress->objectId && @@ -318,7 +318,7 @@ GetDistributableDependenciesForObject(const ObjectAddress *target) /* filter the ones that can be distributed */ ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { /* * TODO: maybe we can optimize the logic applied in below line. Actually we @@ -402,7 +402,7 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) INCLUDE_IDENTITY, creatingShellTableOnRemoteNode); TableDDLCommand *tableDDLCommand = NULL; - foreach_ptr(tableDDLCommand, tableDDLCommands) + foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); commandList = lappend(commandList, GetTableDDLCommand( @@ -564,7 +564,7 @@ GetAllDependencyCreateDDLCommands(const List *dependencies) List *commands = NIL; ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { commands = list_concat(commands, GetDependencyCreateDDLCommands(dependency)); } @@ -712,7 +712,7 @@ bool ShouldPropagateAnyObject(List *addresses) { ObjectAddress *address = NULL; - foreach_ptr(address, addresses) + foreach_declared_ptr(address, addresses) { if (ShouldPropagateObject(address)) { @@ -734,7 +734,7 @@ FilterObjectAddressListByPredicate(List *objectAddressList, AddressPredicate pre List *result = NIL; ObjectAddress *address = NULL; - foreach_ptr(address, objectAddressList) + foreach_declared_ptr(address, objectAddressList) { if (predicate(address)) { diff --git a/src/backend/distributed/commands/domain.c b/src/backend/distributed/commands/domain.c index 82ef80c0f..d62428ce4 100644 --- a/src/backend/distributed/commands/domain.c +++ b/src/backend/distributed/commands/domain.c @@ -210,7 +210,7 @@ MakeCollateClauseFromOid(Oid collationOid) getObjectIdentityParts(&collateAddress, &objName, &objArgs, false); char *name = NULL; - foreach_ptr(name, objName) + foreach_declared_ptr(name, objName) { collateClause->collname = lappend(collateClause->collname, makeString(name)); } diff --git a/src/backend/distributed/commands/extension.c b/src/backend/distributed/commands/extension.c index 8d4c6431b..17f9ff575 100644 --- a/src/backend/distributed/commands/extension.c +++ b/src/backend/distributed/commands/extension.c @@ -274,7 +274,7 @@ PreprocessDropExtensionStmt(Node *node, const char *queryString, /* unmark each distributed extension */ ObjectAddress *address = NULL; - foreach_ptr(address, distributedExtensionAddresses) + foreach_declared_ptr(address, distributedExtensionAddresses) { UnmarkObjectDistributed(address); } @@ -313,7 +313,7 @@ FilterDistributedExtensions(List *extensionObjectList) List *extensionNameList = NIL; String *objectName = NULL; - foreach_ptr(objectName, extensionObjectList) + foreach_declared_ptr(objectName, extensionObjectList) { const char *extensionName = strVal(objectName); const bool missingOk = true; @@ -351,7 +351,7 @@ ExtensionNameListToObjectAddressList(List *extensionObjectList) List *extensionObjectAddressList = NIL; String *objectName; - foreach_ptr(objectName, extensionObjectList) + foreach_declared_ptr(objectName, extensionObjectList) { /* * We set missingOk to false as we assume all the objects in @@ -527,7 +527,7 @@ MarkExistingObjectDependenciesDistributedIfSupported() List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid citusTableId = InvalidOid; - foreach_oid(citusTableId, citusTableIdList) + foreach_declared_oid(citusTableId, citusTableIdList) { if (!ShouldMarkRelationDistributed(citusTableId)) { @@ -571,7 +571,7 @@ MarkExistingObjectDependenciesDistributedIfSupported() */ List *viewList = GetAllViews(); Oid viewOid = InvalidOid; - foreach_oid(viewOid, viewList) + foreach_declared_oid(viewOid, viewList) { if (!ShouldMarkRelationDistributed(viewOid)) { @@ -605,7 +605,7 @@ MarkExistingObjectDependenciesDistributedIfSupported() List *distributedObjectAddressList = GetDistributedObjectAddressList(); ObjectAddress *distributedObjectAddress = NULL; - foreach_ptr(distributedObjectAddress, distributedObjectAddressList) + foreach_declared_ptr(distributedObjectAddress, distributedObjectAddressList) { List *distributableDependencyObjectAddresses = GetDistributableDependenciesForObject(distributedObjectAddress); @@ -627,7 +627,7 @@ MarkExistingObjectDependenciesDistributedIfSupported() SetLocalEnableMetadataSync(false); ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, uniqueObjectAddresses) + foreach_declared_ptr(objectAddress, uniqueObjectAddresses) { MarkObjectDistributed(objectAddress); } @@ -831,7 +831,7 @@ IsDropCitusExtensionStmt(Node *parseTree) /* now that we have a DropStmt, check if citus extension is among the objects to dropped */ String *objectName; - foreach_ptr(objectName, dropStmt->objects) + foreach_declared_ptr(objectName, dropStmt->objects) { const char *extensionName = strVal(objectName); @@ -1061,7 +1061,7 @@ GenerateGrantCommandsOnExtensionDependentFDWs(Oid extensionId) List *FDWOids = GetDependentFDWsToExtension(extensionId); Oid FDWOid = InvalidOid; - foreach_oid(FDWOid, FDWOids) + foreach_declared_oid(FDWOid, FDWOids) { Acl *aclEntry = GetPrivilegesForFDW(FDWOid); diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 2f60c3fb1..b7162b1a4 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -202,7 +202,7 @@ ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDis List *foreignKeyOids = GetForeignKeyOids(referencingTableId, flags); Oid foreignKeyOid = InvalidOid; - foreach_oid(foreignKeyOid, foreignKeyOids) + foreach_declared_oid(foreignKeyOid, foreignKeyOids) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyOid)); @@ -414,7 +414,7 @@ ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple) List *setDefaultAttrs = ForeignKeyGetDefaultingAttrs(pgConstraintTuple); AttrNumber setDefaultAttr = InvalidAttrNumber; - foreach_int(setDefaultAttr, setDefaultAttrs) + foreach_declared_int(setDefaultAttr, setDefaultAttrs) { if (ColumnDefaultsToNextVal(pgConstraintForm->conrelid, setDefaultAttr)) { @@ -727,7 +727,7 @@ ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId) GetForeignKeyIdsForColumn(columnName, relationId, searchForeignKeyColumnFlags); Oid foreignKeyId = InvalidOid; - foreach_oid(foreignKeyId, foreignKeyIdsColumnAppeared) + foreach_declared_oid(foreignKeyId, foreignKeyIdsColumnAppeared) { Oid referencedTableId = GetReferencedTableId(foreignKeyId); if (IsCitusTableType(referencedTableId, REFERENCE_TABLE)) @@ -901,7 +901,7 @@ GetForeignConstraintCommandsInternal(Oid relationId, int flags) int saveNestLevel = PushEmptySearchPath(); Oid foreignKeyOid = InvalidOid; - foreach_oid(foreignKeyOid, foreignKeyOids) + foreach_declared_oid(foreignKeyOid, foreignKeyOids) { char *statementDef = pg_get_constraintdef_command(foreignKeyOid); @@ -1157,7 +1157,7 @@ static Oid FindForeignKeyOidWithName(List *foreignKeyOids, const char *inputConstraintName) { Oid foreignKeyOid = InvalidOid; - foreach_oid(foreignKeyOid, foreignKeyOids) + foreach_declared_oid(foreignKeyOid, foreignKeyOids) { char *constraintName = get_constraint_name(foreignKeyOid); @@ -1472,7 +1472,7 @@ RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId) List *foreignKeysRelationInvolved = list_concat(referencingForeignKeys, referencedForeignKeys); Oid foreignKeyId = InvalidOid; - foreach_oid(foreignKeyId, foreignKeysRelationInvolved) + foreach_declared_oid(foreignKeyId, foreignKeysRelationInvolved) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyId)); if (!HeapTupleIsValid(heapTuple)) diff --git a/src/backend/distributed/commands/foreign_data_wrapper.c b/src/backend/distributed/commands/foreign_data_wrapper.c index a181e63a7..e095f9894 100644 --- a/src/backend/distributed/commands/foreign_data_wrapper.c +++ b/src/backend/distributed/commands/foreign_data_wrapper.c @@ -86,7 +86,7 @@ static bool NameListHasFDWOwnedByDistributedExtension(List *FDWNames) { String *FDWValue = NULL; - foreach_ptr(FDWValue, FDWNames) + foreach_declared_ptr(FDWValue, FDWNames) { /* captures the extension address during lookup */ ObjectAddress *extensionAddress = palloc0(sizeof(ObjectAddress)); diff --git a/src/backend/distributed/commands/foreign_server.c b/src/backend/distributed/commands/foreign_server.c index d2e575564..096690e2b 100644 --- a/src/backend/distributed/commands/foreign_server.c +++ b/src/backend/distributed/commands/foreign_server.c @@ -229,7 +229,7 @@ RecreateForeignServerStmt(Oid serverId) int location = -1; DefElem *option = NULL; - foreach_ptr(option, server->options) + foreach_declared_ptr(option, server->options) { DefElem *copyOption = makeDefElem(option->defname, option->arg, location); createStmt->options = lappend(createStmt->options, copyOption); @@ -247,7 +247,7 @@ static bool NameListHasDistributedServer(List *serverNames) { String *serverValue = NULL; - foreach_ptr(serverValue, serverNames) + foreach_declared_ptr(serverValue, serverNames) { List *addresses = GetObjectAddressByServerName(strVal(serverValue), false); diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 3a029b47f..57232d965 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -256,7 +256,7 @@ create_distributed_function(PG_FUNCTION_ARGS) createFunctionSQL, alterFunctionOwnerSQL); List *grantDDLCommands = GrantOnFunctionDDLCommands(funcOid); char *grantOnFunctionSQL = NULL; - foreach_ptr(grantOnFunctionSQL, grantDDLCommands) + foreach_declared_ptr(grantOnFunctionSQL, grantDDLCommands) { appendStringInfo(&ddlCommand, ";%s", grantOnFunctionSQL); } @@ -370,7 +370,7 @@ ErrorIfAnyNodeDoesNotHaveMetadata(void) ActivePrimaryNonCoordinatorNodeList(ShareLock); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if (!workerNode->hasMetadata) { @@ -1477,7 +1477,7 @@ CreateFunctionStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) objectWithArgs->objname = stmt->funcname; FunctionParameter *funcParam = NULL; - foreach_ptr(funcParam, stmt->parameters) + foreach_declared_ptr(funcParam, stmt->parameters) { if (ShouldAddFunctionSignature(funcParam->mode)) { @@ -1520,7 +1520,7 @@ DefineAggregateStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess if (stmt->args != NIL) { FunctionParameter *funcParam = NULL; - foreach_ptr(funcParam, linitial(stmt->args)) + foreach_declared_ptr(funcParam, linitial(stmt->args)) { objectWithArgs->objargs = lappend(objectWithArgs->objargs, funcParam->argType); @@ -1529,7 +1529,7 @@ DefineAggregateStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess else { DefElem *defItem = NULL; - foreach_ptr(defItem, stmt->definition) + foreach_declared_ptr(defItem, stmt->definition) { /* * If no explicit args are given, pg includes basetype in the signature. @@ -1934,7 +1934,7 @@ static void ErrorIfUnsupportedAlterFunctionStmt(AlterFunctionStmt *stmt) { DefElem *action = NULL; - foreach_ptr(action, stmt->actions) + foreach_declared_ptr(action, stmt->actions) { if (strcmp(action->defname, "set") == 0) { @@ -2041,7 +2041,7 @@ PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, List *grantFunctionList = NIL; ObjectAddress *functionAddress = NULL; - foreach_ptr(functionAddress, distributedFunctions) + foreach_declared_ptr(functionAddress, distributedFunctions) { ObjectWithArgs *distFunction = ObjectWithArgsFromOid( functionAddress->objectId); @@ -2084,7 +2084,7 @@ PostprocessGrantOnFunctionStmt(Node *node, const char *queryString) } ObjectAddress *functionAddress = NULL; - foreach_ptr(functionAddress, distributedFunctions) + foreach_declared_ptr(functionAddress, distributedFunctions) { EnsureAllObjectDependenciesExistOnAllNodes(list_make1(functionAddress)); } @@ -2121,7 +2121,7 @@ FilterDistributedFunctions(GrantStmt *grantStmt) /* iterate over all namespace names provided to get their oid's */ String *namespaceValue = NULL; - foreach_ptr(namespaceValue, grantStmt->objects) + foreach_declared_ptr(namespaceValue, grantStmt->objects) { char *nspname = strVal(namespaceValue); bool missing_ok = false; @@ -2133,7 +2133,7 @@ FilterDistributedFunctions(GrantStmt *grantStmt) * iterate over all distributed functions to filter the ones * that belong to one of the namespaces from above */ - foreach_ptr(distributedFunction, distributedFunctionList) + foreach_declared_ptr(distributedFunction, distributedFunctionList) { Oid namespaceOid = get_func_namespace(distributedFunction->objectId); @@ -2152,7 +2152,7 @@ FilterDistributedFunctions(GrantStmt *grantStmt) { bool missingOk = false; ObjectWithArgs *objectWithArgs = NULL; - foreach_ptr(objectWithArgs, grantStmt->objects) + foreach_declared_ptr(objectWithArgs, grantStmt->objects) { ObjectAddress *functionAddress = palloc0(sizeof(ObjectAddress)); functionAddress->classId = ProcedureRelationId; diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 5874f1780..2b6941393 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -334,7 +334,7 @@ ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcesso List *indexIdList = RelationGetIndexList(relation); Oid indexId = InvalidOid; - foreach_oid(indexId, indexIdList) + foreach_declared_oid(indexId, indexIdList) { HeapTuple indexTuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(indexId)); if (!HeapTupleIsValid(indexTuple)) @@ -703,7 +703,7 @@ PreprocessDropIndexStmt(Node *node, const char *dropIndexCommand, /* check if any of the indexes being dropped belong to a distributed table */ List *objectNameList = NULL; - foreach_ptr(objectNameList, dropIndexStatement->objects) + foreach_declared_ptr(objectNameList, dropIndexStatement->objects) { struct DropRelationCallbackState state; uint32 rvrFlags = RVR_MISSING_OK; @@ -874,7 +874,7 @@ ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement) /* error out if any of the subcommands are unsupported */ List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; @@ -926,7 +926,7 @@ CreateIndexTaskList(IndexStmt *indexStmt) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; @@ -971,7 +971,7 @@ CreateReindexTaskList(Oid relationId, ReindexStmt *reindexStmt) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; @@ -1220,7 +1220,7 @@ ErrorIfUnsupportedIndexStmt(IndexStmt *createIndexStatement) Var *partitionKey = DistPartitionKeyOrError(relationId); List *indexParameterList = createIndexStatement->indexParams; IndexElem *indexElement = NULL; - foreach_ptr(indexElement, indexParameterList) + foreach_declared_ptr(indexElement, indexParameterList) { const char *columnName = indexElement->name; @@ -1289,7 +1289,7 @@ DropIndexTaskList(Oid relationId, Oid indexId, DropStmt *dropStmt) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardIndexName = pstrdup(indexName); diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 1517c1d5a..b9e905850 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -1957,7 +1957,7 @@ ShardIntervalListHasLocalPlacements(List *shardIntervalList) { int32 localGroupId = GetLocalGroupId(); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { if (ActiveShardPlacementOnGroup(localGroupId, shardInterval->shardId) != NULL) { @@ -2452,7 +2452,7 @@ ProcessAppendToShardOption(Oid relationId, CopyStmt *copyStatement) bool appendToShardSet = false; DefElem *defel = NULL; - foreach_ptr(defel, copyStatement->options) + foreach_declared_ptr(defel, copyStatement->options) { if (strncmp(defel->defname, APPEND_TO_SHARD_OPTION, NAMEDATALEN) == 0) { diff --git a/src/backend/distributed/commands/policy.c b/src/backend/distributed/commands/policy.c index a2a926b66..97292e29d 100644 --- a/src/backend/distributed/commands/policy.c +++ b/src/backend/distributed/commands/policy.c @@ -48,7 +48,7 @@ CreatePolicyCommands(Oid relationId) List *policyList = GetPolicyListForRelation(relationId); RowSecurityPolicy *policy; - foreach_ptr(policy, policyList) + foreach_declared_ptr(policy, policyList) { char *createPolicyCommand = CreatePolicyCommandForPolicy(relationId, policy); commands = lappend(commands, makeTableDDLCommandString(createPolicyCommand)); @@ -88,7 +88,7 @@ GetPolicyListForRelation(Oid relationId) List *policyList = NIL; RowSecurityPolicy *policy; - foreach_ptr(policy, relation->rd_rsdesc->policies) + foreach_declared_ptr(policy, relation->rd_rsdesc->policies) { policyList = lappend(policyList, policy); } @@ -310,7 +310,7 @@ GetPolicyByName(Oid relationId, const char *policyName) List *policyList = GetPolicyListForRelation(relationId); RowSecurityPolicy *policy = NULL; - foreach_ptr(policy, policyList) + foreach_declared_ptr(policy, policyList) { if (strncmp(policy->policy_name, policyName, NAMEDATALEN) == 0) { diff --git a/src/backend/distributed/commands/publication.c b/src/backend/distributed/commands/publication.c index 67d33832c..efb017977 100644 --- a/src/backend/distributed/commands/publication.c +++ b/src/backend/distributed/commands/publication.c @@ -158,7 +158,7 @@ BuildCreatePublicationStmt(Oid publicationId) List *schemaIds = GetPublicationSchemas(publicationId); Oid schemaId = InvalidOid; - foreach_oid(schemaId, schemaIds) + foreach_declared_oid(schemaId, schemaIds) { char *schemaName = get_namespace_name(schemaId); @@ -182,7 +182,7 @@ BuildCreatePublicationStmt(Oid publicationId) /* mainly for consistent ordering in test output */ relationIds = SortList(relationIds, CompareOids); - foreach_oid(relationId, relationIds) + foreach_declared_oid(relationId, relationIds) { #if (PG_VERSION_NUM >= PG_VERSION_15) bool tableOnly = false; @@ -420,7 +420,7 @@ GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd) List *publicationIds = GetRelationPublications(relationId); Oid publicationId = InvalidOid; - foreach_oid(publicationId, publicationIds) + foreach_declared_oid(publicationId, publicationIds) { char *command = GetAlterPublicationTableDDLCommand(publicationId, relationId, isAdd); diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 85af6e368..05959ca3e 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -161,7 +161,7 @@ PostprocessAlterRoleStmt(Node *node, const char *queryString) AlterRoleStmt *stmt = castNode(AlterRoleStmt, node); DefElem *option = NULL; - foreach_ptr(option, stmt->options) + foreach_declared_ptr(option, stmt->options) { if (strcasecmp(option->defname, "password") == 0) { @@ -562,7 +562,7 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) { List *grantRoleStmts = GenerateGrantRoleStmtsOfRole(roleOid); Node *stmt = NULL; - foreach_ptr(stmt, grantRoleStmts) + foreach_declared_ptr(stmt, grantRoleStmts) { completeRoleList = lappend(completeRoleList, DeparseTreeNode(stmt)); } @@ -760,7 +760,7 @@ MakeSetStatementArguments(char *configurationName, char *configurationValue) } char *configuration = NULL; - foreach_ptr(configuration, configurationList) + foreach_declared_ptr(configuration, configurationList) { Node *arg = makeStringConst(configuration, -1); args = lappend(args, arg); @@ -796,7 +796,7 @@ GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options) List *stmts = NIL; DefElem *option = NULL; - foreach_ptr(option, options) + foreach_declared_ptr(option, options) { if (strcmp(option->defname, "adminmembers") != 0 && strcmp(option->defname, "rolemembers") != 0 && @@ -938,7 +938,7 @@ PreprocessCreateRoleStmt(Node *node, const char *queryString, /* deparse all grant statements and add them to the commands list */ Node *stmt = NULL; - foreach_ptr(stmt, grantRoleStmts) + foreach_declared_ptr(stmt, grantRoleStmts) { commands = lappend(commands, DeparseTreeNode(stmt)); } @@ -1064,7 +1064,7 @@ void UnmarkRolesDistributed(List *roles) { Node *roleNode = NULL; - foreach_ptr(roleNode, roles) + foreach_declared_ptr(roleNode, roles) { RoleSpec *role = castNode(RoleSpec, roleNode); ObjectAddress roleAddress = { 0 }; @@ -1094,7 +1094,7 @@ FilterDistributedRoles(List *roles) { List *distributedRoles = NIL; Node *roleNode = NULL; - foreach_ptr(roleNode, roles) + foreach_declared_ptr(roleNode, roles) { RoleSpec *role = castNode(RoleSpec, roleNode); Oid roleOid = get_rolespec_oid(role, true); @@ -1189,7 +1189,7 @@ PostprocessGrantRoleStmt(Node *node, const char *queryString) GrantRoleStmt *stmt = castNode(GrantRoleStmt, node); RoleSpec *role = NULL; - foreach_ptr(role, stmt->grantee_roles) + foreach_declared_ptr(role, stmt->grantee_roles) { Oid roleOid = get_rolespec_oid(role, false); ObjectAddress *roleAddress = palloc0(sizeof(ObjectAddress)); @@ -1213,7 +1213,7 @@ IsGrantRoleWithInheritOrSetOption(GrantRoleStmt *stmt) { #if PG_VERSION_NUM >= PG_VERSION_16 DefElem *opt = NULL; - foreach_ptr(opt, stmt->opt) + foreach_declared_ptr(opt, stmt->opt) { if (strcmp(opt->defname, "inherit") == 0 || strcmp(opt->defname, "set") == 0) { diff --git a/src/backend/distributed/commands/schema.c b/src/backend/distributed/commands/schema.c index 7f79897fa..b079fe3f6 100644 --- a/src/backend/distributed/commands/schema.c +++ b/src/backend/distributed/commands/schema.c @@ -162,7 +162,7 @@ PreprocessDropSchemaStmt(Node *node, const char *queryString, EnsureSequentialMode(OBJECT_SCHEMA); String *schemaVal = NULL; - foreach_ptr(schemaVal, distributedSchemas) + foreach_declared_ptr(schemaVal, distributedSchemas) { if (SchemaHasDistributedTableWithFKey(strVal(schemaVal))) { @@ -322,7 +322,7 @@ FilterDistributedSchemas(List *schemas) List *distributedSchemas = NIL; String *schemaValue = NULL; - foreach_ptr(schemaValue, schemas) + foreach_declared_ptr(schemaValue, schemas) { const char *schemaName = strVal(schemaValue); Oid schemaOid = get_namespace_oid(schemaName, true); @@ -443,7 +443,7 @@ GetGrantCommandsFromCreateSchemaStmt(Node *node) CreateSchemaStmt *stmt = castNode(CreateSchemaStmt, node); Node *element = NULL; - foreach_ptr(element, stmt->schemaElts) + foreach_declared_ptr(element, stmt->schemaElts) { if (!IsA(element, GrantStmt)) { @@ -480,7 +480,7 @@ static bool CreateSchemaStmtCreatesTable(CreateSchemaStmt *stmt) { Node *element = NULL; - foreach_ptr(element, stmt->schemaElts) + foreach_declared_ptr(element, stmt->schemaElts) { /* * CREATE TABLE AS and CREATE FOREIGN TABLE commands cannot be diff --git a/src/backend/distributed/commands/schema_based_sharding.c b/src/backend/distributed/commands/schema_based_sharding.c index 7cde96982..6635d6817 100644 --- a/src/backend/distributed/commands/schema_based_sharding.c +++ b/src/backend/distributed/commands/schema_based_sharding.c @@ -174,7 +174,7 @@ EnsureTableKindSupportedForTenantSchema(Oid relationId) List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { ErrorIfIllegalPartitioningInTenantSchema(relationId, partitionRelationId); } @@ -199,7 +199,7 @@ EnsureFKeysForTenantTable(Oid relationId) int fKeyReferencingFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *referencingForeignKeys = GetForeignKeyOids(relationId, fKeyReferencingFlags); Oid foreignKeyId = InvalidOid; - foreach_oid(foreignKeyId, referencingForeignKeys) + foreach_declared_oid(foreignKeyId, referencingForeignKeys) { Oid referencingTableId = GetReferencingTableId(foreignKeyId); Oid referencedTableId = GetReferencedTableId(foreignKeyId); @@ -232,7 +232,7 @@ EnsureFKeysForTenantTable(Oid relationId) int fKeyReferencedFlags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *referencedForeignKeys = GetForeignKeyOids(relationId, fKeyReferencedFlags); - foreach_oid(foreignKeyId, referencedForeignKeys) + foreach_declared_oid(foreignKeyId, referencedForeignKeys) { Oid referencingTableId = GetReferencingTableId(foreignKeyId); Oid referencedTableId = GetReferencedTableId(foreignKeyId); @@ -429,7 +429,7 @@ EnsureSchemaCanBeDistributed(Oid schemaId, List *schemaTableIdList) } Oid relationId = InvalidOid; - foreach_oid(relationId, schemaTableIdList) + foreach_declared_oid(relationId, schemaTableIdList) { EnsureTenantTable(relationId, "citus_schema_distribute"); } @@ -637,7 +637,7 @@ citus_schema_distribute(PG_FUNCTION_ARGS) List *tableIdListInSchema = SchemaGetNonShardTableIdList(schemaId); List *tableIdListToConvert = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, tableIdListInSchema) + foreach_declared_oid(relationId, tableIdListInSchema) { /* prevent concurrent drop of the relation */ LockRelationOid(relationId, AccessShareLock); @@ -675,7 +675,7 @@ citus_schema_distribute(PG_FUNCTION_ARGS) * tables. */ List *originalForeignKeyRecreationCommands = NIL; - foreach_oid(relationId, tableIdListToConvert) + foreach_declared_oid(relationId, tableIdListToConvert) { List *fkeyCommandsForRelation = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, @@ -741,7 +741,7 @@ citus_schema_undistribute(PG_FUNCTION_ARGS) List *tableIdListInSchema = SchemaGetNonShardTableIdList(schemaId); List *tableIdListToConvert = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, tableIdListInSchema) + foreach_declared_oid(relationId, tableIdListInSchema) { /* prevent concurrent drop of the relation */ LockRelationOid(relationId, AccessShareLock); @@ -883,7 +883,7 @@ TenantSchemaPickAnchorShardId(Oid schemaId) } Oid relationId = InvalidOid; - foreach_oid(relationId, tablesInSchema) + foreach_declared_oid(relationId, tablesInSchema) { /* * Make sure the relation isn't dropped for the remainder of diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index cfb55faf7..4af4c4853 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -123,7 +123,7 @@ static bool OptionsSpecifyOwnedBy(List *optionList, Oid *ownedByTableId) { DefElem *defElem = NULL; - foreach_ptr(defElem, optionList) + foreach_declared_ptr(defElem, optionList) { if (strcmp(defElem->defname, "owned_by") == 0) { @@ -202,7 +202,7 @@ ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList, } Oid ownedSequenceId = InvalidOid; - foreach_oid(ownedSequenceId, columnOwnedSequences) + foreach_declared_oid(ownedSequenceId, columnOwnedSequences) { /* * A column might have multiple sequences one via OWNED BY one another @@ -288,7 +288,7 @@ PreprocessDropSequenceStmt(Node *node, const char *queryString, */ List *deletingSequencesList = stmt->objects; List *objectNameList = NULL; - foreach_ptr(objectNameList, deletingSequencesList) + foreach_declared_ptr(objectNameList, deletingSequencesList) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); @@ -322,7 +322,7 @@ PreprocessDropSequenceStmt(Node *node, const char *queryString, /* remove the entries for the distributed objects on dropping */ ObjectAddress *address = NULL; - foreach_ptr(address, distributedSequenceAddresses) + foreach_declared_ptr(address, distributedSequenceAddresses) { UnmarkObjectDistributed(address); } @@ -356,7 +356,7 @@ SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess) List *droppingSequencesList = dropSeqStmt->objects; List *objectNameList = NULL; - foreach_ptr(objectNameList, droppingSequencesList) + foreach_declared_ptr(objectNameList, droppingSequencesList) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); @@ -476,7 +476,7 @@ PreprocessAlterSequenceStmt(Node *node, const char *queryString, { List *options = stmt->options; DefElem *defel = NULL; - foreach_ptr(defel, options) + foreach_declared_ptr(defel, options) { if (strcmp(defel->defname, "as") == 0) { @@ -511,7 +511,7 @@ SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char depTyp Oid relationId; List *relations = GetDependentRelationsWithSequence(sequenceAddress->objectId, depType); - foreach_oid(relationId, relations) + foreach_declared_oid(relationId, relations) { if (IsCitusTable(relationId)) { @@ -930,7 +930,7 @@ PostprocessGrantOnSequenceStmt(Node *node, const char *queryString) EnsureCoordinator(); RangeVar *sequence = NULL; - foreach_ptr(sequence, distributedSequences) + foreach_declared_ptr(sequence, distributedSequences) { ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); Oid sequenceOid = RangeVarGetRelid(sequence, NoLock, false); @@ -1014,7 +1014,7 @@ FilterDistributedSequences(GrantStmt *stmt) /* iterate over all namespace names provided to get their oid's */ List *namespaceOidList = NIL; String *namespaceValue = NULL; - foreach_ptr(namespaceValue, stmt->objects) + foreach_declared_ptr(namespaceValue, stmt->objects) { char *nspname = strVal(namespaceValue); bool missing_ok = false; @@ -1028,7 +1028,7 @@ FilterDistributedSequences(GrantStmt *stmt) */ List *distributedSequenceList = DistributedSequenceList(); ObjectAddress *sequenceAddress = NULL; - foreach_ptr(sequenceAddress, distributedSequenceList) + foreach_declared_ptr(sequenceAddress, distributedSequenceList) { Oid namespaceOid = get_rel_namespace(sequenceAddress->objectId); @@ -1052,7 +1052,7 @@ FilterDistributedSequences(GrantStmt *stmt) { bool missing_ok = false; RangeVar *sequenceRangeVar = NULL; - foreach_ptr(sequenceRangeVar, stmt->objects) + foreach_declared_ptr(sequenceRangeVar, stmt->objects) { Oid sequenceOid = RangeVarGetRelid(sequenceRangeVar, NoLock, missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); diff --git a/src/backend/distributed/commands/statistics.c b/src/backend/distributed/commands/statistics.c index 5fac767fd..45d79afe4 100644 --- a/src/backend/distributed/commands/statistics.c +++ b/src/backend/distributed/commands/statistics.c @@ -184,7 +184,7 @@ PreprocessDropStatisticsStmt(Node *node, const char *queryString, List *ddlJobs = NIL; List *processedStatsOids = NIL; List *objectNameList = NULL; - foreach_ptr(objectNameList, dropStatisticsStmt->objects) + foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { Oid statsOid = get_statistics_object_oid(objectNameList, dropStatisticsStmt->missing_ok); @@ -234,7 +234,7 @@ DropStatisticsObjectAddress(Node *node, bool missing_ok, bool isPostprocess) List *objectAddresses = NIL; List *objectNameList = NULL; - foreach_ptr(objectNameList, dropStatisticsStmt->objects) + foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { Oid statsOid = get_statistics_object_oid(objectNameList, dropStatisticsStmt->missing_ok); @@ -535,7 +535,7 @@ GetExplicitStatisticsCommandList(Oid relationId) int saveNestLevel = PushEmptySearchPath(); Oid statisticsId = InvalidOid; - foreach_oid(statisticsId, statisticsIdList) + foreach_declared_oid(statisticsId, statisticsIdList) { /* we need create commands for already created stats before distribution */ Datum commandText = DirectFunctionCall1(pg_get_statisticsobjdef, @@ -606,7 +606,7 @@ GetExplicitStatisticsSchemaIdList(Oid relationId) RelationClose(relation); Oid statsId = InvalidOid; - foreach_oid(statsId, statsIdList) + foreach_declared_oid(statsId, statsIdList) { HeapTuple heapTuple = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsId)); if (!HeapTupleIsValid(heapTuple)) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 30b028b79..e65f57961 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -154,7 +154,7 @@ PreprocessDropTableStmt(Node *node, const char *queryString, Assert(dropTableStatement->removeType == OBJECT_TABLE); List *tableNameList = NULL; - foreach_ptr(tableNameList, dropTableStatement->objects) + foreach_declared_ptr(tableNameList, dropTableStatement->objects) { RangeVar *tableRangeVar = makeRangeVarFromNameList(tableNameList); bool missingOK = true; @@ -202,7 +202,7 @@ PreprocessDropTableStmt(Node *node, const char *queryString, SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { char *detachPartitionCommand = GenerateDetachPartitionCommand(partitionRelationId); @@ -263,7 +263,7 @@ PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString) } RangeVar *parentRelation = NULL; - foreach_ptr(parentRelation, createStatement->inhRelations) + foreach_declared_ptr(parentRelation, createStatement->inhRelations) { Oid parentRelationId = RangeVarGetRelid(parentRelation, NoLock, missingOk); @@ -480,7 +480,7 @@ PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, { List *commandList = alterTableStatement->cmds; AlterTableCmd *alterTableCommand = NULL; - foreach_ptr(alterTableCommand, commandList) + foreach_declared_ptr(alterTableCommand, commandList) { if (alterTableCommand->subtype == AT_AttachPartition) { @@ -792,7 +792,7 @@ ChooseForeignKeyConstraintNameAddition(List *columnNames) String *columnNameString = NULL; - foreach_ptr(columnNameString, columnNames) + foreach_declared_ptr(columnNameString, columnNames) { const char *name = strVal(columnNameString); @@ -1314,7 +1314,7 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, AlterTableCmd *newCmd = makeNode(AlterTableCmd); AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; @@ -1418,7 +1418,7 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { @@ -1442,7 +1442,7 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, deparseAT = true; constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (ConstrTypeCitusCanDefaultName(constraint->contype)) { @@ -1467,7 +1467,7 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, */ constraint = NULL; int constraintIdx = 0; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { @@ -1696,7 +1696,7 @@ DeparserSupportsAlterTableAddColumn(AlterTableStmt *alterTableStatement, { ColumnDef *columnDefinition = (ColumnDef *) addColumnSubCommand->def; Constraint *constraint = NULL; - foreach_ptr(constraint, columnDefinition->constraints) + foreach_declared_ptr(constraint, columnDefinition->constraints) { if (constraint->contype == CONSTR_CHECK) { @@ -1792,7 +1792,7 @@ static bool RelationIdListContainsCitusTableType(List *relationIdList, CitusTableType citusTableType) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (IsCitusTableType(relationId, citusTableType)) { @@ -1812,7 +1812,7 @@ static bool RelationIdListContainsPostgresTable(List *relationIdList) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (OidIsValid(relationId) && !IsCitusTable(relationId)) { @@ -1851,7 +1851,7 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement * change in below loop due to CreateCitusLocalTable. */ RangeVar *relationRangeVar; - foreach_ptr(relationRangeVar, relationRangeVarList) + foreach_declared_ptr(relationRangeVar, relationRangeVarList) { List *commandList = alterTableStatement->cmds; LOCKMODE lockMode = AlterTableGetLockLevel(commandList); @@ -1979,7 +1979,7 @@ RangeVarListHasLocalRelationConvertedByUser(List *relationRangeVarList, AlterTableStmt *alterTableStatement) { RangeVar *relationRangeVar; - foreach_ptr(relationRangeVar, relationRangeVarList) + foreach_declared_ptr(relationRangeVar, relationRangeVarList) { /* * Here we iterate the relation list, and if at least one of the relations @@ -2076,7 +2076,7 @@ GetAlterTableAddFKeyConstraintList(AlterTableStmt *alterTableStatement) List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { List *commandForeignKeyConstraintList = GetAlterTableCommandFKeyConstraintList(command); @@ -2116,7 +2116,7 @@ GetAlterTableCommandFKeyConstraintList(AlterTableCmd *command) List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { @@ -2139,7 +2139,7 @@ GetRangeVarListFromFKeyConstraintList(List *fKeyConstraintList) List *rightRelationRangeVarList = NIL; Constraint *fKeyConstraint = NULL; - foreach_ptr(fKeyConstraint, fKeyConstraintList) + foreach_declared_ptr(fKeyConstraint, fKeyConstraintList) { RangeVar *rightRelationRangeVar = fKeyConstraint->pktable; rightRelationRangeVarList = lappend(rightRelationRangeVarList, @@ -2160,7 +2160,7 @@ GetRelationIdListFromRangeVarList(List *rangeVarList, LOCKMODE lockMode, bool mi List *relationIdList = NIL; RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, rangeVarList) + foreach_declared_ptr(rangeVar, rangeVarList) { Oid rightRelationId = RangeVarGetRelid(rangeVar, lockMode, missingOk); relationIdList = lappend_oid(relationIdList, rightRelationId); @@ -2234,7 +2234,7 @@ AlterTableDropsForeignKey(AlterTableStmt *alterTableStatement) Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); AlterTableCmd *command = NULL; - foreach_ptr(command, alterTableStatement->cmds) + foreach_declared_ptr(command, alterTableStatement->cmds) { AlterTableType alterTableType = command->subtype; @@ -2296,7 +2296,7 @@ AnyForeignKeyDependsOnIndex(Oid indexId) GetPgDependTuplesForDependingObjects(dependentObjectClassId, dependentObjectId); HeapTuple dependencyTuple = NULL; - foreach_ptr(dependencyTuple, dependencyTupleList) + foreach_declared_ptr(dependencyTuple, dependencyTupleList) { Form_pg_depend dependencyForm = (Form_pg_depend) GETSTRUCT(dependencyTuple); Oid dependingClassId = dependencyForm->classid; @@ -2484,7 +2484,7 @@ SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement, * shards anyway. */ AlterTableCmd *command = NULL; - foreach_ptr(command, alterTableStatement->cmds) + foreach_declared_ptr(command, alterTableStatement->cmds) { AlterTableType alterTableType = command->subtype; @@ -2565,7 +2565,7 @@ ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement) /* then check if any of subcommands drop partition column.*/ List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; if (alterTableType == AT_DropColumn) @@ -2634,7 +2634,7 @@ PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; @@ -2670,7 +2670,7 @@ PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) } Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->conname == NULL && (constraint->contype == CONSTR_PRIMARY || @@ -2690,7 +2690,7 @@ PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) * that sequence is supported */ constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { @@ -2802,7 +2802,7 @@ FixAlterTableStmtIndexNames(AlterTableStmt *alterTableStatement) List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; @@ -3165,7 +3165,7 @@ ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, List *indexOidList = RelationGetIndexList(relation); Oid indexOid = InvalidOid; - foreach_oid(indexOid, indexOidList) + foreach_declared_oid(indexOid, indexOidList) { Relation indexDesc = index_open(indexOid, RowExclusiveLock); bool hasDistributionColumn = false; @@ -3310,7 +3310,7 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) /* error out if any of the subcommands are unsupported */ AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; @@ -3385,7 +3385,7 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) Constraint *columnConstraint = NULL; - foreach_ptr(columnConstraint, column->constraints) + foreach_declared_ptr(columnConstraint, column->constraints) { if (columnConstraint->contype == CONSTR_IDENTITY) { @@ -3417,7 +3417,7 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) List *columnConstraints = column->constraints; Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { @@ -3770,7 +3770,7 @@ SetupExecutionModeForAlterTable(Oid relationId, AlterTableCmd *command) List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { @@ -3970,10 +3970,10 @@ SetInterShardDDLTaskPlacementList(Task *task, ShardInterval *leftShardInterval, List *intersectedPlacementList = NIL; ShardPlacement *leftShardPlacement = NULL; - foreach_ptr(leftShardPlacement, leftShardPlacementList) + foreach_declared_ptr(leftShardPlacement, leftShardPlacementList) { ShardPlacement *rightShardPlacement = NULL; - foreach_ptr(rightShardPlacement, rightShardPlacementList) + foreach_declared_ptr(rightShardPlacement, rightShardPlacementList) { if (leftShardPlacement->nodeId == rightShardPlacement->nodeId) { diff --git a/src/backend/distributed/commands/trigger.c b/src/backend/distributed/commands/trigger.c index 74cb6259f..01ee72d31 100644 --- a/src/backend/distributed/commands/trigger.c +++ b/src/backend/distributed/commands/trigger.c @@ -81,7 +81,7 @@ GetExplicitTriggerCommandList(Oid relationId) List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; - foreach_oid(triggerId, triggerIdList) + foreach_declared_oid(triggerId, triggerIdList) { bool prettyOutput = false; Datum commandText = DirectFunctionCall2(pg_get_triggerdef_ext, @@ -742,7 +742,7 @@ ErrorIfRelationHasUnsupportedTrigger(Oid relationId) List *relationTriggerList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; - foreach_oid(triggerId, relationTriggerList) + foreach_declared_oid(triggerId, relationTriggerList) { ObjectAddress triggerObjectAddress = InvalidObjectAddress; ObjectAddressSet(triggerObjectAddress, TriggerRelationId, triggerId); diff --git a/src/backend/distributed/commands/truncate.c b/src/backend/distributed/commands/truncate.c index 0eb43f529..46cf5e602 100644 --- a/src/backend/distributed/commands/truncate.c +++ b/src/backend/distributed/commands/truncate.c @@ -135,7 +135,7 @@ TruncateTaskList(Oid relationId) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardRelationName = pstrdup(relationName); @@ -264,7 +264,7 @@ ErrorIfUnsupportedTruncateStmt(TruncateStmt *truncateStatement) { List *relationList = truncateStatement->relations; RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, relationList) + foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); @@ -294,7 +294,7 @@ static void EnsurePartitionTableNotReplicatedForTruncate(TruncateStmt *truncateStatement) { RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, truncateStatement->relations) + foreach_declared_ptr(rangeVar, truncateStatement->relations) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); @@ -322,7 +322,7 @@ ExecuteTruncateStmtSequentialIfNecessary(TruncateStmt *command) bool failOK = false; RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, relationList) + foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, failOK); diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index e1bbd5972..4148e442d 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -436,7 +436,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, bool analyze = false; DefElem *option = NULL; - foreach_ptr(option, explainStmt->options) + foreach_declared_ptr(option, explainStmt->options) { if (strcmp(option->defname, "analyze") == 0) { @@ -677,7 +677,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, { AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree; AlterTableCmd *command = NULL; - foreach_ptr(command, alterTableStmt->cmds) + foreach_declared_ptr(command, alterTableStmt->cmds) { AlterTableType alterTableType = command->subtype; @@ -876,7 +876,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } DDLJob *ddlJob = NULL; - foreach_ptr(ddlJob, ddlJobs) + foreach_declared_ptr(ddlJob, ddlJobs) { ExecuteDistributedDDLJob(ddlJob); } @@ -936,7 +936,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, { List *addresses = GetObjectAddressListFromParseTree(parsetree, false, true); ObjectAddress *address = NULL; - foreach_ptr(address, addresses) + foreach_declared_ptr(address, addresses) { MarkObjectDistributed(address); TrackPropagatedObject(address); @@ -959,7 +959,7 @@ UndistributeDisconnectedCitusLocalTables(void) citusLocalTableIdList = SortList(citusLocalTableIdList, CompareOids); Oid citusLocalTableId = InvalidOid; - foreach_oid(citusLocalTableId, citusLocalTableIdList) + foreach_declared_oid(citusLocalTableId, citusLocalTableIdList) { /* acquire ShareRowExclusiveLock to prevent concurrent foreign key creation */ LOCKMODE lockMode = ShareRowExclusiveLock; @@ -1341,7 +1341,7 @@ CurrentSearchPath(void) bool schemaAdded = false; Oid searchPathOid = InvalidOid; - foreach_oid(searchPathOid, searchPathList) + foreach_declared_oid(searchPathOid, searchPathList) { char *schemaName = get_namespace_name(searchPathOid); @@ -1475,7 +1475,7 @@ DDLTaskList(Oid relationId, const char *commandString) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; StringInfo applyCommand = makeStringInfo(); @@ -1529,7 +1529,7 @@ NodeDDLTaskList(TargetWorkerSet targets, List *commands) SetTaskQueryStringList(task, commands); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodes) + foreach_declared_ptr(workerNode, workerNodes) { ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); targetPlacement->nodeName = workerNode->workerName; diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index 6250d92b4..d2e339e58 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -135,7 +135,7 @@ VacuumRelationIdList(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumParams) List *relationIdList = NIL; RangeVar *vacuumRelation = NULL; - foreach_ptr(vacuumRelation, vacuumRelationList) + foreach_declared_ptr(vacuumRelation, vacuumRelationList) { /* * If skip_locked option is enabled, we are skipping that relation @@ -164,7 +164,7 @@ static bool IsDistributedVacuumStmt(List *vacuumRelationIdList) { Oid relationId = InvalidOid; - foreach_oid(relationId, vacuumRelationIdList) + foreach_declared_oid(relationId, vacuumRelationIdList) { if (OidIsValid(relationId) && IsCitusTable(relationId)) { @@ -188,7 +188,7 @@ ExecuteVacuumOnDistributedTables(VacuumStmt *vacuumStmt, List *relationIdList, int executedVacuumCount = 0; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (IsCitusTable(relationId)) { @@ -254,7 +254,7 @@ VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColum LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardRelationName = pstrdup(relationName); @@ -475,7 +475,7 @@ DeparseVacuumColumnNames(List *columnNameList) appendStringInfoString(columnNames, " ("); String *columnName = NULL; - foreach_ptr(columnName, columnNameList) + foreach_declared_ptr(columnName, columnNameList) { appendStringInfo(columnNames, "%s,", strVal(columnName)); } @@ -510,7 +510,7 @@ ExtractVacuumTargetRels(VacuumStmt *vacuumStmt) List *vacuumList = NIL; VacuumRelation *vacuumRelation = NULL; - foreach_ptr(vacuumRelation, vacuumStmt->rels) + foreach_declared_ptr(vacuumRelation, vacuumStmt->rels) { vacuumList = lappend(vacuumList, vacuumRelation->relation); } @@ -554,7 +554,7 @@ VacuumStmtParams(VacuumStmt *vacstmt) /* Parse options list */ DefElem *opt = NULL; - foreach_ptr(opt, vacstmt->options) + foreach_declared_ptr(opt, vacstmt->options) { /* Parse common options for VACUUM and ANALYZE */ if (strcmp(opt->defname, "verbose") == 0) @@ -727,7 +727,7 @@ ExecuteUnqualifiedVacuumTasks(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumPa int32 localNodeGroupId = GetLocalGroupId(); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodes) + foreach_declared_ptr(workerNode, workerNodes) { if (workerNode->groupId != localNodeGroupId) { diff --git a/src/backend/distributed/commands/view.c b/src/backend/distributed/commands/view.c index 0c39be4ca..aa9675a4d 100644 --- a/src/backend/distributed/commands/view.c +++ b/src/backend/distributed/commands/view.c @@ -69,7 +69,7 @@ ViewHasDistributedRelationDependency(ObjectAddress *viewObjectAddress) List *dependencies = GetAllDependenciesForObject(viewObjectAddress); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { if (dependency->classId == RelationRelationId && IsAnyObjectDistributed( list_make1(dependency))) @@ -304,7 +304,7 @@ DropViewStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess) List *objectAddresses = NIL; List *possiblyQualifiedViewName = NULL; - foreach_ptr(possiblyQualifiedViewName, dropStmt->objects) + foreach_declared_ptr(possiblyQualifiedViewName, dropStmt->objects) { RangeVar *viewRangeVar = makeRangeVarFromNameList(possiblyQualifiedViewName); Oid viewOid = RangeVarGetRelid(viewRangeVar, AccessShareLock, @@ -332,7 +332,7 @@ FilterNameListForDistributedViews(List *viewNamesList, bool missing_ok) List *distributedViewNames = NIL; List *possiblyQualifiedViewName = NULL; - foreach_ptr(possiblyQualifiedViewName, viewNamesList) + foreach_declared_ptr(possiblyQualifiedViewName, viewNamesList) { char *viewName = NULL; char *schemaName = NULL; diff --git a/src/backend/distributed/connection/connection_management.c b/src/backend/distributed/connection/connection_management.c index f8e4816ed..a8d8bad8a 100644 --- a/src/backend/distributed/connection/connection_management.c +++ b/src/backend/distributed/connection/connection_management.c @@ -879,7 +879,7 @@ WaitEventSetFromMultiConnectionStates(List *connections, int *waitCount) numEventsAdded += 2; MultiConnectionPollState *connectionState = NULL; - foreach_ptr(connectionState, connections) + foreach_declared_ptr(connectionState, connections) { if (numEventsAdded >= eventSetSize) { @@ -961,7 +961,7 @@ FinishConnectionListEstablishment(List *multiConnectionList) int waitCount = 0; MultiConnection *connection = NULL; - foreach_ptr(connection, multiConnectionList) + foreach_declared_ptr(connection, multiConnectionList) { MultiConnectionPollState *connectionState = palloc0(sizeof(MultiConnectionPollState)); @@ -1160,7 +1160,7 @@ static void CloseNotReadyMultiConnectionStates(List *connectionStates) { MultiConnectionPollState *connectionState = NULL; - foreach_ptr(connectionState, connectionStates) + foreach_declared_ptr(connectionState, connectionStates) { MultiConnection *connection = connectionState->connection; diff --git a/src/backend/distributed/connection/locally_reserved_shared_connections.c b/src/backend/distributed/connection/locally_reserved_shared_connections.c index 69b03c116..d63711d5c 100644 --- a/src/backend/distributed/connection/locally_reserved_shared_connections.c +++ b/src/backend/distributed/connection/locally_reserved_shared_connections.c @@ -360,7 +360,7 @@ EnsureConnectionPossibilityForNodeList(List *nodeList) nodeList = SortList(nodeList, CompareWorkerNodes); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, nodeList) + foreach_declared_ptr(workerNode, nodeList) { bool waitForConnection = true; EnsureConnectionPossibilityForNode(workerNode, waitForConnection); diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c index 10c99bd80..841deba08 100644 --- a/src/backend/distributed/connection/placement_connection.c +++ b/src/backend/distributed/connection/placement_connection.c @@ -370,7 +370,7 @@ AssignPlacementListToConnection(List *placementAccessList, MultiConnection *conn const char *userName = connection->user; ShardPlacementAccess *placementAccess = NULL; - foreach_ptr(placementAccess, placementAccessList) + foreach_declared_ptr(placementAccess, placementAccessList) { ShardPlacement *placement = placementAccess->placement; ShardPlacementAccessType accessType = placementAccess->accessType; @@ -533,7 +533,7 @@ FindPlacementListConnection(int flags, List *placementAccessList, const char *us * suitable connection found for a placement in the placementAccessList. */ ShardPlacementAccess *placementAccess = NULL; - foreach_ptr(placementAccess, placementAccessList) + foreach_declared_ptr(placementAccess, placementAccessList) { ShardPlacement *placement = placementAccess->placement; ShardPlacementAccessType accessType = placementAccess->accessType; diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index cbd74ff51..7a9e0601d 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -392,7 +392,7 @@ void ExecuteCriticalRemoteCommandList(MultiConnection *connection, List *commandList) { const char *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { ExecuteCriticalRemoteCommand(connection, command); } @@ -435,7 +435,7 @@ ExecuteRemoteCommandInConnectionList(List *nodeConnectionList, const char *comma { MultiConnection *connection = NULL; - foreach_ptr(connection, nodeConnectionList) + foreach_declared_ptr(connection, nodeConnectionList) { int querySent = SendRemoteCommand(connection, command); @@ -446,7 +446,7 @@ ExecuteRemoteCommandInConnectionList(List *nodeConnectionList, const char *comma } /* Process the result */ - foreach_ptr(connection, nodeConnectionList) + foreach_declared_ptr(connection, nodeConnectionList) { bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); @@ -887,7 +887,7 @@ WaitForAllConnections(List *connectionList, bool raiseInterrupts) /* convert connection list to an array such that we can move items around */ MultiConnection *connectionItem = NULL; - foreach_ptr(connectionItem, connectionList) + foreach_declared_ptr(connectionItem, connectionList) { allConnections[connectionIndex] = connectionItem; connectionReady[connectionIndex] = false; diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index f99462058..3b387799b 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -938,7 +938,7 @@ bool IsReindexWithParam_compat(ReindexStmt *reindexStmt, char *param) { DefElem *opt = NULL; - foreach_ptr(opt, reindexStmt->params) + foreach_declared_ptr(opt, reindexStmt->params) { if (strcmp(opt->defname, param) == 0) { @@ -963,7 +963,7 @@ AddVacuumParams(ReindexStmt *reindexStmt, StringInfo buffer) char *tableSpaceName = NULL; DefElem *opt = NULL; - foreach_ptr(opt, reindexStmt->params) + foreach_declared_ptr(opt, reindexStmt->params) { if (strcmp(opt->defname, "tablespace") == 0) { diff --git a/src/backend/distributed/deparser/deparse.c b/src/backend/distributed/deparser/deparse.c index 8312d6407..9963a84f2 100644 --- a/src/backend/distributed/deparser/deparse.c +++ b/src/backend/distributed/deparser/deparse.c @@ -47,7 +47,7 @@ DeparseTreeNodes(List *stmts) { List *sqls = NIL; Node *stmt = NULL; - foreach_ptr(stmt, stmts) + foreach_declared_ptr(stmt, stmts) { sqls = lappend(sqls, DeparseTreeNode(stmt)); } diff --git a/src/backend/distributed/deparser/deparse_domain_stmts.c b/src/backend/distributed/deparser/deparse_domain_stmts.c index e517074ec..9702eb310 100644 --- a/src/backend/distributed/deparser/deparse_domain_stmts.c +++ b/src/backend/distributed/deparser/deparse_domain_stmts.c @@ -70,7 +70,7 @@ DeparseCreateDomainStmt(Node *node) } Constraint *constraint = NULL; - foreach_ptr(constraint, stmt->constraints) + foreach_declared_ptr(constraint, stmt->constraints) { AppendConstraint(&buf, constraint, stmt->domainname, stmt->typeName); } @@ -117,7 +117,7 @@ DeparseDropDomainStmt(Node *node) TypeName *domainName = NULL; bool first = true; - foreach_ptr(domainName, stmt->objects) + foreach_declared_ptr(domainName, stmt->objects) { if (!first) { diff --git a/src/backend/distributed/deparser/deparse_extension_stmts.c b/src/backend/distributed/deparser/deparse_extension_stmts.c index 92d54602f..256d22214 100644 --- a/src/backend/distributed/deparser/deparse_extension_stmts.c +++ b/src/backend/distributed/deparser/deparse_extension_stmts.c @@ -40,7 +40,7 @@ DefElem * GetExtensionOption(List *extensionOptions, const char *defname) { DefElem *defElement = NULL; - foreach_ptr(defElement, extensionOptions) + foreach_declared_ptr(defElement, extensionOptions) { if (IsA(defElement, DefElem) && strncmp(defElement->defname, defname, NAMEDATALEN) == 0) @@ -112,7 +112,7 @@ AppendCreateExtensionStmtOptions(StringInfo buf, List *options) /* Add the options to the statement */ DefElem *defElem = NULL; - foreach_ptr(defElem, options) + foreach_declared_ptr(defElem, options) { if (strcmp(defElem->defname, "schema") == 0) { @@ -181,7 +181,7 @@ AppendAlterExtensionStmt(StringInfo buf, AlterExtensionStmt *alterExtensionStmt) * the options. */ DefElem *option = NULL; - foreach_ptr(option, optionsList) + foreach_declared_ptr(option, optionsList) { if (strcmp(option->defname, "new_version") == 0) { diff --git a/src/backend/distributed/deparser/deparse_foreign_server_stmts.c b/src/backend/distributed/deparser/deparse_foreign_server_stmts.c index 9c708a771..6b278f757 100644 --- a/src/backend/distributed/deparser/deparse_foreign_server_stmts.c +++ b/src/backend/distributed/deparser/deparse_foreign_server_stmts.c @@ -176,7 +176,7 @@ AppendAlterForeignServerOptions(StringInfo buf, AlterForeignServerStmt *stmt) DefElemAction action = DEFELEM_UNSPEC; DefElem *def = NULL; - foreach_ptr(def, stmt->options) + foreach_declared_ptr(def, stmt->options) { if (def->defaction != DEFELEM_UNSPEC) { @@ -242,7 +242,7 @@ static void AppendServerNames(StringInfo buf, DropStmt *stmt) { String *serverValue = NULL; - foreach_ptr(serverValue, stmt->objects) + foreach_declared_ptr(serverValue, stmt->objects) { const char *serverString = quote_identifier(strVal(serverValue)); appendStringInfo(buf, "%s", serverString); diff --git a/src/backend/distributed/deparser/deparse_publication_stmts.c b/src/backend/distributed/deparser/deparse_publication_stmts.c index 8e3118171..35068266e 100644 --- a/src/backend/distributed/deparser/deparse_publication_stmts.c +++ b/src/backend/distributed/deparser/deparse_publication_stmts.c @@ -118,7 +118,7 @@ AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, * Check whether there are objects to propagate, mainly to know whether * we should include "FOR". */ - foreach_ptr(publicationObject, stmt->pubobjects) + foreach_declared_ptr(publicationObject, stmt->pubobjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { @@ -156,7 +156,7 @@ AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, * Check whether there are tables to propagate, mainly to know whether * we should include "FOR". */ - foreach_ptr(rangeVar, stmt->tables) + foreach_declared_ptr(rangeVar, stmt->tables) { if (includeLocalTables || IsCitusTableRangeVar(rangeVar, NoLock, false)) { @@ -198,7 +198,7 @@ AppendPublicationObjects(StringInfo buf, List *publicationObjects, PublicationObjSpec *publicationObject = NULL; bool appendedObject = false; - foreach_ptr(publicationObject, publicationObjects) + foreach_declared_ptr(publicationObject, publicationObjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { @@ -334,7 +334,7 @@ AppendTables(StringInfo buf, List *tables, bool includeLocalTables) RangeVar *rangeVar = NULL; bool appendedObject = false; - foreach_ptr(rangeVar, tables) + foreach_declared_ptr(rangeVar, tables) { if (!includeLocalTables && !IsCitusTableRangeVar(rangeVar, NoLock, false)) diff --git a/src/backend/distributed/deparser/deparse_role_stmts.c b/src/backend/distributed/deparser/deparse_role_stmts.c index 0194d0d67..c4d6aa214 100644 --- a/src/backend/distributed/deparser/deparse_role_stmts.c +++ b/src/backend/distributed/deparser/deparse_role_stmts.c @@ -359,7 +359,7 @@ AppendRevokeAdminOptionFor(StringInfo buf, GrantRoleStmt *stmt) if (!stmt->is_grant) { DefElem *opt = NULL; - foreach_ptr(opt, stmt->opt) + foreach_declared_ptr(opt, stmt->opt) { if (strcmp(opt->defname, "admin") == 0) { @@ -384,7 +384,7 @@ AppendGrantWithAdminOption(StringInfo buf, GrantRoleStmt *stmt) { #if PG_VERSION_NUM >= PG_VERSION_16 DefElem *opt = NULL; - foreach_ptr(opt, stmt->opt) + foreach_declared_ptr(opt, stmt->opt) { bool admin_option = false; char *optval = defGetString(opt); diff --git a/src/backend/distributed/deparser/deparse_schema_stmts.c b/src/backend/distributed/deparser/deparse_schema_stmts.c index 0a9c49801..50e3974c0 100644 --- a/src/backend/distributed/deparser/deparse_schema_stmts.c +++ b/src/backend/distributed/deparser/deparse_schema_stmts.c @@ -152,7 +152,7 @@ AppendDropSchemaStmt(StringInfo buf, DropStmt *stmt) } String *schemaValue = NULL; - foreach_ptr(schemaValue, stmt->objects) + foreach_declared_ptr(schemaValue, stmt->objects) { const char *schemaString = quote_identifier(strVal(schemaValue)); appendStringInfo(buf, "%s", schemaString); diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 99b9d1c2d..4d7211939 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -216,7 +216,7 @@ AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) appendStringInfoString(buf, " ("); String *statType = NULL; - foreach_ptr(statType, stmt->stat_types) + foreach_declared_ptr(statType, stmt->stat_types) { appendStringInfoString(buf, strVal(statType)); @@ -235,7 +235,7 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { StatsElem *column = NULL; - foreach_ptr(column, stmt->exprs) + foreach_declared_ptr(column, stmt->exprs) { if (!column->name) { diff --git a/src/backend/distributed/deparser/deparse_text_search.c b/src/backend/distributed/deparser/deparse_text_search.c index e0c750d0d..30b6eb9e6 100644 --- a/src/backend/distributed/deparser/deparse_text_search.c +++ b/src/backend/distributed/deparser/deparse_text_search.c @@ -86,7 +86,7 @@ AppendDefElemList(StringInfo buf, List *defelems, char *objectName) { DefElem *defelem = NULL; bool first = true; - foreach_ptr(defelem, defelems) + foreach_declared_ptr(defelem, defelems) { if (!first) { @@ -133,7 +133,7 @@ DeparseDropTextSearchConfigurationStmt(Node *node) appendStringInfoString(&buf, "DROP TEXT SEARCH CONFIGURATION "); List *nameList = NIL; bool first = true; - foreach_ptr(nameList, stmt->objects) + foreach_declared_ptr(nameList, stmt->objects) { if (!first) { @@ -171,7 +171,7 @@ DeparseDropTextSearchDictionaryStmt(Node *node) appendStringInfoString(&buf, "DROP TEXT SEARCH DICTIONARY "); List *nameList = NIL; bool first = true; - foreach_ptr(nameList, stmt->objects) + foreach_declared_ptr(nameList, stmt->objects) { if (!first) { @@ -466,7 +466,7 @@ AppendStringInfoTokentypeList(StringInfo buf, List *tokentypes) { String *tokentype = NULL; bool first = true; - foreach_ptr(tokentype, tokentypes) + foreach_declared_ptr(tokentype, tokentypes) { if (nodeTag(tokentype) != T_String) { @@ -494,7 +494,7 @@ AppendStringInfoDictnames(StringInfo buf, List *dicts) { List *dictNames = NIL; bool first = true; - foreach_ptr(dictNames, dicts) + foreach_declared_ptr(dictNames, dicts) { if (!first) { diff --git a/src/backend/distributed/deparser/deparse_view_stmts.c b/src/backend/distributed/deparser/deparse_view_stmts.c index 5592aec9d..2e046c099 100644 --- a/src/backend/distributed/deparser/deparse_view_stmts.c +++ b/src/backend/distributed/deparser/deparse_view_stmts.c @@ -88,7 +88,7 @@ AppendViewNameList(StringInfo buf, List *viewNamesList) { bool isFirstView = true; List *qualifiedViewName = NULL; - foreach_ptr(qualifiedViewName, viewNamesList) + foreach_declared_ptr(qualifiedViewName, viewNamesList) { char *quotedQualifiedVieName = NameListToQuotedString(qualifiedViewName); if (!isFirstView) diff --git a/src/backend/distributed/deparser/qualify_collation_stmt.c b/src/backend/distributed/deparser/qualify_collation_stmt.c index dad3b7a0e..36bde6957 100644 --- a/src/backend/distributed/deparser/qualify_collation_stmt.c +++ b/src/backend/distributed/deparser/qualify_collation_stmt.c @@ -83,7 +83,7 @@ QualifyDropCollationStmt(Node *node) List *names = NIL; List *name = NIL; - foreach_ptr(name, stmt->objects) + foreach_declared_ptr(name, stmt->objects) { names = lappend(names, QualifyCollationName(name)); } diff --git a/src/backend/distributed/deparser/qualify_domain.c b/src/backend/distributed/deparser/qualify_domain.c index 2e163dad0..acf48e6ff 100644 --- a/src/backend/distributed/deparser/qualify_domain.c +++ b/src/backend/distributed/deparser/qualify_domain.c @@ -67,7 +67,7 @@ QualifyDropDomainStmt(Node *node) DropStmt *stmt = castNode(DropStmt, node); TypeName *domainName = NULL; - foreach_ptr(domainName, stmt->objects) + foreach_declared_ptr(domainName, stmt->objects) { QualifyTypeName(domainName, stmt->missing_ok); } @@ -249,7 +249,7 @@ QualifyCollate(CollateClause *collClause, bool missing_ok) collClause->collname = NIL; char *name = NULL; - foreach_ptr(name, objName) + foreach_declared_ptr(name, objName) { collClause->collname = lappend(collClause->collname, makeString(name)); } diff --git a/src/backend/distributed/deparser/qualify_publication_stmt.c b/src/backend/distributed/deparser/qualify_publication_stmt.c index 73ffe3a35..c47f52e15 100644 --- a/src/backend/distributed/deparser/qualify_publication_stmt.c +++ b/src/backend/distributed/deparser/qualify_publication_stmt.c @@ -55,7 +55,7 @@ QualifyPublicationObjects(List *publicationObjects) { PublicationObjSpec *publicationObject = NULL; - foreach_ptr(publicationObject, publicationObjects) + foreach_declared_ptr(publicationObject, publicationObjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { @@ -78,7 +78,7 @@ QualifyTables(List *tables) { RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, tables) + foreach_declared_ptr(rangeVar, tables) { QualifyPublicationRangeVar(rangeVar); } diff --git a/src/backend/distributed/deparser/qualify_sequence_stmt.c b/src/backend/distributed/deparser/qualify_sequence_stmt.c index 1a0ecc831..c56d0fda0 100644 --- a/src/backend/distributed/deparser/qualify_sequence_stmt.c +++ b/src/backend/distributed/deparser/qualify_sequence_stmt.c @@ -148,7 +148,7 @@ QualifyDropSequenceStmt(Node *node) List *objectNameListWithSchema = NIL; List *objectNameList = NULL; - foreach_ptr(objectNameList, stmt->objects) + foreach_declared_ptr(objectNameList, stmt->objects) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); @@ -192,7 +192,7 @@ QualifyGrantOnSequenceStmt(Node *node) } List *qualifiedSequenceRangeVars = NIL; RangeVar *sequenceRangeVar = NULL; - foreach_ptr(sequenceRangeVar, stmt->objects) + foreach_declared_ptr(sequenceRangeVar, stmt->objects) { if (sequenceRangeVar->schemaname == NULL) { diff --git a/src/backend/distributed/deparser/qualify_statistics_stmt.c b/src/backend/distributed/deparser/qualify_statistics_stmt.c index ba8e8a764..7a99e6dfa 100644 --- a/src/backend/distributed/deparser/qualify_statistics_stmt.c +++ b/src/backend/distributed/deparser/qualify_statistics_stmt.c @@ -73,7 +73,7 @@ QualifyDropStatisticsStmt(Node *node) List *objectNameListWithSchema = NIL; List *objectNameList = NULL; - foreach_ptr(objectNameList, dropStatisticsStmt->objects) + foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { RangeVar *stat = makeRangeVarFromNameList(objectNameList); diff --git a/src/backend/distributed/deparser/qualify_text_search_stmts.c b/src/backend/distributed/deparser/qualify_text_search_stmts.c index df1e140a4..451cb2fb0 100644 --- a/src/backend/distributed/deparser/qualify_text_search_stmts.c +++ b/src/backend/distributed/deparser/qualify_text_search_stmts.c @@ -46,7 +46,7 @@ QualifyDropTextSearchConfigurationStmt(Node *node) List *qualifiedObjects = NIL; List *objName = NIL; - foreach_ptr(objName, stmt->objects) + foreach_declared_ptr(objName, stmt->objects) { char *schemaName = NULL; char *tsconfigName = NULL; @@ -87,7 +87,7 @@ QualifyDropTextSearchDictionaryStmt(Node *node) List *qualifiedObjects = NIL; List *objName = NIL; - foreach_ptr(objName, stmt->objects) + foreach_declared_ptr(objName, stmt->objects) { char *schemaName = NULL; char *tsdictName = NULL; @@ -141,7 +141,7 @@ QualifyAlterTextSearchConfigurationStmt(Node *node) bool useNewDicts = false; List *dicts = NULL; List *dictName = NIL; - foreach_ptr(dictName, stmt->dicts) + foreach_declared_ptr(dictName, stmt->dicts) { DeconstructQualifiedName(dictName, &schemaName, &objName); diff --git a/src/backend/distributed/deparser/qualify_view_stmt.c b/src/backend/distributed/deparser/qualify_view_stmt.c index af3fb280a..4f4daf71e 100644 --- a/src/backend/distributed/deparser/qualify_view_stmt.c +++ b/src/backend/distributed/deparser/qualify_view_stmt.c @@ -31,7 +31,7 @@ QualifyDropViewStmt(Node *node) List *qualifiedViewNames = NIL; List *possiblyQualifiedViewName = NULL; - foreach_ptr(possiblyQualifiedViewName, stmt->objects) + foreach_declared_ptr(possiblyQualifiedViewName, stmt->objects) { char *viewName = NULL; char *schemaName = NULL; diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c index b5566985a..f7e7c2337 100644 --- a/src/backend/distributed/executor/adaptive_executor.c +++ b/src/backend/distributed/executor/adaptive_executor.c @@ -1415,7 +1415,7 @@ AssignTasksToConnectionsOrWorkerPool(DistributedExecution *execution) List *taskList = execution->remoteTaskList; Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { bool placementExecutionReady = true; int placementExecutionIndex = 0; @@ -1438,7 +1438,7 @@ AssignTasksToConnectionsOrWorkerPool(DistributedExecution *execution) SetAttributeInputMetadata(execution, shardCommandExecution); ShardPlacement *taskPlacement = NULL; - foreach_ptr(taskPlacement, task->taskPlacementList) + foreach_declared_ptr(taskPlacement, task->taskPlacementList) { int connectionFlags = 0; char *nodeName = NULL; @@ -1583,7 +1583,7 @@ AssignTasksToConnectionsOrWorkerPool(DistributedExecution *execution) * connection may be be returned multiple times by GetPlacementListConnectionIfCached. */ WorkerSession *session = NULL; - foreach_ptr(session, execution->sessionList) + foreach_declared_ptr(session, execution->sessionList) { MultiConnection *connection = session->connection; @@ -1706,7 +1706,7 @@ static WorkerPool * FindOrCreateWorkerPool(DistributedExecution *execution, char *nodeName, int nodePort) { WorkerPool *workerPool = NULL; - foreach_ptr(workerPool, execution->workerList) + foreach_declared_ptr(workerPool, execution->workerList) { if (strncmp(nodeName, workerPool->nodeName, WORKER_LENGTH) == 0 && nodePort == workerPool->nodePort) @@ -1753,7 +1753,7 @@ FindOrCreateWorkerSession(WorkerPool *workerPool, MultiConnection *connection) static uint64 sessionId = 1; WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { if (session->connection == connection) { @@ -1856,7 +1856,7 @@ SequentialRunDistributedExecution(DistributedExecution *execution) */ MultiShardConnectionType = SEQUENTIAL_CONNECTION; Task *taskToExecute = NULL; - foreach_ptr(taskToExecute, taskList) + foreach_declared_ptr(taskToExecute, taskList) { execution->remoteAndLocalTaskList = list_make1(taskToExecute); execution->remoteTaskList = list_make1(taskToExecute); @@ -1896,7 +1896,7 @@ RunDistributedExecution(DistributedExecution *execution) { /* Preemptively step state machines in case of immediate errors */ WorkerSession *session = NULL; - foreach_ptr(session, execution->sessionList) + foreach_declared_ptr(session, execution->sessionList) { ConnectionStateMachine(session); } @@ -1928,7 +1928,7 @@ RunDistributedExecution(DistributedExecution *execution) HasIncompleteConnectionEstablishment(execution))) { WorkerPool *workerPool = NULL; - foreach_ptr(workerPool, execution->workerList) + foreach_declared_ptr(workerPool, execution->workerList) { ManageWorkerPool(workerPool); } @@ -2013,7 +2013,7 @@ ProcessSessionsWithFailedWaitEventSetOperations(DistributedExecution *execution) { bool foundFailedSession = false; WorkerSession *session = NULL; - foreach_ptr(session, execution->sessionList) + foreach_declared_ptr(session, execution->sessionList) { if (session->waitEventSetIndex == WAIT_EVENT_SET_INDEX_FAILED) { @@ -2057,7 +2057,7 @@ HasIncompleteConnectionEstablishment(DistributedExecution *execution) } WorkerSession *session = NULL; - foreach_ptr(session, execution->sessionList) + foreach_declared_ptr(session, execution->sessionList) { MultiConnection *connection = session->connection; if (connection->connectionState == MULTI_CONNECTION_INITIAL || @@ -2535,7 +2535,7 @@ AvgTaskExecutionTimeApproximation(WorkerPool *workerPool) INSTR_TIME_SET_CURRENT(now); WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { /* * Involve the tasks that are currently running. We do this to @@ -2573,7 +2573,7 @@ AvgConnectionEstablishmentTime(WorkerPool *workerPool) int sessionCount = 0; WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; @@ -2729,7 +2729,7 @@ OpenNewConnections(WorkerPool *workerPool, int newConnectionCount, #endif WorkerSession *session = NULL; - foreach_ptr(session, newSessionsList) + foreach_declared_ptr(session, newSessionsList) { /* immediately run the state machine to handle potential failure */ ConnectionStateMachine(session); @@ -2847,7 +2847,7 @@ static void MarkEstablishingSessionsTimedOut(WorkerPool *workerPool) { WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; @@ -2899,7 +2899,7 @@ NextEventTimeout(DistributedExecution *execution) long eventTimeout = 1000; /* milliseconds */ WorkerPool *workerPool = NULL; - foreach_ptr(workerPool, execution->workerList) + foreach_declared_ptr(workerPool, execution->workerList) { if (workerPool->failureState == WORKER_POOL_FAILED) { @@ -4240,7 +4240,7 @@ WorkerPoolFailed(WorkerPool *workerPool) } WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { WorkerSessionFailed(session); } @@ -4265,7 +4265,7 @@ WorkerPoolFailed(WorkerPool *workerPool) List *workerList = workerPool->distributedExecution->workerList; WorkerPool *pool = NULL; - foreach_ptr(pool, workerList) + foreach_declared_ptr(pool, workerList) { /* failed pools or pools without any connection attempts ignored */ if (pool->failureState == WORKER_POOL_FAILED || @@ -4618,7 +4618,7 @@ PlacementExecutionReady(TaskPlacementExecution *placementExecution) /* wake up an idle connection by checking whether the connection is writeable */ WorkerSession *session = NULL; - foreach_ptr(session, workerPool->sessionList) + foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; RemoteTransaction *transaction = &(connection->remoteTransaction); @@ -4743,7 +4743,7 @@ BuildWaitEventSet(List *sessionList) CreateWaitEventSet(CurrentMemoryContext, eventSetSize); WorkerSession *session = NULL; - foreach_ptr(session, sessionList) + foreach_declared_ptr(session, sessionList) { AddSessionToWaitEventSet(session, waitEventSet); } @@ -4841,7 +4841,7 @@ static void RebuildWaitEventSetFlags(WaitEventSet *waitEventSet, List *sessionList) { WorkerSession *session = NULL; - foreach_ptr(session, sessionList) + foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; int waitEventSetIndex = session->waitEventSetIndex; @@ -4897,7 +4897,7 @@ CleanUpSessions(DistributedExecution *execution) /* always trigger wait event set in the first round */ WorkerSession *session = NULL; - foreach_ptr(session, sessionList) + foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; @@ -4978,7 +4978,7 @@ static void UnclaimAllSessionConnections(List *sessionList) { WorkerSession *session = NULL; - foreach_ptr(session, sessionList) + foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; diff --git a/src/backend/distributed/executor/citus_custom_scan.c b/src/backend/distributed/executor/citus_custom_scan.c index 34a2f3d90..e072ba0fa 100644 --- a/src/backend/distributed/executor/citus_custom_scan.c +++ b/src/backend/distributed/executor/citus_custom_scan.c @@ -524,7 +524,7 @@ static bool AnchorShardsInTaskListExist(List *taskList) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { if (!ShardExists(task->anchorShardId)) { diff --git a/src/backend/distributed/executor/directed_acyclic_graph_execution.c b/src/backend/distributed/executor/directed_acyclic_graph_execution.c index 15b0272dd..48e8fbb64 100644 --- a/src/backend/distributed/executor/directed_acyclic_graph_execution.c +++ b/src/backend/distributed/executor/directed_acyclic_graph_execution.c @@ -94,7 +94,7 @@ FindExecutableTasks(List *allTasks, HTAB *completedTasks) List *curTasks = NIL; Task *task = NULL; - foreach_ptr(task, allTasks) + foreach_declared_ptr(task, allTasks) { if (IsAllDependencyCompleted(task, completedTasks) && !IsTaskAlreadyCompleted(task, completedTasks)) @@ -118,7 +118,7 @@ RemoveMergeTasks(List *taskList) List *prunedTaskList = NIL; Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { if (task->taskType != MERGE_TASK) { @@ -139,7 +139,7 @@ AddCompletedTasks(List *curCompletedTasks, HTAB *completedTasks) bool found; Task *task = NULL; - foreach_ptr(task, curCompletedTasks) + foreach_declared_ptr(task, curCompletedTasks) { TaskHashKey taskKey = { task->jobId, task->taskId }; hash_search(completedTasks, &taskKey, HASH_ENTER, &found); @@ -172,7 +172,7 @@ IsAllDependencyCompleted(Task *targetTask, HTAB *completedTasks) bool found = false; Task *task = NULL; - foreach_ptr(task, targetTask->dependentTaskList) + foreach_declared_ptr(task, targetTask->dependentTaskList) { TaskHashKey taskKey = { task->jobId, task->taskId }; diff --git a/src/backend/distributed/executor/distributed_execution_locks.c b/src/backend/distributed/executor/distributed_execution_locks.c index 4424accb7..9c9f09a4c 100644 --- a/src/backend/distributed/executor/distributed_execution_locks.c +++ b/src/backend/distributed/executor/distributed_execution_locks.c @@ -198,7 +198,7 @@ AcquireExecutorShardLocksForExecution(RowModifyLevel modLevel, List *taskList) List *requiresConsistentSnapshotRelationShardList = NIL; Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { ShardInterval *anchorShardInterval = LoadShardInterval(task->anchorShardId); anchorShardIntervalList = lappend(anchorShardIntervalList, anchorShardInterval); @@ -344,7 +344,7 @@ AcquireMetadataLocks(List *taskList) */ Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { LockShardDistributionMetadata(task->anchorShardId, ShareLock); } @@ -379,7 +379,7 @@ AcquireExecutorShardLocksForRelationRowLockList(List *relationRowLockList) * them. */ RelationRowLock *relationRowLock = NULL; - foreach_ptr(relationRowLock, relationRowLockList) + foreach_declared_ptr(relationRowLock, relationRowLockList) { LockClauseStrength rowLockStrength = relationRowLock->rowLockStrength; Oid relationId = relationRowLock->relationId; @@ -412,7 +412,7 @@ void LockPartitionsInRelationList(List *relationIdList, LOCKMODE lockmode) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (PartitionedTable(relationId)) { @@ -437,7 +437,7 @@ LockPartitionRelations(Oid relationId, LOCKMODE lockMode) */ List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { LockRelationOid(partitionRelationId, lockMode); } diff --git a/src/backend/distributed/executor/distributed_intermediate_results.c b/src/backend/distributed/executor/distributed_intermediate_results.c index c5ac27fb6..24e8ca8d8 100644 --- a/src/backend/distributed/executor/distributed_intermediate_results.c +++ b/src/backend/distributed/executor/distributed_intermediate_results.c @@ -206,7 +206,7 @@ WrapTasksForPartitioning(const char *resultIdPrefix, List *selectTaskList, intervalTypeMod); Task *selectTask = NULL; - foreach_ptr(selectTask, selectTaskList) + foreach_declared_ptr(selectTask, selectTaskList) { char *taskPrefix = SourceShardPrefix(resultIdPrefix, selectTask->anchorShardId); char *partitionMethodString = targetRelation->partitionMethod == 'h' ? @@ -490,7 +490,7 @@ ColocateFragmentsWithRelation(List *fragmentList, CitusTableCacheEntry *targetRe List **shardResultIdList = palloc0(shardCount * sizeof(List *)); DistributedResultFragment *sourceFragment = NULL; - foreach_ptr(sourceFragment, fragmentList) + foreach_declared_ptr(sourceFragment, fragmentList) { int shardIndex = sourceFragment->targetShardIndex; @@ -520,11 +520,11 @@ ColocationTransfers(List *fragmentList, CitusTableCacheEntry *targetRelation) HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); DistributedResultFragment *fragment = NULL; - foreach_ptr(fragment, fragmentList) + foreach_declared_ptr(fragment, fragmentList) { List *placementList = ActiveShardPlacementList(fragment->targetShardId); ShardPlacement *placement = NULL; - foreach_ptr(placement, placementList) + foreach_declared_ptr(placement, placementList) { NodePair transferKey = { .sourceNodeId = fragment->nodeId, @@ -576,7 +576,7 @@ FragmentTransferTaskList(List *fragmentListTransfers) List *fetchTaskList = NIL; NodeToNodeFragmentsTransfer *fragmentsTransfer = NULL; - foreach_ptr(fragmentsTransfer, fragmentListTransfers) + foreach_declared_ptr(fragmentsTransfer, fragmentListTransfers) { uint32 targetNodeId = fragmentsTransfer->nodes.targetNodeId; @@ -629,7 +629,7 @@ QueryStringForFragmentsTransfer(NodeToNodeFragmentsTransfer *fragmentsTransfer) appendStringInfoString(fragmentNamesArrayString, "ARRAY["); DistributedResultFragment *fragment = NULL; - foreach_ptr(fragment, fragmentsTransfer->fragmentList) + foreach_declared_ptr(fragment, fragmentsTransfer->fragmentList) { const char *fragmentName = fragment->resultId; diff --git a/src/backend/distributed/executor/executor_util_tasks.c b/src/backend/distributed/executor/executor_util_tasks.c index cef005042..22b328e21 100644 --- a/src/backend/distributed/executor/executor_util_tasks.c +++ b/src/backend/distributed/executor/executor_util_tasks.c @@ -163,7 +163,7 @@ bool TaskListCannotBeExecutedInTransaction(List *taskList) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { if (task->cannotBeExecutedInTransction) { @@ -190,7 +190,7 @@ SelectForUpdateOnReferenceTable(List *taskList) Task *task = (Task *) linitial(taskList); RelationRowLock *relationRowLock = NULL; - foreach_ptr(relationRowLock, task->relationRowLockList) + foreach_declared_ptr(relationRowLock, task->relationRowLockList) { Oid relationId = relationRowLock->relationId; @@ -239,7 +239,7 @@ bool ModifiedTableReplicated(List *taskList) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { int64 shardId = task->anchorShardId; diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index f5fbb3f78..582ec0d3c 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -244,7 +244,7 @@ NonPushableInsertSelectExecScan(CustomScanState *node) * on shards with connections. */ Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { uint64 shardId = task->anchorShardId; bool shardModified = false; @@ -381,7 +381,7 @@ BuildColumnNameListFromTargetList(Oid targetRelationId, List *insertTargetList) /* build the list of column names for the COPY statement */ TargetEntry *insertTargetEntry = NULL; - foreach_ptr(insertTargetEntry, insertTargetList) + foreach_declared_ptr(insertTargetEntry, insertTargetList) { columnNameList = lappend(columnNameList, insertTargetEntry->resname); } @@ -402,7 +402,7 @@ PartitionColumnIndexFromColumnList(Oid relationId, List *columnNameList) int partitionColumnIndex = 0; const char *columnName = NULL; - foreach_ptr(columnName, columnNameList) + foreach_declared_ptr(columnName, columnNameList) { AttrNumber attrNumber = get_attnum(relationId, columnName); @@ -428,7 +428,7 @@ DistributionColumnIndex(List *insertTargetList, Var *distributionColumn) { TargetEntry *insertTargetEntry = NULL; int targetEntryIndex = 0; - foreach_ptr(insertTargetEntry, insertTargetList) + foreach_declared_ptr(insertTargetEntry, insertTargetList) { if (insertTargetEntry->resno == distributionColumn->varattno) { @@ -452,7 +452,7 @@ WrapTaskListForProjection(List *taskList, List *projectedTargetEntries) StringInfo projectedColumnsString = makeStringInfo(); int entryIndex = 0; TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, projectedTargetEntries) + foreach_declared_ptr(targetEntry, projectedTargetEntries) { if (entryIndex != 0) { @@ -467,7 +467,7 @@ WrapTaskListForProjection(List *taskList, List *projectedTargetEntries) } Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { StringInfo wrappedQuery = makeStringInfo(); appendStringInfo(wrappedQuery, "SELECT %s FROM (%s) subquery", diff --git a/src/backend/distributed/executor/intermediate_results.c b/src/backend/distributed/executor/intermediate_results.c index 0e18d4416..c1a96ef25 100644 --- a/src/backend/distributed/executor/intermediate_results.c +++ b/src/backend/distributed/executor/intermediate_results.c @@ -308,7 +308,7 @@ PrepareIntermediateResultBroadcast(RemoteFileDestReceiver *resultDest) } WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, initialNodeList) + foreach_declared_ptr(workerNode, initialNodeList) { int flags = 0; @@ -328,7 +328,7 @@ PrepareIntermediateResultBroadcast(RemoteFileDestReceiver *resultDest) RemoteTransactionsBeginIfNecessary(connectionList); MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { StringInfo copyCommand = ConstructCopyResultStatement(resultId); @@ -339,7 +339,7 @@ PrepareIntermediateResultBroadcast(RemoteFileDestReceiver *resultDest) } } - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; @@ -518,7 +518,7 @@ static void BroadcastCopyData(StringInfo dataBuffer, List *connectionList) { MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { SendCopyDataOverConnection(dataBuffer, connection); } @@ -714,7 +714,7 @@ void RemoveIntermediateResultsDirectories(void) { char *directoryElement = NULL; - foreach_ptr(directoryElement, CreatedResultsDirectories) + foreach_declared_ptr(directoryElement, CreatedResultsDirectories) { /* * The shared directory is renamed before deleting it. Otherwise it diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index 5af115357..65494bbc4 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -253,7 +253,7 @@ ExecuteLocalTaskListExtended(List *taskList, ALLOCSET_DEFAULT_SIZES); Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { MemoryContext oldContext = MemoryContextSwitchTo(loopContext); @@ -304,7 +304,7 @@ ExecuteLocalTaskListExtended(List *taskList, LOCKMODE lockMode = GetQueryLockMode(jobQuery); Oid relationId = InvalidOid; - foreach_oid(relationId, localPlan->relationOids) + foreach_declared_oid(relationId, localPlan->relationOids) { LockRelationOid(relationId, lockMode); } @@ -393,7 +393,7 @@ SetColocationIdAndPartitionKeyValueForTasks(List *taskList, Job *workerJob) if (workerJob->colocationId != INVALID_COLOCATION_ID) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { task->colocationId = workerJob->colocationId; task->partitionKeyValue = workerJob->partitionKeyValue; @@ -412,7 +412,7 @@ LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tuple { char *queryString = NULL; uint64 totalProcessedRows = 0; - foreach_ptr(queryString, queryStrings) + foreach_declared_ptr(queryString, queryStrings) { Query *shardQuery = ParseQueryString(queryString, NULL, @@ -490,7 +490,7 @@ ExecuteUtilityCommand(const char *taskQueryCommand) List *parseTreeList = pg_parse_query(taskQueryCommand); RawStmt *taskRawStmt = NULL; - foreach_ptr(taskRawStmt, parseTreeList) + foreach_declared_ptr(taskRawStmt, parseTreeList) { Node *taskRawParseTree = taskRawStmt->stmt; @@ -580,7 +580,7 @@ ExtractLocalAndRemoteTasks(bool readOnly, List *taskList, List **localTaskList, *localTaskList = NIL; Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { List *localTaskPlacementList = NULL; List *remoteTaskPlacementList = NULL; @@ -645,7 +645,7 @@ SplitLocalAndRemotePlacements(List *taskPlacementList, List **localTaskPlacement *remoteTaskPlacementList = NIL; ShardPlacement *taskPlacement = NULL; - foreach_ptr(taskPlacement, taskPlacementList) + foreach_declared_ptr(taskPlacement, taskPlacementList) { if (taskPlacement->groupId == localGroupId) { @@ -817,7 +817,7 @@ RecordNonDistTableAccessesForTask(Task *task) List *placementAccessList = PlacementAccessListForTask(task, taskPlacement); ShardPlacementAccess *placementAccess = NULL; - foreach_ptr(placementAccess, placementAccessList) + foreach_declared_ptr(placementAccess, placementAccessList) { uint64 placementAccessShardId = placementAccess->placement->shardId; if (placementAccessShardId == INVALID_SHARD_ID) @@ -968,7 +968,7 @@ AnyTaskAccessesLocalNode(List *taskList) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { if (TaskAccessesLocalNode(task)) { @@ -990,7 +990,7 @@ TaskAccessesLocalNode(Task *task) int32 localGroupId = GetLocalGroupId(); ShardPlacement *taskPlacement = NULL; - foreach_ptr(taskPlacement, task->taskPlacementList) + foreach_declared_ptr(taskPlacement, task->taskPlacementList) { if (taskPlacement->groupId == localGroupId) { diff --git a/src/backend/distributed/executor/merge_executor.c b/src/backend/distributed/executor/merge_executor.c index 969b03faf..ce1eb0073 100644 --- a/src/backend/distributed/executor/merge_executor.c +++ b/src/backend/distributed/executor/merge_executor.c @@ -258,7 +258,7 @@ ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) * on shards with connections. */ Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { uint64 shardId = task->anchorShardId; bool shardModified = false; diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 45a791af4..c8ebf90bd 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -224,7 +224,7 @@ CitusExecutorRun(QueryDesc *queryDesc, */ List *citusCustomScanStates = FindCitusCustomScanStates(queryDesc->planstate); CitusScanState *citusScanState = NULL; - foreach_ptr(citusScanState, citusCustomScanStates) + foreach_declared_ptr(citusScanState, citusCustomScanStates) { if (citusScanState->PreExecScan) { @@ -512,7 +512,7 @@ SortTupleStore(CitusScanState *scanState) * for sorting the tuples. */ TargetEntry *returningEntry = NULL; - foreach_ptr(returningEntry, targetList) + foreach_declared_ptr(returningEntry, targetList) { Oid sortop = InvalidOid; diff --git a/src/backend/distributed/executor/placement_access.c b/src/backend/distributed/executor/placement_access.c index a8573de7c..1046ae539 100644 --- a/src/backend/distributed/executor/placement_access.c +++ b/src/backend/distributed/executor/placement_access.c @@ -126,7 +126,7 @@ BuildPlacementAccessList(int32 groupId, List *relationShardList, List *placementAccessList = NIL; RelationShard *relationShard = NULL; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { ShardPlacement *placement = ActiveShardPlacementOnGroup(groupId, relationShard->shardId); diff --git a/src/backend/distributed/executor/repartition_join_execution.c b/src/backend/distributed/executor/repartition_join_execution.c index 8dce12390..d72e030b5 100644 --- a/src/backend/distributed/executor/repartition_join_execution.c +++ b/src/backend/distributed/executor/repartition_join_execution.c @@ -93,7 +93,7 @@ TraverseJobTree(Job *curJob, List **jobIds) *jobIds = lappend(*jobIds, jobIdPointer); Job *childJob = NULL; - foreach_ptr(childJob, curJob->dependentJobList) + foreach_declared_ptr(childJob, curJob->dependentJobList) { TraverseJobTree(childJob, jobIds); } diff --git a/src/backend/distributed/executor/subplan_execution.c b/src/backend/distributed/executor/subplan_execution.c index 4e81bb486..ef2838343 100644 --- a/src/backend/distributed/executor/subplan_execution.c +++ b/src/backend/distributed/executor/subplan_execution.c @@ -59,7 +59,7 @@ ExecuteSubPlans(DistributedPlan *distributedPlan) UseCoordinatedTransaction(); DistributedSubPlan *subPlan = NULL; - foreach_ptr(subPlan, subPlanList) + foreach_declared_ptr(subPlan, subPlanList) { PlannedStmt *plannedStmt = subPlan->plan; uint32 subPlanId = subPlan->subPlanId; diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index 6932c453c..683a24587 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -207,7 +207,7 @@ GetUniqueDependenciesList(List *objectAddressesList) InitObjectAddressCollector(&objectAddressCollector); ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, objectAddressesList) + foreach_declared_ptr(objectAddress, objectAddressesList) { if (IsObjectAddressCollected(*objectAddress, &objectAddressCollector)) { @@ -334,7 +334,7 @@ OrderObjectAddressListInDependencyOrder(List *objectAddressList) InitObjectAddressCollector(&collector); ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, objectAddressList) + foreach_declared_ptr(objectAddress, objectAddressList) { if (IsObjectAddressCollected(*objectAddress, &collector)) { @@ -403,7 +403,7 @@ RecurseObjectDependencies(ObjectAddress target, expandFn expand, followFn follow /* iterate all entries and recurse depth first */ DependencyDefinition *dependencyDefinition = NULL; - foreach_ptr(dependencyDefinition, dependenyDefinitionList) + foreach_declared_ptr(dependencyDefinition, dependenyDefinitionList) { if (follow == NULL || !follow(collector, dependencyDefinition)) { @@ -870,7 +870,7 @@ bool ErrorOrWarnIfAnyObjectHasUnsupportedDependency(List *objectAddresses) { ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, objectAddresses) + foreach_declared_ptr(objectAddress, objectAddresses) { if (ErrorOrWarnIfObjectHasUnsupportedDependency(objectAddress)) { @@ -963,7 +963,7 @@ DeferErrorIfAnyObjectHasUnsupportedDependency(const List *objectAddresses) { DeferredErrorMessage *deferredErrorMessage = NULL; ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, objectAddresses) + foreach_declared_ptr(objectAddress, objectAddresses) { deferredErrorMessage = DeferErrorIfHasUnsupportedDependency(objectAddress); if (deferredErrorMessage) @@ -996,7 +996,7 @@ GetUndistributableDependency(const ObjectAddress *objectAddress) return NULL; } - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { /* * Objects with the id smaller than FirstNormalObjectId should be created within @@ -1173,7 +1173,7 @@ IsAnyObjectAddressOwnedByExtension(const List *targets, ObjectAddress *extensionAddress) { ObjectAddress *target = NULL; - foreach_ptr(target, targets) + foreach_declared_ptr(target, targets) { if (IsObjectAddressOwnedByExtension(target, extensionAddress)) { @@ -1564,7 +1564,7 @@ ExpandCitusSupportedTypes(ObjectAddressCollector *collector, ObjectAddress targe List *FDWOids = GetDependentFDWsToExtension(extensionId); Oid FDWOid = InvalidOid; - foreach_oid(FDWOid, FDWOids) + foreach_declared_oid(FDWOid, FDWOids) { List *dependentRoleIds = GetDependentRoleIdsFDW(FDWOid); List *dependencies = @@ -1850,7 +1850,7 @@ GetViewRuleReferenceDependencyList(Oid viewId) List *nonInternalDependenciesOfDependingRules = NIL; HeapTuple depTup = NULL; - foreach_ptr(depTup, dependencyTupleList) + foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); @@ -1873,7 +1873,7 @@ GetViewRuleReferenceDependencyList(Oid viewId) List *ruleDependencies = DependencyDefinitionFromPgDepend(ruleAddress); DependencyDefinition *dependencyDef = NULL; - foreach_ptr(dependencyDef, ruleDependencies) + foreach_declared_ptr(dependencyDef, ruleDependencies) { /* * Follow all dependencies of the internally dependent rule dependencies @@ -1908,7 +1908,7 @@ GetRelationSequenceDependencyList(Oid relationId) List *seqIdList = NIL; SequenceInfo *seqInfo = NULL; - foreach_ptr(seqInfo, seqInfoList) + foreach_declared_ptr(seqInfo, seqInfoList) { seqIdList = lappend_oid(seqIdList, seqInfo->sequenceOid); } @@ -1981,7 +1981,7 @@ GetRelationTriggerFunctionDependencyList(Oid relationId) List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; - foreach_oid(triggerId, triggerIdList) + foreach_declared_oid(triggerId, triggerIdList) { Oid functionId = GetTriggerFunctionId(triggerId); DependencyDefinition *dependency = @@ -2006,7 +2006,7 @@ GetPublicationRelationsDependencyList(Oid publicationId) Oid relationId = InvalidOid; - foreach_oid(relationId, allRelationIds) + foreach_declared_oid(relationId, allRelationIds) { if (!IsCitusTable(relationId)) { @@ -2088,7 +2088,7 @@ CreateObjectAddressDependencyDefList(Oid classId, List *objectIdList) { List *dependencyList = NIL; Oid objectId = InvalidOid; - foreach_oid(objectId, objectIdList) + foreach_declared_oid(objectId, objectIdList) { DependencyDefinition *dependency = CreateObjectAddressDependencyDef(classId, objectId); @@ -2162,7 +2162,7 @@ BuildViewDependencyGraph(Oid relationId, HTAB *nodeMap) targetObjectId); HeapTuple depTup = NULL; - foreach_ptr(depTup, dependencyTupleList) + foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); @@ -2241,7 +2241,7 @@ GetDependingViews(Oid relationId) foreach_ptr_append(node, nodeQueue) { ViewDependencyNode *dependingNode = NULL; - foreach_ptr(dependingNode, node->dependingNodes) + foreach_declared_ptr(dependingNode, node->dependingNodes) { ObjectAddress relationAddress = { 0 }; ObjectAddressSet(relationAddress, RelationRelationId, dependingNode->id); diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 3f5824979..11f67ac87 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -433,7 +433,7 @@ bool IsAnyObjectDistributed(const List *addresses) { ObjectAddress *address = NULL; - foreach_ptr(address, addresses) + foreach_declared_ptr(address, addresses) { if (IsObjectDistributed(address)) { diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index b15586818..09560c0fb 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -919,7 +919,7 @@ CitusTableList(void) List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid relationId = InvalidOid; - foreach_oid(relationId, citusTableIdList) + foreach_declared_oid(relationId, citusTableIdList) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); @@ -1890,7 +1890,7 @@ BuildCachedShardList(CitusTableCacheEntry *cacheEntry) sizeof(int)); HeapTuple shardTuple = NULL; - foreach_ptr(shardTuple, distShardTupleList) + foreach_declared_ptr(shardTuple, distShardTupleList) { ShardInterval *shardInterval = TupleToShardInterval(shardTuple, distShardTupleDesc, @@ -2028,7 +2028,7 @@ BuildCachedShardList(CitusTableCacheEntry *cacheEntry) GroupShardPlacement *placementArray = palloc0(numberOfPlacements * sizeof(GroupShardPlacement)); GroupShardPlacement *srcPlacement = NULL; - foreach_ptr(srcPlacement, placementList) + foreach_declared_ptr(srcPlacement, placementList) { placementArray[placementOffset] = *srcPlacement; placementOffset++; @@ -4332,7 +4332,7 @@ InitializeWorkerNodeCache(void) /* iterate over the worker node list */ WorkerNode *currentNode = NULL; - foreach_ptr(currentNode, workerNodeList) + foreach_declared_ptr(currentNode, workerNodeList) { bool handleFound = false; @@ -4509,7 +4509,7 @@ GetLocalNodeId(void) List *workerNodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if (workerNode->groupId == localGroupId && workerNode->isActive) @@ -5097,7 +5097,7 @@ CitusTableCacheFlushInvalidatedEntries() if (DistTableCacheHash != NULL && DistTableCacheExpired != NIL) { CitusTableCacheEntry *cacheEntry = NULL; - foreach_ptr(cacheEntry, DistTableCacheExpired) + foreach_declared_ptr(cacheEntry, DistTableCacheExpired) { ResetCitusTableCacheEntry(cacheEntry); } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 9407573b4..9abd60e5a 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -301,7 +301,7 @@ CreateDependingViewsOnWorkers(Oid relationId) SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); Oid viewOid = InvalidOid; - foreach_oid(viewOid, views) + foreach_declared_oid(viewOid, views) { if (!ShouldMarkRelationDistributed(viewOid)) { @@ -341,7 +341,7 @@ AddTableToPublications(Oid relationId) SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); - foreach_oid(publicationId, publicationIds) + foreach_declared_oid(publicationId, publicationIds) { ObjectAddress *publicationAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*publicationAddress, PublicationRelationId, publicationId); @@ -824,7 +824,7 @@ NodeListInsertCommand(List *workerNodeList) /* iterate over the worker nodes, add the values */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { char *hasMetadataString = workerNode->hasMetadata ? "TRUE" : "FALSE"; char *metadataSyncedString = workerNode->metadataSynced ? "TRUE" : "FALSE"; @@ -935,7 +935,7 @@ MarkObjectsDistributedCreateCommand(List *addresses, char *name = NULL; bool firstInNameLoop = true; - foreach_ptr(name, names) + foreach_declared_ptr(name, names) { if (!firstInNameLoop) { @@ -950,7 +950,7 @@ MarkObjectsDistributedCreateCommand(List *addresses, char *arg; bool firstInArgLoop = true; - foreach_ptr(arg, args) + foreach_declared_ptr(arg, args) { if (!firstInArgLoop) { @@ -1206,13 +1206,13 @@ ShardListInsertCommand(List *shardIntervalList) ShardInterval *shardInterval = NULL; bool firstPlacementProcessed = false; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; List *shardPlacementList = ActiveShardPlacementList(shardId); ShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { if (firstPlacementProcessed) { @@ -1246,7 +1246,7 @@ ShardListInsertCommand(List *shardIntervalList) "WITH shard_data(relationname, shardid, storagetype, " "shardminvalue, shardmaxvalue) AS (VALUES "); - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; Oid distributedRelationId = shardInterval->relationId; @@ -1683,7 +1683,7 @@ GetDependentRelationsWithSequence(Oid sequenceOid, char depType) Oid attrDefOid; List *attrDefOids = GetAttrDefsFromSequence(sequenceOid); - foreach_oid(attrDefOid, attrDefOids) + foreach_declared_oid(attrDefOid, attrDefOids) { ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); relations = lappend_oid(relations, columnAddress.objectId); @@ -1879,7 +1879,7 @@ GetDependentFunctionsWithRelation(Oid relationId) table_close(depRel, AccessShareLock); ObjectAddress *referencingObject = NULL; - foreach_ptr(referencingObject, referencingObjects) + foreach_declared_ptr(referencingObject, referencingObjects) { functionOids = list_concat(functionOids, GetFunctionDependenciesForObjects(referencingObject)); @@ -2669,7 +2669,7 @@ HasMetadataWorkers(void) List *workerNodeList = ActiveReadableNonCoordinatorNodeList(); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if (workerNode->hasMetadata) { @@ -2702,7 +2702,7 @@ CreateInterTableRelationshipOfRelationOnWorkers(Oid relationId) SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); const char *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } @@ -2755,14 +2755,14 @@ CreateShellTableOnWorkers(Oid relationId) creatingShellTableOnRemoteNode); TableDDLCommand *tableDDLCommand = NULL; - foreach_ptr(tableDDLCommand, tableDDLCommands) + foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); commandList = lappend(commandList, GetTableDDLCommand(tableDDLCommand)); } const char *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } @@ -2786,7 +2786,7 @@ CreateTableMetadataOnWorkers(Oid relationId) /* send the commands one by one */ const char *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } @@ -2810,7 +2810,7 @@ DetachPartitionCommandList(void) /* we iterate over all distributed partitioned tables and DETACH their partitions */ CitusTableCacheEntry *cacheEntry = NULL; - foreach_ptr(cacheEntry, distributedTableList) + foreach_declared_ptr(cacheEntry, distributedTableList) { if (!PartitionedTable(cacheEntry->relationId)) { @@ -2874,7 +2874,7 @@ SyncNodeMetadataToNodesOptional(void) List *syncedWorkerList = NIL; List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerList) + foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata && !workerNode->metadataSynced) { @@ -2894,7 +2894,7 @@ SyncNodeMetadataToNodesOptional(void) } } - foreach_ptr(workerNode, syncedWorkerList) + foreach_declared_ptr(workerNode, syncedWorkerList) { SetWorkerColumnOptional(workerNode, Anum_pg_dist_node_metadatasynced, BoolGetDatum(true)); @@ -2939,7 +2939,7 @@ SyncNodeMetadataToNodes(void) List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerList) + foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata) { @@ -3178,7 +3178,7 @@ ShouldInitiateMetadataSync(bool *lockFailure) List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerList) + foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata && !workerNode->metadataSynced) { @@ -3542,7 +3542,7 @@ EnsureShardMetadataIsSane(Oid relationId, int64 shardId, char storageType, GetFunctionInfo(intervalTypeId, BTREE_AM_OID, BTORDER_PROC); HeapTuple shardTuple = NULL; - foreach_ptr(shardTuple, distShardTupleList) + foreach_declared_ptr(shardTuple, distShardTupleList) { ShardInterval *shardInterval = TupleToShardInterval(shardTuple, distShardTupleDesc, @@ -3838,7 +3838,7 @@ citus_internal_delete_shard_metadata(PG_FUNCTION_ARGS) List *shardPlacementList = ShardPlacementList(shardId); ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, shardPlacementList) + foreach_declared_ptr(shardPlacement, shardPlacementList) { DeleteShardPlacementRow(shardPlacement->placementId); } @@ -4343,7 +4343,7 @@ SetMetadataSyncNodesFromNodeList(MetadataSyncContext *context, List *nodeList) List *activatedWorkerNodeList = NIL; WorkerNode *node = NULL; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { if (NodeIsPrimary(node)) { @@ -4378,7 +4378,7 @@ EstablishAndSetMetadataSyncBareConnections(MetadataSyncContext *context) /* establish bare connections to activated worker nodes */ List *bareConnectionList = NIL; WorkerNode *node = NULL; - foreach_ptr(node, context->activatedWorkerNodeList) + foreach_declared_ptr(node, context->activatedWorkerNodeList) { MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, node->workerName, @@ -4987,7 +4987,7 @@ SendDependencyCreationCommands(MetadataSyncContext *context) ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(commandsContext); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { if (!MetadataSyncCollectsCommands(context)) { diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index 792a4aeba..b020f4603 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -415,7 +415,7 @@ OpenConnectionToNodes(List *workerNodeList) { List *connectionList = NIL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -439,7 +439,7 @@ GenerateShardStatisticsQueryList(List *workerNodeList, List *citusTableIds) { List *shardStatisticsQueryList = NIL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { char *shardStatisticsQuery = GenerateAllShardStatisticsQueryForNode(workerNode, citusTableIds); @@ -460,7 +460,7 @@ ReceiveShardIdAndSizeResults(List *connectionList, Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; Datum values[SHARD_SIZES_COLUMN_COUNT]; @@ -551,7 +551,7 @@ DistributedTableSize(Oid relationId, SizeQueryType sizeQueryType, bool failOnErr List *workerNodeList = ActiveReadableNodeList(); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { uint64 relationSizeOnNode = 0; @@ -754,7 +754,7 @@ GenerateSizeQueryOnMultiplePlacements(List *shardIntervalList, List *nonPartitionedShardNames = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { if (optimizePartitionCalculations && PartitionTable(shardInterval->relationId)) { @@ -829,7 +829,7 @@ GenerateSizeQueryForRelationNameList(List *quotedShardNames, char *sizeFunction) bool addComma = false; char *quotedShardName = NULL; - foreach_ptr(quotedShardName, quotedShardNames) + foreach_declared_ptr(quotedShardName, quotedShardNames) { if (addComma) { @@ -930,7 +930,7 @@ GenerateAllShardStatisticsQueryForNode(WorkerNode *workerNode, List *citusTableI appendStringInfoString(allShardStatisticsQuery, " FROM (VALUES "); Oid relationId = InvalidOid; - foreach_oid(relationId, citusTableIds) + foreach_declared_oid(relationId, citusTableIds) { /* * Ensure the table still exists by trying to acquire a lock on it @@ -977,7 +977,7 @@ GenerateShardIdNameValuesForShardList(List *shardIntervalList, bool firstValue) StringInfo selectQuery = makeStringInfo(); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { if (!firstValue) { @@ -1096,7 +1096,7 @@ TableShardReplicationFactor(Oid relationId) List *shardIntervalList = LoadShardIntervalList(relationId); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; @@ -1187,7 +1187,7 @@ LoadUnsortedShardIntervalListViaCatalog(Oid relationId) &intervalTypeMod); HeapTuple distShardTuple = NULL; - foreach_ptr(distShardTuple, distShardTuples) + foreach_declared_ptr(distShardTuple, distShardTuples) { ShardInterval *interval = TupleToShardInterval(distShardTuple, distShardTupleDesc, @@ -1436,7 +1436,7 @@ FilterShardPlacementList(List *shardPlacementList, bool (*filter)(ShardPlacement List *filteredShardPlacementList = NIL; ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, shardPlacementList) + foreach_declared_ptr(shardPlacement, shardPlacementList) { if (filter(shardPlacement)) { @@ -1460,7 +1460,7 @@ FilterActiveShardPlacementListByNode(List *shardPlacementList, WorkerNode *worke List *filteredShardPlacementList = NIL; ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, activeShardPlacementList) + foreach_declared_ptr(shardPlacement, activeShardPlacementList) { if (IsPlacementOnWorkerNode(shardPlacement, workerNode)) { @@ -1484,7 +1484,7 @@ ActiveShardPlacementListOnGroup(uint64 shardId, int32 groupId) List *activePlacementList = ActiveShardPlacementList(shardId); ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, activePlacementList) + foreach_declared_ptr(shardPlacement, activePlacementList) { if (shardPlacement->groupId == groupId) { @@ -3280,7 +3280,7 @@ ResetRunningBackgroundTasks(void) /* there are tasks that need to release their lock before we can continue */ int64 *taskId = NULL; - foreach_ptr(taskId, taskIdsToWait) + foreach_declared_ptr(taskId, taskIdsToWait) { LOCKTAG locktag = { 0 }; SET_LOCKTAG_BACKGROUND_TASK(locktag, *taskId); diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c index bc6f89959..4e728f9ed 100644 --- a/src/backend/distributed/metadata/node_metadata.c +++ b/src/backend/distributed/metadata/node_metadata.c @@ -987,7 +987,7 @@ MarkNodesNotSyncedInLoopBackConnection(MetadataSyncContext *context, List *commandList = NIL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, context->activatedWorkerNodeList) + foreach_declared_ptr(workerNode, context->activatedWorkerNodeList) { /* * We need to prevent self deadlock when we access pg_dist_node using separate @@ -1020,7 +1020,7 @@ SetNodeMetadata(MetadataSyncContext *context, bool localOnly) List *updatedActivatedNodeList = NIL; WorkerNode *node = NULL; - foreach_ptr(node, context->activatedWorkerNodeList) + foreach_declared_ptr(node, context->activatedWorkerNodeList) { node = SetWorkerColumnLocalOnly(node, Anum_pg_dist_node_isactive, BoolGetDatum(true)); @@ -1039,7 +1039,7 @@ SetNodeMetadata(MetadataSyncContext *context, bool localOnly) if (!localOnly && EnableMetadataSync) { WorkerNode *node = NULL; - foreach_ptr(node, context->activatedWorkerNodeList) + foreach_declared_ptr(node, context->activatedWorkerNodeList) { SetNodeStateViaMetadataContext(context, node, BoolGetDatum(true)); } @@ -1835,7 +1835,7 @@ FindNodeAnyClusterByNodeId(uint32 nodeId) List *nodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *node = NULL; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { if (node->nodeId == nodeId) { @@ -1857,7 +1857,7 @@ FindNodeWithNodeId(int nodeId, bool missingOk) List *nodeList = ActiveReadableNodeList(); WorkerNode *node = NULL; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { if (node->nodeId == nodeId) { @@ -1885,7 +1885,7 @@ FindCoordinatorNodeId() List *nodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *node = NULL; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { if (NodeIsCoordinator(node)) { @@ -2015,7 +2015,7 @@ ErrorIfNodeContainsNonRemovablePlacements(WorkerNode *workerNode) shardPlacements = SortList(shardPlacements, CompareGroupShardPlacements); GroupShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacements) + foreach_declared_ptr(placement, shardPlacements) { if (!PlacementHasActivePlacementOnAnotherGroup(placement)) { @@ -2051,7 +2051,7 @@ PlacementHasActivePlacementOnAnotherGroup(GroupShardPlacement *sourcePlacement) bool foundActivePlacementOnAnotherGroup = false; ShardPlacement *activePlacement = NULL; - foreach_ptr(activePlacement, activePlacementList) + foreach_declared_ptr(activePlacement, activePlacementList) { if (activePlacement->groupId != sourcePlacement->groupId) { @@ -2402,7 +2402,7 @@ SetWorkerColumnOptional(WorkerNode *workerNode, int columnIndex, Datum value) /* open connections in parallel */ WorkerNode *worker = NULL; - foreach_ptr(worker, workerNodeList) + foreach_declared_ptr(worker, workerNodeList) { bool success = SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction( worker->workerName, worker->workerPort, @@ -3104,7 +3104,7 @@ static void ErrorIfAnyNodeNotExist(List *nodeList) { WorkerNode *node = NULL; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { /* * First, locally mark the node is active, if everything goes well, @@ -3153,7 +3153,7 @@ static void SendDeletionCommandsForReplicatedTablePlacements(MetadataSyncContext *context) { WorkerNode *node = NULL; - foreach_ptr(node, context->activatedWorkerNodeList) + foreach_declared_ptr(node, context->activatedWorkerNodeList) { if (!node->isActive) { diff --git a/src/backend/distributed/operations/citus_create_restore_point.c b/src/backend/distributed/operations/citus_create_restore_point.c index 8a5e738e4..18081b6e4 100644 --- a/src/backend/distributed/operations/citus_create_restore_point.c +++ b/src/backend/distributed/operations/citus_create_restore_point.c @@ -122,7 +122,7 @@ OpenConnectionsToAllWorkerNodes(LOCKMODE lockMode) List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(lockMode); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { MultiConnection *connection = StartNodeConnection(connectionFlags, workerNode->workerName, @@ -164,7 +164,7 @@ CreateRemoteRestorePoints(char *restoreName, List *connectionList) const char *parameterValues[1] = { restoreName }; MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommandParams(connection, CREATE_RESTORE_POINT_COMMAND, parameterCount, parameterTypes, @@ -175,7 +175,7 @@ CreateRemoteRestorePoints(char *restoreName, List *connectionList) } } - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { PGresult *result = GetRemoteCommandResult(connection, true); if (!IsResponseOK(result)) diff --git a/src/backend/distributed/operations/create_shards.c b/src/backend/distributed/operations/create_shards.c index b5372d4e6..e4798ea9c 100644 --- a/src/backend/distributed/operations/create_shards.c +++ b/src/backend/distributed/operations/create_shards.c @@ -207,7 +207,7 @@ CreateShardsWithRoundRobinPolicy(Oid distributedTableId, int32 shardCount, * each placement insertion. */ uint64 *shardIdPtr; - foreach_ptr(shardIdPtr, insertedShardIds) + foreach_declared_ptr(shardIdPtr, insertedShardIds) { List *placementsForShard = ShardPlacementList(*shardIdPtr); insertedShardPlacements = list_concat(insertedShardPlacements, @@ -265,7 +265,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool char targetShardStorageType = ShardStorageType(targetRelationId); ShardInterval *sourceShardInterval = NULL; - foreach_ptr(sourceShardInterval, sourceShardIntervalList) + foreach_declared_ptr(sourceShardInterval, sourceShardIntervalList) { uint64 sourceShardId = sourceShardInterval->shardId; uint64 *newShardIdPtr = (uint64 *) palloc0(sizeof(uint64)); @@ -293,7 +293,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool shardMinValueText, shardMaxValueText); ShardPlacement *sourcePlacement = NULL; - foreach_ptr(sourcePlacement, sourceShardPlacementList) + foreach_declared_ptr(sourcePlacement, sourceShardPlacementList) { int32 groupId = sourcePlacement->groupId; const uint64 shardSize = 0; @@ -311,7 +311,7 @@ CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool * each placement insertion. */ uint64 *shardIdPtr; - foreach_ptr(shardIdPtr, insertedShardIds) + foreach_declared_ptr(shardIdPtr, insertedShardIds) { List *placementsForShard = ShardPlacementList(*shardIdPtr); insertedShardPlacements = list_concat(insertedShardPlacements, diff --git a/src/backend/distributed/operations/delete_protocol.c b/src/backend/distributed/operations/delete_protocol.c index c36121b00..3a03e77e5 100644 --- a/src/backend/distributed/operations/delete_protocol.c +++ b/src/backend/distributed/operations/delete_protocol.c @@ -250,12 +250,12 @@ DropShards(Oid relationId, char *schemaName, char *relationName, bool shouldExecuteTasksLocally = ShouldExecuteTasksLocally(dropTaskList); Task *task = NULL; - foreach_ptr(task, dropTaskList) + foreach_declared_ptr(task, dropTaskList) { uint64 shardId = task->anchorShardId; ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, task->taskPlacementList) + foreach_declared_ptr(shardPlacement, task->taskPlacementList) { uint64 shardPlacementId = shardPlacement->placementId; int32 shardPlacementGroupId = shardPlacement->groupId; @@ -350,7 +350,7 @@ DropTaskList(Oid relationId, char *schemaName, char *relationName, int taskId = 1; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, deletableShardIntervalList) + foreach_declared_ptr(shardInterval, deletableShardIntervalList) { Assert(shardInterval->relationId == relationId); diff --git a/src/backend/distributed/operations/health_check.c b/src/backend/distributed/operations/health_check.c index c908606c1..e54d80b7e 100644 --- a/src/backend/distributed/operations/health_check.c +++ b/src/backend/distributed/operations/health_check.c @@ -119,11 +119,11 @@ StoreAllConnectivityChecks(Tuplestorestate *tupleStore, TupleDesc tupleDescripto /* * We iterate over the workerNodeList twice, for source and target worker nodes. This - * operation is safe for foreach_ptr macro, as long as we use different variables for + * operation is safe for foreach_declared_ptr macro, as long as we use different variables for * each iteration. */ WorkerNode *sourceWorkerNode = NULL; - foreach_ptr(sourceWorkerNode, workerNodeList) + foreach_declared_ptr(sourceWorkerNode, workerNodeList) { const char *sourceNodeName = sourceWorkerNode->workerName; const int sourceNodePort = sourceWorkerNode->workerPort; @@ -135,7 +135,7 @@ StoreAllConnectivityChecks(Tuplestorestate *tupleStore, TupleDesc tupleDescripto /* the second iteration over workerNodeList for the target worker nodes. */ WorkerNode *targetWorkerNode = NULL; - foreach_ptr(targetWorkerNode, workerNodeList) + foreach_declared_ptr(targetWorkerNode, workerNodeList) { const char *targetNodeName = targetWorkerNode->workerName; const int targetNodePort = targetWorkerNode->workerPort; diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index 52e44bea0..8a633e3dc 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -645,7 +645,7 @@ GetPreLoadTableCreationCommands(Oid relationId, if (tableACLList != NIL) { char *tableACLCommand = NULL; - foreach_ptr(tableACLCommand, tableACLList) + foreach_declared_ptr(tableACLCommand, tableACLList) { tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString(tableACLCommand)); @@ -822,7 +822,7 @@ GetTableRowLevelSecurityCommands(Oid relationId) List *rowLevelSecurityEnableCommands = pg_get_row_level_security_commands(relationId); char *rowLevelSecurityCommand = NULL; - foreach_ptr(rowLevelSecurityCommand, rowLevelSecurityEnableCommands) + foreach_declared_ptr(rowLevelSecurityCommand, rowLevelSecurityEnableCommands) { rowLevelSecurityCommandList = lappend( rowLevelSecurityCommandList, diff --git a/src/backend/distributed/operations/replicate_none_dist_table_shard.c b/src/backend/distributed/operations/replicate_none_dist_table_shard.c index 33a98ee42..aa48b488a 100644 --- a/src/backend/distributed/operations/replicate_none_dist_table_shard.c +++ b/src/backend/distributed/operations/replicate_none_dist_table_shard.c @@ -63,7 +63,7 @@ NoneDistTableReplicateCoordinatorPlacement(Oid noneDistTableId, /* insert new placements to pg_dist_placement */ List *insertedPlacementList = NIL; WorkerNode *targetNode = NULL; - foreach_ptr(targetNode, targetNodeList) + foreach_declared_ptr(targetNode, targetNodeList) { ShardPlacement *shardPlacement = InsertShardPlacementRowGlobally(shardId, GetNextPlacementId(), @@ -215,7 +215,7 @@ CreateForeignKeysFromReferenceTablesOnShards(Oid noneDistTableId) List *taskList = NIL; char *command = NULL; - foreach_ptr(command, ddlCommandList) + foreach_declared_ptr(command, ddlCommandList) { List *commandTaskList = InterShardDDLTaskList( ForeignConstraintGetReferencingTableId(command), diff --git a/src/backend/distributed/operations/shard_cleaner.c b/src/backend/distributed/operations/shard_cleaner.c index 790414530..0f57ecda2 100644 --- a/src/backend/distributed/operations/shard_cleaner.c +++ b/src/backend/distributed/operations/shard_cleaner.c @@ -264,7 +264,7 @@ DropOrphanedResourcesForCleanup() int failedResourceCountForCleanup = 0; CleanupRecord *record = NULL; - foreach_ptr(record, cleanupRecordList) + foreach_declared_ptr(record, cleanupRecordList) { if (!PrimaryNodeForGroup(record->nodeGroupId, NULL)) { @@ -374,7 +374,7 @@ FinalizeOperationNeedingCleanupOnSuccess(const char *operationName) int failedShardCountOnComplete = 0; CleanupRecord *record = NULL; - foreach_ptr(record, currentOperationRecordList) + foreach_declared_ptr(record, currentOperationRecordList) { if (record->policy == CLEANUP_ALWAYS) { diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index 8b58f5993..653c7ce23 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -357,7 +357,7 @@ CheckRebalanceStateInvariants(const RebalanceState *state) Assert(state != NULL); Assert(list_length(state->fillStateListAsc) == list_length(state->fillStateListDesc)); - foreach_ptr(fillState, state->fillStateListAsc) + foreach_declared_ptr(fillState, state->fillStateListAsc) { float4 totalCost = 0; ShardCost *shardCost = NULL; @@ -376,7 +376,7 @@ CheckRebalanceStateInvariants(const RebalanceState *state) fillState); - foreach_ptr(shardCost, fillState->shardCostListDesc) + foreach_declared_ptr(shardCost, fillState->shardCostListDesc) { if (prevShardCost != NULL) { @@ -520,7 +520,7 @@ GetRebalanceSteps(RebalanceOptions *options) List *activeWorkerList = SortedActiveWorkers(); int shardAllowedNodeCount = 0; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, activeWorkerList) + foreach_declared_ptr(workerNode, activeWorkerList) { if (workerNode->shouldHaveShards) { @@ -539,7 +539,7 @@ GetRebalanceSteps(RebalanceOptions *options) List *unbalancedShards = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, options->relationIdList) + foreach_declared_oid(relationId, options->relationIdList) { List *shardPlacementList = FullShardPlacementList(relationId, options->excludedShardArray); @@ -1335,7 +1335,7 @@ get_rebalance_progress(PG_FUNCTION_ARGS) &segmentList); ProgressMonitorData *monitor = NULL; - foreach_ptr(monitor, rebalanceMonitorList) + foreach_declared_ptr(monitor, rebalanceMonitorList) { PlacementUpdateEventProgress *placementUpdateEvents = ProgressMonitorSteps( monitor); @@ -1846,7 +1846,7 @@ NonColocatedDistRelationIdList(void) HTAB *alreadySelectedColocationIds = hash_create("RebalanceColocationIdSet", capacity, &info, flags); - foreach_oid(tableId, allCitusTablesList) + foreach_declared_oid(tableId, allCitusTablesList) { bool foundInSet = false; CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry( @@ -1912,7 +1912,7 @@ RebalanceTableShards(RebalanceOptions *options, Oid shardReplicationModeOid) * is required for logical replication to replicate UPDATE and DELETE commands. */ PlacementUpdateEvent *placementUpdate = NULL; - foreach_ptr(placementUpdate, placementUpdateList) + foreach_declared_ptr(placementUpdate, placementUpdateList) { Oid relationId = RelationIdForShard(placementUpdate->shardId); List *colocatedTableList = ColocatedTableList(relationId); @@ -1947,7 +1947,7 @@ static void ErrorOnConcurrentRebalance(RebalanceOptions *options) { Oid relationId = InvalidOid; - foreach_oid(relationId, options->relationIdList) + foreach_declared_oid(relationId, options->relationIdList) { /* this provides the legacy error when the lock can't be acquired */ AcquireRebalanceColocationLock(relationId, options->operationName); @@ -2038,7 +2038,7 @@ GenerateTaskMoveDependencyList(PlacementUpdateEvent *move, int64 colocationId, if (found) { int64 *taskId = NULL; - foreach_ptr(taskId, shardMoveSourceNodeHashEntry->taskIds) + foreach_declared_ptr(taskId, shardMoveSourceNodeHashEntry->taskIds) { hash_search(dependsList, taskId, HASH_ENTER, NULL); } @@ -2122,13 +2122,13 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo const char shardTransferMode = LookupShardTransferMode(shardReplicationModeOid); List *colocatedTableList = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, options->relationIdList) + foreach_declared_oid(relationId, options->relationIdList) { colocatedTableList = list_concat(colocatedTableList, ColocatedTableList(relationId)); } Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { EnsureTableOwner(colocatedTableId); } @@ -2150,7 +2150,7 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo * is required for logical replication to replicate UPDATE and DELETE commands. */ PlacementUpdateEvent *placementUpdate = NULL; - foreach_ptr(placementUpdate, placementUpdateList) + foreach_declared_ptr(placementUpdate, placementUpdateList) { relationId = RelationIdForShard(placementUpdate->shardId); List *colocatedTables = ColocatedTableList(relationId); @@ -2203,7 +2203,7 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo ShardMoveDependencies shardMoveDependencies = InitializeShardMoveDependencies(); - foreach_ptr(move, placementUpdateList) + foreach_declared_ptr(move, placementUpdateList) { resetStringInfo(&buf); @@ -2360,7 +2360,7 @@ ExecuteRebalancerCommandInSeparateTransaction(char *command) List *setCommands = GetSetCommandListForNewConnections(); char *setCommand = NULL; - foreach_ptr(setCommand, setCommands) + foreach_declared_ptr(setCommand, setCommands) { commandList = lappend(commandList, setCommand); } @@ -2428,14 +2428,14 @@ RebalancePlacementUpdates(List *workerNodeList, List *activeShardPlacementListLi List *shardPlacementList = NIL; List *placementUpdateList = NIL; - foreach_ptr(shardPlacementList, activeShardPlacementListList) + foreach_declared_ptr(shardPlacementList, activeShardPlacementListList) { state = InitRebalanceState(workerNodeList, shardPlacementList, functions); rebalanceStates = lappend(rebalanceStates, state); } - foreach_ptr(state, rebalanceStates) + foreach_declared_ptr(state, rebalanceStates) { state->placementUpdateList = placementUpdateList; MoveShardsAwayFromDisallowedNodes(state); @@ -2444,7 +2444,7 @@ RebalancePlacementUpdates(List *workerNodeList, List *activeShardPlacementListLi if (!drainOnly) { - foreach_ptr(state, rebalanceStates) + foreach_declared_ptr(state, rebalanceStates) { state->placementUpdateList = placementUpdateList; @@ -2476,13 +2476,13 @@ RebalancePlacementUpdates(List *workerNodeList, List *activeShardPlacementListLi } } - foreach_ptr(state, rebalanceStates) + foreach_declared_ptr(state, rebalanceStates) { hash_destroy(state->placementsHash); } int64 ignoredMoves = 0; - foreach_ptr(state, rebalanceStates) + foreach_declared_ptr(state, rebalanceStates) { ignoredMoves += state->ignoredMoves; } @@ -2537,7 +2537,7 @@ InitRebalanceState(List *workerNodeList, List *shardPlacementList, state->placementsHash = ShardPlacementsListToHash(shardPlacementList); /* create empty fill state for all of the worker nodes */ - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { NodeFillState *fillState = palloc0(sizeof(NodeFillState)); fillState->node = workerNode; @@ -2620,7 +2620,7 @@ FindFillStateForPlacement(RebalanceState *state, ShardPlacement *placement) NodeFillState *fillState = NULL; /* Find the correct fill state to add the placement to and do that */ - foreach_ptr(fillState, state->fillStateListAsc) + foreach_declared_ptr(fillState, state->fillStateListAsc) { if (IsPlacementOnWorkerNode(placement, fillState->node)) { @@ -2732,7 +2732,7 @@ MoveShardsAwayFromDisallowedNodes(RebalanceState *state) CompareDisallowedPlacementDesc); /* Move shards off of nodes they are not allowed on */ - foreach_ptr(disallowedPlacement, state->disallowedPlacementList) + foreach_declared_ptr(disallowedPlacement, state->disallowedPlacementList) { NodeFillState *targetFillState = FindAllowedTargetFillState( state, disallowedPlacement->shardCost->shardId); @@ -2787,7 +2787,7 @@ static NodeFillState * FindAllowedTargetFillState(RebalanceState *state, uint64 shardId) { NodeFillState *targetFillState = NULL; - foreach_ptr(targetFillState, state->fillStateListAsc) + foreach_declared_ptr(targetFillState, state->fillStateListAsc) { bool hasShard = PlacementsHashFind( state->placementsHash, @@ -2913,7 +2913,7 @@ FindAndMoveShardCost(float4 utilizationLowerBound, * find a source node for the move, starting at the node with the highest * utilization */ - foreach_ptr(sourceFillState, state->fillStateListDesc) + foreach_declared_ptr(sourceFillState, state->fillStateListDesc) { /* Don't move shards away from nodes that are already too empty, we're * done searching */ @@ -2924,7 +2924,7 @@ FindAndMoveShardCost(float4 utilizationLowerBound, /* find a target node for the move, starting at the node with the * lowest utilization */ - foreach_ptr(targetFillState, state->fillStateListAsc) + foreach_declared_ptr(targetFillState, state->fillStateListAsc) { ShardCost *shardCost = NULL; @@ -2947,7 +2947,7 @@ FindAndMoveShardCost(float4 utilizationLowerBound, /* find a shardcost that can be moved between between nodes that * makes the cost distribution more equal */ - foreach_ptr(shardCost, sourceFillState->shardCostListDesc) + foreach_declared_ptr(shardCost, sourceFillState->shardCostListDesc) { bool targetHasShard = PlacementsHashFind(state->placementsHash, shardCost->shardId, diff --git a/src/backend/distributed/operations/shard_split.c b/src/backend/distributed/operations/shard_split.c index cf9f301b7..265e1ba81 100644 --- a/src/backend/distributed/operations/shard_split.c +++ b/src/backend/distributed/operations/shard_split.c @@ -302,7 +302,7 @@ ErrorIfCannotSplitShardExtended(SplitOperation splitOperation, NullableDatum lastShardSplitPoint = { 0, true /*isnull*/ }; Datum shardSplitPoint; - foreach_int(shardSplitPoint, shardSplitPointsList) + foreach_declared_int(shardSplitPoint, shardSplitPointsList) { int32 shardSplitPointValue = DatumGetInt32(shardSplitPoint); @@ -399,7 +399,7 @@ GetWorkerNodesFromWorkerIds(List *nodeIdsForPlacementList) { List *workersForPlacementList = NIL; int32 nodeId; - foreach_int(nodeId, nodeIdsForPlacementList) + foreach_declared_int(nodeId, nodeIdsForPlacementList) { uint32 nodeIdValue = (uint32) nodeId; WorkerNode *workerNode = LookupNodeByNodeId(nodeIdValue); @@ -464,7 +464,7 @@ SplitShard(SplitMode splitMode, /* sort the tables to avoid deadlocks */ colocatedTableList = SortList(colocatedTableList, CompareOids); Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { /* * Block concurrent DDL / TRUNCATE commands on the relation. Similarly, @@ -694,7 +694,7 @@ CreateSplitShardsForShardGroup(List *shardGroupSplitIntervalListList, * Iterate over all the shards in the shard group. */ List *shardIntervalList = NIL; - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; @@ -778,7 +778,7 @@ CreateAuxiliaryStructuresForShardGroup(List *shardGroupSplitIntervalListList, /* * Iterate over all the shards in the shard group. */ - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; @@ -1029,7 +1029,7 @@ CreateSplitIntervalsForShardGroup(List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList = NIL; ShardInterval *shardToSplitInterval = NULL; - foreach_ptr(shardToSplitInterval, sourceColocatedShardIntervalList) + foreach_declared_ptr(shardToSplitInterval, sourceColocatedShardIntervalList) { List *shardSplitIntervalList = NIL; CreateSplitIntervalsForShard(shardToSplitInterval, splitPointsForShard, @@ -1121,7 +1121,7 @@ UpdateDistributionColumnsForShardGroup(List *colocatedShardList, uint32 colocationId) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, colocatedShardList) + foreach_declared_ptr(shardInterval, colocatedShardList) { Oid relationId = shardInterval->relationId; Var *distributionColumn = GetDistributionColumnFromMap(distributionColumnMap, @@ -1162,7 +1162,7 @@ InsertSplitChildrenShardMetadata(List *shardGroupSplitIntervalListList, /* * Iterate over all the shards in the shard group. */ - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { /* * Iterate on split shards list for a given shard and insert metadata. @@ -1195,7 +1195,7 @@ InsertSplitChildrenShardMetadata(List *shardGroupSplitIntervalListList, /* send commands to synced nodes one by one */ List *splitOffShardMetadataCommandList = ShardListInsertCommand(syncedShardList); char *command = NULL; - foreach_ptr(command, splitOffShardMetadataCommandList) + foreach_declared_ptr(command, splitOffShardMetadataCommandList) { SendCommandToWorkersWithMetadata(command); } @@ -1216,7 +1216,7 @@ CreatePartitioningHierarchyForBlockingSplit(List *shardGroupSplitIntervalListLis /* * Iterate over all the shards in the shard group. */ - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; @@ -1255,7 +1255,7 @@ CreateForeignKeyConstraints(List *shardGroupSplitIntervalListList, /* * Iterate over all the shards in the shard group. */ - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; @@ -1281,7 +1281,7 @@ CreateForeignKeyConstraints(List *shardGroupSplitIntervalListList, referenceTableForeignConstraintList); char *constraintCommand = NULL; - foreach_ptr(constraintCommand, constraintCommandList) + foreach_declared_ptr(constraintCommand, constraintCommandList) { SendCommandToWorker( workerPlacementNode->workerName, @@ -1685,7 +1685,7 @@ CreateDummyShardsForShardGroup(HTAB *mapOfPlacementToDummyShardList, } ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, sourceColocatedShardIntervalList) + foreach_declared_ptr(shardInterval, sourceColocatedShardIntervalList) { /* Populate list of commands necessary to create shard interval on destination */ List *splitShardCreationCommandList = GetPreLoadTableCreationCommands( @@ -1739,7 +1739,7 @@ CreateDummyShardsForShardGroup(HTAB *mapOfPlacementToDummyShardList, * If the target shard was created on source node as placement, skip it (See Note 2 from function description). */ List *shardIntervalList = NULL; - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; workerPlacementNode = NULL; @@ -1818,7 +1818,7 @@ CreateWorkerForPlacementSet(List *workersForPlacementList) hashFlags); WorkerNode *workerForPlacement = NULL; - foreach_ptr(workerForPlacement, workersForPlacementList) + foreach_declared_ptr(workerForPlacement, workersForPlacementList) { void *hashKey = (void *) workerForPlacement; hash_search(workerForPlacementSet, hashKey, HASH_ENTER, NULL); diff --git a/src/backend/distributed/operations/shard_transfer.c b/src/backend/distributed/operations/shard_transfer.c index 49d468cb1..8e3306a2d 100644 --- a/src/backend/distributed/operations/shard_transfer.c +++ b/src/backend/distributed/operations/shard_transfer.c @@ -492,7 +492,7 @@ TransferShards(int64 shardId, char *sourceNodeName, DropOrphanedResourcesInSeparateTransaction(); ShardInterval *colocatedShard = NULL; - foreach_ptr(colocatedShard, colocatedShardList) + foreach_declared_ptr(colocatedShard, colocatedShardList) { /* * This is to prevent any race condition possibility among the shard moves. @@ -519,7 +519,7 @@ TransferShards(int64 shardId, char *sourceNodeName, * metadata workers. */ colocatedShard = NULL; - foreach_ptr(colocatedShard, colocatedShardList) + foreach_declared_ptr(colocatedShard, colocatedShardList) { uint64 colocatedShardId = colocatedShard->shardId; uint32 groupId = GroupForNode(targetNodeName, targetNodePort); @@ -612,7 +612,7 @@ InsertCleanupRecordsForShardPlacementsOnNode(List *shardIntervalList, int32 groupId) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { /* get shard name */ char *qualifiedShardName = ConstructQualifiedShardName(shardInterval); @@ -649,7 +649,7 @@ IsShardListOnNode(List *colocatedShardList, char *targetNodeName, uint32 targetN * We exhaustively search all co-located shards */ ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, colocatedShardList) + foreach_declared_ptr(shardInterval, colocatedShardList) { uint64 shardId = shardInterval->shardId; List *placementList = ActiveShardPlacementListOnGroup(shardId, @@ -672,7 +672,7 @@ static void LockColocatedRelationsForMove(List *colocatedTableList) { Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { LockRelationOid(colocatedTableId, ShareUpdateExclusiveLock); } @@ -688,7 +688,7 @@ ErrorIfForeignTableForShardTransfer(List *colocatedTableList, ShardTransferType transferType) { Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { if (IsForeignTable(colocatedTableId)) { @@ -714,7 +714,7 @@ EnsureAllShardsCanBeCopied(List *colocatedShardList, char *targetNodeName, uint32 targetNodePort) { ShardInterval *colocatedShard = NULL; - foreach_ptr(colocatedShard, colocatedShardList) + foreach_declared_ptr(colocatedShard, colocatedShardList) { uint64 colocatedShardId = colocatedShard->shardId; @@ -1109,7 +1109,7 @@ void BlockWritesToShardList(List *shardList) { ShardInterval *shard = NULL; - foreach_ptr(shard, shardList) + foreach_declared_ptr(shard, shardList) { /* * We need to lock the referenced reference table metadata to avoid @@ -1280,7 +1280,7 @@ static void EnsureTableListOwner(List *tableIdList) { Oid tableId = InvalidOid; - foreach_oid(tableId, tableIdList) + foreach_declared_oid(tableId, tableIdList) { EnsureTableOwner(tableId); } @@ -1295,7 +1295,7 @@ static void ErrorIfReplicatingDistributedTableWithFKeys(List *tableIdList) { Oid tableId = InvalidOid; - foreach_oid(tableId, tableIdList) + foreach_declared_oid(tableId, tableIdList) { List *foreignConstraintCommandList = GetReferencingForeignConstaintCommands(tableId); @@ -1366,7 +1366,7 @@ CopyShardTablesViaLogicalReplication(List *shardIntervalList, char *sourceNodeNa * target node. We do not create the indexes yet. */ ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { Oid relationId = shardInterval->relationId; uint64 shardId = shardInterval->shardId; @@ -1433,7 +1433,7 @@ CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, /* iterate through the colocated shards and copy each */ ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { /* * For each shard we first create the shard table in a separate @@ -1475,7 +1475,7 @@ CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_CREATING_CONSTRAINTS); - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { List *ddlCommandList = PostLoadShardCreationCommandList(shardInterval, sourceNodeName, @@ -1492,7 +1492,7 @@ CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, * Create DDL commands to Attach child tables to their parents in a partitioning hierarchy. */ List *shardIntervalWithDDCommandsList = NIL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { if (PartitionTable(shardInterval->relationId)) { @@ -1517,7 +1517,7 @@ CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, * Iterate through the colocated shards and create DDL commamnds * to create the foreign constraints. */ - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { List *shardForeignConstraintCommandList = NIL; List *referenceTableForeignConstraintList = NIL; @@ -1536,7 +1536,7 @@ CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, /* Now execute the Partitioning & Foreign constraints creation commads. */ ShardCommandList *shardCommandList = NULL; - foreach_ptr(shardCommandList, shardIntervalWithDDCommandsList) + foreach_declared_ptr(shardCommandList, shardIntervalWithDDCommandsList) { char *tableOwner = TableOwner(shardCommandList->shardInterval->relationId); SendCommandListToWorkerOutsideTransaction(targetNodeName, targetNodePort, @@ -1566,7 +1566,7 @@ CopyShardsToNode(WorkerNode *sourceNode, WorkerNode *targetNode, List *shardInte int taskId = 0; List *copyTaskList = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { /* * Skip copying data for partitioned tables, because they contain no @@ -1699,7 +1699,7 @@ SearchShardPlacementInList(List *shardPlacementList, const char *nodeName, uint32 nodePort) { ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, shardPlacementList) + foreach_declared_ptr(shardPlacement, shardPlacementList) { if (strncmp(nodeName, shardPlacement->nodeName, MAX_NODE_LENGTH) == 0 && nodePort == shardPlacement->nodePort) @@ -1820,7 +1820,7 @@ CopyShardForeignConstraintCommandListGrouped(ShardInterval *shardInterval, *referenceTableForeignConstraintList = NIL; const char *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { char *escapedCommand = quote_literal_cstr(command); @@ -1989,7 +1989,7 @@ DropShardPlacementsFromMetadata(List *shardList, char *nodeName, int32 nodePort) { ShardInterval *shardInverval = NULL; - foreach_ptr(shardInverval, shardList) + foreach_declared_ptr(shardInverval, shardList) { uint64 shardId = shardInverval->shardId; List *shardPlacementList = ShardPlacementList(shardId); @@ -2053,7 +2053,7 @@ WorkerApplyShardDDLCommandList(List *ddlCommandList, int64 shardId) List *applyDDLCommandList = NIL; TableDDLCommand *ddlCommand = NULL; - foreach_ptr(ddlCommand, ddlCommandList) + foreach_declared_ptr(ddlCommand, ddlCommandList) { Assert(CitusIsA(ddlCommand, TableDDLCommand)); char *applyDDLCommand = GetShardedTableDDLCommand(ddlCommand, shardId, NULL); @@ -2087,7 +2087,7 @@ UpdatePlacementUpdateStatusForShardIntervalList(List *shardIntervalList, } ProgressMonitorData *monitor = NULL; - foreach_ptr(monitor, rebalanceMonitorList) + foreach_declared_ptr(monitor, rebalanceMonitorList) { PlacementUpdateEventProgress *steps = ProgressMonitorSteps(monitor); @@ -2098,7 +2098,7 @@ UpdatePlacementUpdateStatusForShardIntervalList(List *shardIntervalList, bool foundInList = false; ShardInterval *candidateShard = NULL; - foreach_ptr(candidateShard, shardIntervalList) + foreach_declared_ptr(candidateShard, shardIntervalList) { if (candidateShard->shardId == currentShardId) { diff --git a/src/backend/distributed/operations/stage_protocol.c b/src/backend/distributed/operations/stage_protocol.c index 5770d648e..9881d8775 100644 --- a/src/backend/distributed/operations/stage_protocol.c +++ b/src/backend/distributed/operations/stage_protocol.c @@ -431,7 +431,7 @@ CreateShardsOnWorkers(Oid distributedRelationId, List *shardPlacements, int poolSize = 1; ShardPlacement *shardPlacement = NULL; - foreach_ptr(shardPlacement, shardPlacements) + foreach_declared_ptr(shardPlacement, shardPlacements) { uint64 shardId = shardPlacement->shardId; ShardInterval *shardInterval = LoadShardInterval(shardId); @@ -516,7 +516,7 @@ RelationShardListForShardCreate(ShardInterval *shardInterval) /* all foregin key constraint relations */ Oid fkeyRelationid = InvalidOid; - foreach_oid(fkeyRelationid, allForeignKeyRelations) + foreach_declared_oid(fkeyRelationid, allForeignKeyRelations) { uint64 fkeyShardId = INVALID_SHARD_ID; @@ -590,7 +590,7 @@ WorkerCreateShardCommandList(Oid relationId, uint64 shardId, char *schemaName = get_namespace_name(schemaId); TableDDLCommand *ddlCommand = NULL; - foreach_ptr(ddlCommand, ddlCommandList) + foreach_declared_ptr(ddlCommand, ddlCommandList) { Assert(CitusIsA(ddlCommand, TableDDLCommand)); char *applyDDLCommand = GetShardedTableDDLCommand(ddlCommand, shardId, @@ -645,7 +645,7 @@ UpdateShardStatistics(int64 shardId) /* get shard's statistics from a shard placement */ ShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { statsOK = WorkerShardStats(placement, relationId, shardQualifiedName, &shardSize); @@ -713,7 +713,7 @@ ReceiveAndUpdateShardsSizes(List *connectionList) "oid visited hash set"); MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { if (PQstatus(connection->pgConn) != CONNECTION_OK) { @@ -809,7 +809,7 @@ UpdateShardSize(uint64 shardId, ShardInterval *shardInterval, Oid relationId, ShardPlacement *placement = NULL; /* update metadata for each shard placement */ - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { uint64 placementId = placement->placementId; int32 groupId = placement->groupId; diff --git a/src/backend/distributed/operations/worker_node_manager.c b/src/backend/distributed/operations/worker_node_manager.c index 192f6b466..66d3f59ab 100644 --- a/src/backend/distributed/operations/worker_node_manager.c +++ b/src/backend/distributed/operations/worker_node_manager.c @@ -421,7 +421,7 @@ GetFirstPrimaryWorkerNode(void) List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(RowShareLock); WorkerNode *firstWorkerNode = NULL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if (firstWorkerNode == NULL || CompareWorkerNodes(&workerNode, &firstWorkerNode) < 0) diff --git a/src/backend/distributed/operations/worker_split_copy_udf.c b/src/backend/distributed/operations/worker_split_copy_udf.c index 03354ea04..eb97dab1a 100644 --- a/src/backend/distributed/operations/worker_split_copy_udf.c +++ b/src/backend/distributed/operations/worker_split_copy_udf.c @@ -146,7 +146,7 @@ TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName, int index = 1; int splitWayCount = list_length(splitCopyInfoList); SplitCopyInfo *splitCopyInfo = NULL; - foreach_ptr(splitCopyInfo, splitCopyInfoList) + foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { char *shardNameCopy = pstrdup(sourceShardToCopyPrefix); AppendShardIdToName(&shardNameCopy, splitCopyInfo->destinationShardId); @@ -236,7 +236,7 @@ BuildMinMaxRangeArrays(List *splitCopyInfoList, ArrayType **minValueArray, SplitCopyInfo *splitCopyInfo = NULL; int index = 0; - foreach_ptr(splitCopyInfo, splitCopyInfoList) + foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { minValues[index] = splitCopyInfo->destinationShardMinHashValue; maxValues[index] = splitCopyInfo->destinationShardMaxHashValue; @@ -269,7 +269,7 @@ CreateShardCopyDestReceivers(EState *estate, ShardInterval *shardIntervalToSplit SplitCopyInfo *splitCopyInfo = NULL; int index = 0; char *sourceShardNamePrefix = get_rel_name(shardIntervalToSplitCopy->relationId); - foreach_ptr(splitCopyInfo, splitCopyInfoList) + foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { Oid destinationShardSchemaOid = get_rel_namespace( shardIntervalToSplitCopy->relationId); diff --git a/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c b/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c index d4775995c..c65893fbc 100644 --- a/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c +++ b/src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c @@ -298,7 +298,7 @@ PopulateShardSplitInfoInSM(ShardSplitInfoSMHeader *shardSplitInfoSMHeader, List *shardSplitInfoList = entry->shardSplitInfoList; ShardSplitInfo *splitShardInfo = NULL; - foreach_ptr(splitShardInfo, shardSplitInfoList) + foreach_declared_ptr(splitShardInfo, shardSplitInfoList) { shardSplitInfoSMHeader->splitInfoArray[splitInfoIndex] = *splitShardInfo; strcpy_s(shardSplitInfoSMHeader->splitInfoArray[splitInfoIndex].slotName, diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index e3aa7b3e6..f81ade91c 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -217,7 +217,7 @@ CitusCustomScanPathPlan(PlannerInfo *root, { TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, citusPath->remoteScan->custom_scan_tlist) + foreach_declared_ptr(targetEntry, citusPath->remoteScan->custom_scan_tlist) { /* we created this list, so we know it only contains Var */ Assert(IsA(targetEntry->expr, Var)); @@ -231,7 +231,7 @@ CitusCustomScanPathPlan(PlannerInfo *root, /* clauses might have been added by the planner, need to add them to our scan */ RestrictInfo *restrictInfo = NULL; List **quals = &citusPath->remoteScan->scan.plan.qual; - foreach_ptr(restrictInfo, clauses) + foreach_declared_ptr(restrictInfo, clauses) { *quals = lappend(*quals, restrictInfo->clause); } @@ -273,7 +273,7 @@ BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetLis /* extract column names from the remoteScanTargetList */ List *columnNameList = NIL; TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, remoteScanTargetList) + foreach_declared_ptr(targetEntry, remoteScanTargetList) { columnNameList = lappend(columnNameList, makeString(targetEntry->resname)); } diff --git a/src/backend/distributed/planner/deparse_shard_query.c b/src/backend/distributed/planner/deparse_shard_query.c index 43b5f1493..6b8ad3fde 100644 --- a/src/backend/distributed/planner/deparse_shard_query.c +++ b/src/backend/distributed/planner/deparse_shard_query.c @@ -67,7 +67,7 @@ RebuildQueryStrings(Job *workerJob) AddInsertAliasIfNeeded(originalQuery); } - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { Query *query = originalQuery; @@ -298,7 +298,7 @@ FindRelationShard(Oid inputRelationId, List *relationShardList) * some, otherwise this query wouldn't be eligible as a router query. * FIXME: We should probably use a hashtable here, to do efficient lookup. */ - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { if (inputRelationId == relationShard->relationId) { diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index f00fd4090..4e88155de 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -1545,7 +1545,7 @@ FinalizeRouterPlan(PlannedStmt *localPlan, CustomScan *customScan) /* extract the column names from the final targetlist*/ TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, customScan->scan.plan.targetlist) + foreach_declared_ptr(targetEntry, customScan->scan.plan.targetlist) { String *columnName = makeString(targetEntry->resname); columnNameList = lappend(columnNameList, columnName); @@ -1586,7 +1586,7 @@ makeCustomScanTargetlistFromExistingTargetList(List *existingTargetlist) /* build a targetlist to read from the custom scan output */ TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, existingTargetlist) + foreach_declared_ptr(targetEntry, existingTargetlist) { Assert(IsA(targetEntry, TargetEntry)); @@ -1636,7 +1636,7 @@ makeTargetListFromCustomScanList(List *custom_scan_tlist) List *targetList = NIL; TargetEntry *targetEntry = NULL; int resno = 1; - foreach_ptr(targetEntry, custom_scan_tlist) + foreach_declared_ptr(targetEntry, custom_scan_tlist) { /* * INDEX_VAR is used to reference back to the TargetEntry in custom_scan_tlist by @@ -2105,7 +2105,7 @@ TranslatedVars(PlannerInfo *root, int relationIndex) { /* postgres deletes translated_vars, hence we deep copy them here */ Node *targetNode = NULL; - foreach_ptr(targetNode, targetAppendRelInfo->translated_vars) + foreach_declared_ptr(targetNode, targetAppendRelInfo->translated_vars) { translatedVars = lappend(translatedVars, copyObject(targetNode)); @@ -2126,7 +2126,7 @@ FindTargetAppendRelInfo(PlannerInfo *root, int relationRteIndex) AppendRelInfo *appendRelInfo = NULL; /* iterate on the queries that are part of UNION ALL subselects */ - foreach_ptr(appendRelInfo, root->append_rel_list) + foreach_declared_ptr(appendRelInfo, root->append_rel_list) { /* * We're only interested in the child rel that is equal to the @@ -2449,7 +2449,7 @@ TranslatedVarsForRteIdentity(int rteIdentity) currentPlannerRestrictionContext->relationRestrictionContext-> relationRestrictionList; RelationRestriction *relationRestriction = NULL; - foreach_ptr(relationRestriction, relationRestrictionList) + foreach_declared_ptr(relationRestriction, relationRestrictionList) { if (GetRTEIdentity(relationRestriction->rte) == rteIdentity) { @@ -2619,7 +2619,7 @@ GetRTEListProperties(List *rangeTableList) RTEListProperties *rteListProperties = palloc0(sizeof(RTEListProperties)); RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (rangeTableEntry->rtekind != RTE_RELATION) { @@ -2712,7 +2712,7 @@ WarnIfListHasForeignDistributedTable(List *rangeTableList) static bool DistributedForeignTableWarningPrompted = false; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (DistributedForeignTableWarningPrompted) { diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index 60d6ce466..178ea235d 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -566,7 +566,7 @@ CreateCombineQueryForRouterPlan(DistributedPlan *distPlan) List *funcCollations = NIL; TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, dependentTargetList) + foreach_declared_ptr(targetEntry, dependentTargetList) { Node *expr = (Node *) targetEntry->expr; @@ -640,7 +640,7 @@ CreateTargetListForCombineQuery(List *targetList) /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; - foreach_ptr(originalTargetEntry, targetList) + foreach_declared_ptr(originalTargetEntry, targetList) { TargetEntry *newTargetEntry = flatCopyTargetEntry(originalTargetEntry); @@ -1571,7 +1571,7 @@ WrapSubquery(Query *subquery) /* create a target list that matches the SELECT */ TargetEntry *selectTargetEntry = NULL; - foreach_ptr(selectTargetEntry, subquery->targetList) + foreach_declared_ptr(selectTargetEntry, subquery->targetList) { /* exactly 1 entry in FROM */ int indexInRangeTable = 1; @@ -1723,7 +1723,7 @@ AddInsertSelectCasts(List *insertTargetList, List *selectTargetList, selectTargetList = list_concat(projectedEntries, nonProjectedEntries); int entryResNo = 1; TargetEntry *selectTargetEntry = NULL; - foreach_ptr(selectTargetEntry, selectTargetList) + foreach_declared_ptr(selectTargetEntry, selectTargetList) { selectTargetEntry->resno = entryResNo++; } diff --git a/src/backend/distributed/planner/intermediate_result_pruning.c b/src/backend/distributed/planner/intermediate_result_pruning.c index d4d9c0e21..fdf226374 100644 --- a/src/backend/distributed/planner/intermediate_result_pruning.c +++ b/src/backend/distributed/planner/intermediate_result_pruning.c @@ -277,7 +277,7 @@ AppendAllWorkerNodes(IntermediateResultsHashEntry *entry) List *workerNodeList = ActiveReadableNodeList(); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { entry->nodeIdList = list_append_unique_int(entry->nodeIdList, workerNode->nodeId); @@ -421,7 +421,7 @@ LogIntermediateResultMulticastSummary(IntermediateResultsHashEntry *entry, } WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { elog(logLevel, "Subplan %s will be sent to %s:%d", resultId, workerNode->workerName, workerNode->workerPort); diff --git a/src/backend/distributed/planner/local_distributed_join_planner.c b/src/backend/distributed/planner/local_distributed_join_planner.c index a6502bf43..2760377bb 100644 --- a/src/backend/distributed/planner/local_distributed_join_planner.c +++ b/src/backend/distributed/planner/local_distributed_join_planner.c @@ -328,7 +328,7 @@ static void ConvertRTEsToSubquery(List *rangeTableEntryDetailsList, RecursivePlanningContext *context) { RangeTableEntryDetails *rangeTableEntryDetails = NULL; - foreach_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) + foreach_declared_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) { RangeTblEntry *rangeTableEntry = rangeTableEntryDetails->rangeTableEntry; List *requiredAttributeNumbers = rangeTableEntryDetails->requiredAttributeNumbers; @@ -351,7 +351,7 @@ static bool AllRangeTableEntriesHaveUniqueIndex(List *rangeTableEntryDetailsList) { RangeTableEntryDetails *rangeTableEntryDetails = NULL; - foreach_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) + foreach_declared_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) { if (!rangeTableEntryDetails->hasConstantFilterOnUniqueColumn) { @@ -420,7 +420,7 @@ HasConstantFilterOnUniqueColumn(RangeTblEntry *rangeTableEntry, AppendUniqueIndexColumnsToList, INCLUDE_INDEX_ALL_STATEMENTS); IndexColumns *indexColumns = NULL; - foreach_ptr(indexColumns, uniqueIndexColumnsList) + foreach_declared_ptr(indexColumns, uniqueIndexColumnsList) { List *uniqueIndexColumnNos = indexColumns->indexColumnNos; if (FirstIsSuperSetOfSecond(rteEqualityColumnsNos, @@ -441,7 +441,7 @@ static bool FirstIsSuperSetOfSecond(List *firstIntList, List *secondIntList) { int curInt = 0; - foreach_int(curInt, secondIntList) + foreach_declared_int(curInt, secondIntList) { if (!list_member_int(firstIntList, curInt)) { @@ -526,7 +526,7 @@ RequiredAttrNumbersForRelationInternal(Query *queryToProcess, int rteIndex) List *requiredAttrNumbers = NIL; Var *var = NULL; - foreach_ptr(var, allVarsInQuery) + foreach_declared_ptr(var, allVarsInQuery) { if (var->varno == rteIndex) { @@ -554,7 +554,7 @@ CreateConversionCandidates(PlannerRestrictionContext *plannerRestrictionContext, RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { /* we're only interested in tables */ if (!IsRecursivelyPlannableRelation(rangeTableEntry)) diff --git a/src/backend/distributed/planner/local_plan_cache.c b/src/backend/distributed/planner/local_plan_cache.c index 2e5ca4e55..443297df0 100644 --- a/src/backend/distributed/planner/local_plan_cache.c +++ b/src/backend/distributed/planner/local_plan_cache.c @@ -244,7 +244,7 @@ GetCachedLocalPlan(Task *task, DistributedPlan *distributedPlan) int32 localGroupId = GetLocalGroupId(); - foreach_ptr(localPlannedStatement, cachedPlanList) + foreach_declared_ptr(localPlannedStatement, cachedPlanList) { if (localPlannedStatement->shardId == task->anchorShardId && localPlannedStatement->localGroupId == localGroupId) diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 1f9d17c43..66eaf71da 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -375,7 +375,7 @@ static void ErrorIfMergeHasUnsupportedTables(Oid targetRelationId, List *rangeTableList) { RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { Oid relationId = rangeTableEntry->relid; @@ -734,7 +734,7 @@ ErrorIfRepartitionMergeNotSupported(Oid targetRelationId, Query *mergeQuery, } MergeAction *action = NULL; - foreach_ptr(action, mergeQuery->mergeActionList) + foreach_declared_ptr(action, mergeQuery->mergeActionList) { if (FindNodeMatchingCheckFunction((Node *) action, IsNodeSubquery)) { @@ -763,7 +763,7 @@ ConvertCteRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte) * Presently, CTEs are only permitted within the USING clause, and thus, * we search for the corresponding one */ - foreach_ptr(candidateCte, mergeQuery->cteList) + foreach_declared_ptr(candidateCte, mergeQuery->cteList) { if (strcmp(candidateCte->ctename, sourceRte->ctename) == 0) { @@ -1018,7 +1018,7 @@ DeferErrorIfRoutableMergeNotSupported(Query *query, List *rangeTableList, List *localTablesList = NIL; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { Oid relationId = rangeTableEntry->relid; @@ -1224,7 +1224,7 @@ ErrorIfMergeQueryQualAndTargetListNotSupported(Oid targetRelationId, Query *orig * within itself. Check each INSERT/UPDATE/DELETE individually. */ MergeAction *action = NULL; - foreach_ptr(action, originalQuery->mergeActionList) + foreach_declared_ptr(action, originalQuery->mergeActionList) { Assert(originalQuery->returningList == NULL); deferredError = MergeQualAndTargetListFunctionsSupported(targetRelationId, @@ -1472,7 +1472,7 @@ FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) bool foundDistributionColumn = false; MergeAction *action = NULL; uint32 targetRangeTableIndex = query->resultRelation; - foreach_ptr(action, query->mergeActionList) + foreach_declared_ptr(action, query->mergeActionList) { /* Skip MATCHED clause as INSERTS are not allowed in it */ if (action->matched) @@ -1502,7 +1502,7 @@ FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) PartitionColumn(targetRelationId, targetRangeTableIndex); TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, action->targetList) + foreach_declared_ptr(targetEntry, action->targetList) { AttrNumber originalAttrNo = targetEntry->resno; diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 1d6a88934..86a9f2a64 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -495,7 +495,7 @@ ExplainJob(CitusScanState *scanState, Job *job, ExplainState *es, { Task *task = NULL; uint64 totalReceivedTupleDataForAllTasks = 0; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { totalReceivedTupleDataForAllTasks += TaskReceivedTupleData(task); } @@ -673,7 +673,7 @@ ExplainTaskList(CitusScanState *scanState, List *taskList, ExplainState *es, } Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { RemoteExplainPlan *remoteExplain = RemoteExplain(task, es, params); remoteExplainList = lappend(remoteExplainList, remoteExplain); @@ -1400,7 +1400,7 @@ void ResetExplainAnalyzeData(List *taskList) { Task *task = NULL; - foreach_ptr(task, taskList) + foreach_declared_ptr(task, taskList) { if (task->fetchedExplainAnalyzePlan != NULL) { @@ -1463,7 +1463,7 @@ ExplainAnalyzeTaskList(List *originalTaskList, List *explainAnalyzeTaskList = NIL; Task *originalTask = NULL; - foreach_ptr(originalTask, originalTaskList) + foreach_declared_ptr(originalTask, originalTaskList) { if (originalTask->queryCount != 1) { diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index 908ed206e..7f25e08cc 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -224,10 +224,10 @@ JoinOnColumns(List *currentPartitionColumnList, Var *candidateColumn, } Var *currentColumn = NULL; - foreach_ptr(currentColumn, currentPartitionColumnList) + foreach_declared_ptr(currentColumn, currentPartitionColumnList) { Node *joinClause = NULL; - foreach_ptr(joinClause, joinClauseList) + foreach_declared_ptr(joinClause, joinClauseList) { if (!NodeIsEqualsOpExpr(joinClause)) { @@ -1094,10 +1094,10 @@ SinglePartitionJoinClause(List *partitionColumnList, List *applicableJoinClauses } Var *partitionColumn = NULL; - foreach_ptr(partitionColumn, partitionColumnList) + foreach_declared_ptr(partitionColumn, partitionColumnList) { Node *applicableJoinClause = NULL; - foreach_ptr(applicableJoinClause, applicableJoinClauses) + foreach_declared_ptr(applicableJoinClause, applicableJoinClauses) { if (!NodeIsEqualsOpExpr(applicableJoinClause)) { @@ -1177,7 +1177,7 @@ OpExpr * DualPartitionJoinClause(List *applicableJoinClauses) { Node *applicableJoinClause = NULL; - foreach_ptr(applicableJoinClause, applicableJoinClauses) + foreach_declared_ptr(applicableJoinClause, applicableJoinClauses) { if (!NodeIsEqualsOpExpr(applicableJoinClause)) { @@ -1262,7 +1262,7 @@ IsApplicableJoinClause(List *leftTableIdList, uint32 rightTableId, Node *joinCla List *varList = pull_var_clause_default(joinClause); Var *var = NULL; bool joinContainsRightTable = false; - foreach_ptr(var, varList) + foreach_declared_ptr(var, varList) { uint32 columnTableId = var->varno; if (rightTableId == columnTableId) @@ -1301,7 +1301,7 @@ ApplicableJoinClauses(List *leftTableIdList, uint32 rightTableId, List *joinClau joinClauseList = JoinClauseList(joinClauseList); Node *joinClause = NULL; - foreach_ptr(joinClause, joinClauseList) + foreach_declared_ptr(joinClause, joinClauseList) { if (IsApplicableJoinClause(leftTableIdList, rightTableId, joinClause)) { diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 76e38237a..393a5a0d3 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -414,7 +414,7 @@ MultiLogicalPlanOptimize(MultiTreeRoot *multiLogicalPlan) /* pull up collect nodes and merge duplicate collects */ List *collectNodeList = FindNodesOfType(logicalPlanNode, T_MultiCollect); MultiCollect *collectNode = NULL; - foreach_ptr(collectNode, collectNodeList) + foreach_declared_ptr(collectNode, collectNodeList) { PullUpCollectLoop(collectNode); } @@ -436,7 +436,7 @@ MultiLogicalPlanOptimize(MultiTreeRoot *multiLogicalPlan) List *tableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); MultiTable *tableNode = NULL; - foreach_ptr(tableNode, tableNodeList) + foreach_declared_ptr(tableNode, tableNodeList) { if (tableNode->relationId == SUBQUERY_RELATION_ID) { @@ -542,7 +542,7 @@ OrSelectClauseList(List *selectClauseList) List *orSelectClauseList = NIL; Node *selectClause = NULL; - foreach_ptr(selectClause, selectClauseList) + foreach_declared_ptr(selectClause, selectClauseList) { bool orClause = is_orclause(selectClause); if (orClause) @@ -968,7 +968,7 @@ SelectClauseTableIdList(List *selectClauseList) List *tableIdList = NIL; Node *selectClause = NULL; - foreach_ptr(selectClause, selectClauseList) + foreach_declared_ptr(selectClause, selectClauseList) { List *selectColumnList = pull_var_clause_default(selectClause); @@ -1077,7 +1077,7 @@ TableIdListColumns(List *tableIdList, List *columnList) List *tableColumnList = NIL; Var *column = NULL; - foreach_ptr(column, columnList) + foreach_declared_ptr(column, columnList) { int columnTableId = (int) column->varno; @@ -1103,7 +1103,7 @@ TableIdListSelectClauses(List *tableIdList, List *selectClauseList) List *tableSelectClauseList = NIL; Node *selectClause = NULL; - foreach_ptr(selectClause, selectClauseList) + foreach_declared_ptr(selectClause, selectClauseList) { List *selectColumnList = pull_var_clause_default(selectClause); if (list_length(selectColumnList) == 0) @@ -1425,7 +1425,7 @@ MasterExtendedOpNode(MultiExtendedOp *originalOpNode, /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; - foreach_ptr(originalTargetEntry, targetEntryList) + foreach_declared_ptr(originalTargetEntry, targetEntryList) { TargetEntry *newTargetEntry = flatCopyTargetEntry(originalTargetEntry); Expr *originalExpression = originalTargetEntry->expr; @@ -1598,7 +1598,7 @@ MasterAggregateExpression(Aggref *originalAggregate, Aggref *aggregate = (Aggref *) copyObject(originalAggregate); TargetEntry *targetEntry; - foreach_ptr(targetEntry, aggregate->args) + foreach_declared_ptr(targetEntry, aggregate->args) { targetEntry->expr = (Expr *) makeVar(masterTableId, walkerContext->columnId, @@ -1611,7 +1611,7 @@ MasterAggregateExpression(Aggref *originalAggregate, aggregate->aggdirectargs = NIL; Expr *directarg; - foreach_ptr(directarg, originalAggregate->aggdirectargs) + foreach_declared_ptr(directarg, originalAggregate->aggdirectargs) { /* * Need to replace nodes that contain any Vars with Vars referring @@ -1662,7 +1662,7 @@ MasterAggregateExpression(Aggref *originalAggregate, /* determine unique vars that were placed in target list by worker */ Var *column = NULL; - foreach_ptr(column, varList) + foreach_declared_ptr(column, varList) { uniqueVarList = list_append_unique(uniqueVarList, copyObject(column)); } @@ -1672,12 +1672,12 @@ MasterAggregateExpression(Aggref *originalAggregate, * worker query target entry column index. */ Var *columnToUpdate = NULL; - foreach_ptr(columnToUpdate, varList) + foreach_declared_ptr(columnToUpdate, varList) { int columnIndex = 0; Var *currentVar = NULL; - foreach_ptr(currentVar, uniqueVarList) + foreach_declared_ptr(currentVar, uniqueVarList) { if (equal(columnToUpdate, currentVar)) { @@ -2526,7 +2526,7 @@ ProcessTargetListForWorkerQuery(List *targetEntryList, /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; - foreach_ptr(originalTargetEntry, targetEntryList) + foreach_declared_ptr(originalTargetEntry, targetEntryList) { Expr *originalExpression = originalTargetEntry->expr; List *newExpressionList = NIL; @@ -2733,7 +2733,7 @@ ProcessWindowFunctionPullUpForWorkerQuery(List *windowClause, List *columnList = pull_var_clause_default((Node *) windowClause); Expr *newExpression = NULL; - foreach_ptr(newExpression, columnList) + foreach_declared_ptr(newExpression, columnList) { TargetEntry *newTargetEntry = makeNode(TargetEntry); @@ -2823,7 +2823,7 @@ bool TargetListHasAggregates(List *targetEntryList) { TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, targetEntryList) + foreach_declared_ptr(targetEntry, targetEntryList) { Expr *targetExpr = targetEntry->expr; bool hasAggregates = contain_aggs_of_level((Node *) targetExpr, 0); @@ -2867,7 +2867,7 @@ ExpandWorkerTargetEntry(List *expressionList, TargetEntry *originalTargetEntry, { /* now create target entries for each new expression */ Expr *newExpression = NULL; - foreach_ptr(newExpression, expressionList) + foreach_declared_ptr(newExpression, expressionList) { /* generate and add the new target entry to the target list */ TargetEntry *newTargetEntry = @@ -2904,7 +2904,7 @@ GetNextSortGroupRef(List *targetEntryList) /* find max of sort group ref index */ TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, targetEntryList) + foreach_declared_ptr(targetEntry, targetEntryList) { if (targetEntry->ressortgroupref > nextSortGroupRefIndex) { @@ -3060,13 +3060,13 @@ WorkerAggregateExpressionList(Aggref *originalAggregate, if (walkerContext->extendedOpNodeProperties->pullUpIntermediateRows) { TargetEntry *targetEntry; - foreach_ptr(targetEntry, originalAggregate->args) + foreach_declared_ptr(targetEntry, originalAggregate->args) { workerAggregateList = lappend(workerAggregateList, targetEntry->expr); } Expr *directarg; - foreach_ptr(directarg, originalAggregate->aggdirectargs) + foreach_declared_ptr(directarg, originalAggregate->aggdirectargs) { /* * The worker aggregation should execute any node that contains any @@ -3099,7 +3099,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate, List *columnList = pull_var_clause_default((Node *) aggregate); Var *column = NULL; - foreach_ptr(column, columnList) + foreach_declared_ptr(column, columnList) { workerAggregateList = list_append_unique(workerAggregateList, column); } @@ -3326,7 +3326,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate, rowExpr->colnames = NIL; TargetEntry *arg = NULL; - foreach_ptr(arg, originalAggregate->args) + foreach_declared_ptr(arg, originalAggregate->args) { rowExpr->args = lappend(rowExpr->args, copyObject(arg->expr)); } @@ -3830,7 +3830,7 @@ HasNonDistributableAggregates(MultiNode *logicalPlanNode) pull_var_clause(havingQual, PVC_INCLUDE_AGGREGATES)); Node *expression = NULL; - foreach_ptr(expression, expressionList) + foreach_declared_ptr(expression, expressionList) { /* only consider aggregate expressions */ if (!IsA(expression, Aggref)) @@ -3936,7 +3936,7 @@ DeferErrorIfHasNonDistributableAggregates(MultiNode *logicalPlanNode) pull_var_clause(havingQual, PVC_INCLUDE_AGGREGATES)); Node *expression = NULL; - foreach_ptr(expression, expressionList) + foreach_declared_ptr(expression, expressionList) { /* only consider aggregate expressions */ if (!IsA(expression, Aggref)) @@ -4079,7 +4079,7 @@ DeferErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, List *columnList = pull_var_clause_default(aggregateArgument); Var *column = NULL; - foreach_ptr(column, columnList) + foreach_declared_ptr(column, columnList) { if (column->varattno <= 0) { @@ -4095,7 +4095,7 @@ DeferErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, List *multiTableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); MultiTable *multiTable = NULL; - foreach_ptr(multiTable, multiTableNodeList) + foreach_declared_ptr(multiTable, multiTableNodeList) { if (multiTable->relationId == SUBQUERY_RELATION_ID || multiTable->relationId == SUBQUERY_PUSHDOWN_RELATION_ID) @@ -4251,7 +4251,7 @@ TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode, bool distinctSupported = true; MultiTable *tableNode = NULL; - foreach_ptr(tableNode, tableNodeList) + foreach_declared_ptr(tableNode, tableNodeList) { Oid relationId = tableNode->relationId; bool tableDistinctSupported = false; @@ -4327,7 +4327,7 @@ GroupedByColumn(List *groupClauseList, List *targetList, Var *column) } SortGroupClause *groupClause = NULL; - foreach_ptr(groupClause, groupClauseList) + foreach_declared_ptr(groupClause, groupClauseList) { TargetEntry *groupTargetEntry = get_sortgroupclause_tle(groupClause, targetList); @@ -4359,7 +4359,7 @@ SubqueryMultiTableList(MultiNode *multiNode) List *multiTableNodeList = FindNodesOfType(multiNode, T_MultiTable); MultiTable *multiTable = NULL; - foreach_ptr(multiTable, multiTableNodeList) + foreach_declared_ptr(multiTable, multiTableNodeList) { Query *subquery = multiTable->subquery; @@ -4383,7 +4383,7 @@ GroupTargetEntryList(List *groupClauseList, List *targetEntryList) List *groupTargetEntryList = NIL; SortGroupClause *groupClause = NULL; - foreach_ptr(groupClause, groupClauseList) + foreach_declared_ptr(groupClause, groupClauseList) { TargetEntry *groupTargetEntry = get_sortgroupclause_tle(groupClause, targetEntryList); @@ -4585,7 +4585,7 @@ FindReferencedTableColumn(Expr *columnExpression, List *parentQueryList, Query * } CommonTableExpr *candidateCte = NULL; - foreach_ptr(candidateCte, cteList) + foreach_declared_ptr(candidateCte, cteList) { if (strcmp(candidateCte->ctename, rangeTableEntry->ctename) == 0) { @@ -4878,7 +4878,7 @@ HasOrderByAggregate(List *sortClauseList, List *targetList) bool hasOrderByAggregate = false; SortGroupClause *sortClause = NULL; - foreach_ptr(sortClause, sortClauseList) + foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); @@ -4904,7 +4904,7 @@ HasOrderByNonCommutativeAggregate(List *sortClauseList, List *targetList) bool hasOrderByNonCommutativeAggregate = false; SortGroupClause *sortClause = NULL; - foreach_ptr(sortClause, sortClauseList) + foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); @@ -4944,7 +4944,7 @@ HasOrderByComplexExpression(List *sortClauseList, List *targetList) bool hasOrderByComplexExpression = false; SortGroupClause *sortClause = NULL; - foreach_ptr(sortClause, sortClauseList) + foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); @@ -4986,7 +4986,7 @@ HasOrderByHllType(List *sortClauseList, List *targetList) Oid hllTypeId = TypeOid(hllSchemaOid, HLL_TYPE_NAME); SortGroupClause *sortClause = NULL; - foreach_ptr(sortClause, sortClauseList) + foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); @@ -5070,12 +5070,12 @@ IsGroupBySubsetOfDistinct(List *groupClauses, List *distinctClauses) } SortGroupClause *groupClause = NULL; - foreach_ptr(groupClause, groupClauses) + foreach_declared_ptr(groupClause, groupClauses) { bool isFound = false; SortGroupClause *distinctClause = NULL; - foreach_ptr(distinctClause, distinctClauses) + foreach_declared_ptr(distinctClause, distinctClauses) { if (groupClause->tleSortGroupRef == distinctClause->tleSortGroupRef) { diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index f62e309f2..ebc71e2a6 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -1414,7 +1414,7 @@ IsJoinClause(Node *clause) } Var *initialVar = castNode(Var, linitial(varList)); - foreach_ptr(var, varList) + foreach_declared_ptr(var, varList) { if (var->varno != initialVar->varno) { diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index fb7f844c7..2fb5b26e3 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -895,7 +895,7 @@ WrapUngroupedVarsInAnyValueAggregate(Node *expression, List *groupClauseList, * subexpression equality check. */ TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, context.groupByTargetEntryList) + foreach_declared_ptr(targetEntry, context.groupByTargetEntryList) { if (!IsA(targetEntry->expr, Var)) { @@ -953,7 +953,7 @@ AddAnyValueAggregates(Node *node, AddAnyValueAggregatesContext *context) * Check whether this Var appears in the GROUP BY. */ TargetEntry *groupByTargetEntry = NULL; - foreach_ptr(groupByTargetEntry, context->groupByTargetEntryList) + foreach_declared_ptr(groupByTargetEntry, context->groupByTargetEntryList) { if (!IsA(groupByTargetEntry->expr, Var)) { @@ -996,7 +996,7 @@ AddAnyValueAggregates(Node *node, AddAnyValueAggregatesContext *context) * Otherwise, continue to descend into subexpressions. */ TargetEntry *groupByTargetEntry = NULL; - foreach_ptr(groupByTargetEntry, context->groupByTargetEntryList) + foreach_declared_ptr(groupByTargetEntry, context->groupByTargetEntryList) { if (equal(node, groupByTargetEntry->expr)) { @@ -1192,7 +1192,7 @@ QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableLis List *funcCollations = NIL; TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, dependentTargetList) + foreach_declared_ptr(targetEntry, dependentTargetList) { Node *expr = (Node *) targetEntry->expr; @@ -2237,7 +2237,7 @@ QueryPushdownSqlTaskList(Query *query, uint64 jobId, } ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, prunedShardList) + foreach_declared_ptr(shardInterval, prunedShardList) { int shardIndex = shardInterval->shardIndex; @@ -2305,7 +2305,7 @@ static bool IsInnerTableOfOuterJoin(RelationRestriction *relationRestriction) { RestrictInfo *joinInfo = NULL; - foreach_ptr(joinInfo, relationRestriction->relOptInfo->joininfo) + foreach_declared_ptr(joinInfo, relationRestriction->relOptInfo->joininfo) { if (joinInfo->outer_relids == NULL) { @@ -3473,7 +3473,7 @@ FetchEqualityAttrNumsForList(List *nodeList) List *attributeNums = NIL; Node *node = NULL; bool hasAtLeastOneEquality = false; - foreach_ptr(node, nodeList) + foreach_declared_ptr(node, nodeList) { List *fetchedEqualityAttrNums = FetchEqualityAttrNumsForRTE(node); @@ -3531,7 +3531,7 @@ FetchEqualityAttrNumsForRTEBoolExpr(BoolExpr *boolExpr) List *attributeNums = NIL; bool hasEquality = true; Node *arg = NULL; - foreach_ptr(arg, boolExpr->args) + foreach_declared_ptr(arg, boolExpr->args) { List *attributeNumsInSubExpression = FetchEqualityAttrNumsForRTE(arg); if (boolExpr->boolop == AND_EXPR) @@ -3622,7 +3622,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *dependen * tables and this new one. */ Node *nextJoinClause = NULL; - foreach_ptr(nextJoinClause, nextJoinClauseList) + foreach_declared_ptr(nextJoinClause, nextJoinClauseList) { if (!NodeIsEqualsOpExpr(nextJoinClause)) { @@ -4183,7 +4183,7 @@ FetchTaskResultNameList(List *mapOutputFetchTaskList) List *resultNameList = NIL; Task *mapOutputFetchTask = NULL; - foreach_ptr(mapOutputFetchTask, mapOutputFetchTaskList) + foreach_declared_ptr(mapOutputFetchTask, mapOutputFetchTaskList) { Task *mapTask = linitial(mapOutputFetchTask->dependentTaskList); int partitionId = mapOutputFetchTask->partitionId; @@ -4344,7 +4344,7 @@ PartitionColumnIndex(Var *targetVar, List *targetList) { TargetEntry *targetEntry = NULL; int resNo = 1; - foreach_ptr(targetEntry, targetList) + foreach_declared_ptr(targetEntry, targetList) { if (IsA(targetEntry->expr, Var)) { @@ -4571,7 +4571,7 @@ RowModifyLevelForQuery(Query *query) { /* skip checking for INSERT as those CTEs are recursively planned */ CommonTableExpr *cte = NULL; - foreach_ptr(cte, query->cteList) + foreach_declared_ptr(cte, query->cteList) { Query *cteQuery = (Query *) cte->ctequery; diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 1f2ad4751..bb1f64488 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -851,7 +851,7 @@ ModifiesLocalTableWithRemoteCitusLocalTable(List *rangeTableList) bool containsRemoteCitusLocalTable = false; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (!IsRecursivelyPlannableRelation(rangeTableEntry)) { @@ -982,7 +982,7 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer ContainsLocalTableDistributedTableJoin(queryTree->rtable); RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (rangeTableEntry->rtekind == RTE_RELATION) { @@ -1744,7 +1744,7 @@ RouterInsertTaskList(Query *query, bool parametersInQueryResolved, } ModifyRoute *modifyRoute = NULL; - foreach_ptr(modifyRoute, modifyRouteList) + foreach_declared_ptr(modifyRoute, modifyRouteList) { Task *modifyTask = CreateTask(MODIFY_TASK); modifyTask->anchorShardId = modifyRoute->shardId; @@ -2137,7 +2137,7 @@ SingleShardTaskList(Query *query, uint64 jobId, List *relationShardList, /* assume ErrorIfQueryHasUnroutableModifyingCTE checked query already */ CommonTableExpr *cte = NULL; - foreach_ptr(cte, query->cteList) + foreach_declared_ptr(cte, query->cteList) { Query *cteQuery = (Query *) cte->ctequery; @@ -2482,7 +2482,7 @@ AllShardsColocated(List *relationShardList) int colocationId = -1; CitusTableType tableType = ANY_CITUS_TABLE_TYPE; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { Oid relationId = relationShard->relationId; uint64 shardId = relationShard->shardId; @@ -2607,7 +2607,7 @@ CreateTaskPlacementListForShardIntervals(List *shardIntervalListList, bool shard * If there is a local table, we only allow the local placement to * be used. If there is none, we disallow the query. */ - foreach_ptr(taskPlacement, shardPlacementList) + foreach_declared_ptr(taskPlacement, shardPlacementList) { if (taskPlacement->groupId == GetLocalGroupId()) { @@ -3024,7 +3024,7 @@ PlacementsForWorkersContainingAllShards(List *shardIntervalListList) List *currentPlacementList = NIL; List *shardIntervalList = NIL; - foreach_ptr(shardIntervalList, shardIntervalListList) + foreach_declared_ptr(shardIntervalList, shardIntervalListList) { if (shardIntervalList == NIL) { @@ -3923,7 +3923,7 @@ ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree) char replicationModel = 0; CommonTableExpr *cte = NULL; - foreach_ptr(cte, queryTree->cteList) + foreach_declared_ptr(cte, queryTree->cteList) { Query *cteQuery = (Query *) cte->ctequery; diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 2eda4e42a..65de8680c 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -1187,7 +1187,7 @@ DeferErrorIfUnsupportedTableCombination(Query *queryTree) ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); - foreach_int(joinTreeTableIndex, joinTreeTableIndexList) + foreach_declared_int(joinTreeTableIndex, joinTreeTableIndexList) { /* * Join tree's range table index starts from 1 in the query tree. But, @@ -2010,7 +2010,7 @@ CreateSubqueryTargetListAndAdjustVars(List *columnList) Var *column = NULL; List *subqueryTargetEntryList = NIL; - foreach_ptr(column, columnList) + foreach_declared_ptr(column, columnList) { /* * To avoid adding the same column multiple times, we first check whether there @@ -2064,7 +2064,7 @@ static AttrNumber FindResnoForVarInTargetList(List *targetList, int varno, int varattno) { TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, targetList) + foreach_declared_ptr(targetEntry, targetList) { if (!IsA(targetEntry->expr, Var)) { @@ -2127,7 +2127,7 @@ PartitionColumnForPushedDownSubquery(Query *query) List *targetEntryList = query->targetList; TargetEntry *targetEntry = NULL; - foreach_ptr(targetEntry, targetEntryList) + foreach_declared_ptr(targetEntry, targetEntryList) { if (targetEntry->resjunk) { diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 9f520fa5f..9335b5ffc 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1736,7 +1736,7 @@ NodeContainsSubqueryReferencingOuterQuery(Node *node) ExtractSublinkWalker(node, &sublinks); SubLink *sublink; - foreach_ptr(sublink, sublinks) + foreach_declared_ptr(sublink, sublinks) { if (ContainsReferencesToOuterQuery(castNode(Query, sublink->subselect))) { @@ -1894,7 +1894,7 @@ GenerateRequiredColNamesFromTargetList(List *targetList) { TargetEntry *entry = NULL; List *innerSubqueryColNames = NIL; - foreach_ptr(entry, targetList) + foreach_declared_ptr(entry, targetList) { if (IsA(entry->expr, Var)) { @@ -1921,7 +1921,7 @@ UpdateVarNosInNode(Node *node, Index newVarNo) List *varList = pull_var_clause(node, PVC_RECURSE_AGGREGATES | PVC_RECURSE_PLACEHOLDERS); Var *var = NULL; - foreach_ptr(var, varList) + foreach_declared_ptr(var, varList) { var->varno = newVarNo; } @@ -1958,7 +1958,7 @@ ContainsLocalTableDistributedTableJoin(List *rangeTableList) bool containsDistributedTable = false; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, rangeTableList) + foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (FindNodeMatchingCheckFunctionInRangeTableList(list_make1(rangeTableEntry), IsDistributedOrReferenceTableRTE)) diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 83d7cbcdb..89516640a 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -1516,7 +1516,7 @@ ParentCountPriorToAppendRel(List *appendRelList, AppendRelInfo *targetAppendRelI int targetParentIndex = targetAppendRelInfo->parent_relid; Bitmapset *parent_ids = NULL; AppendRelInfo *appendRelInfo = NULL; - foreach_ptr(appendRelInfo, appendRelList) + foreach_declared_ptr(appendRelInfo, appendRelList) { int curParentIndex = appendRelInfo->parent_relid; if (curParentIndex <= targetParentIndex) @@ -1962,7 +1962,7 @@ AllDistributedRelationsInRestrictionContextColocated( List *relationIdList = NIL; /* check whether all relations exists in the main restriction list */ - foreach_ptr(relationRestriction, restrictionContext->relationRestrictionList) + foreach_declared_ptr(relationRestriction, restrictionContext->relationRestrictionList) { relationIdList = lappend_oid(relationIdList, relationRestriction->relationId); } @@ -1981,7 +1981,7 @@ AllDistributedRelationsInRTEListColocated(List *rangeTableEntryList) RangeTblEntry *rangeTableEntry = NULL; List *relationIdList = NIL; - foreach_ptr(rangeTableEntry, rangeTableEntryList) + foreach_declared_ptr(rangeTableEntry, rangeTableEntryList) { relationIdList = lappend_oid(relationIdList, rangeTableEntry->relid); } @@ -2000,7 +2000,7 @@ AllDistributedRelationsInListColocated(List *relationList) int initialColocationId = INVALID_COLOCATION_ID; Oid relationId = InvalidOid; - foreach_oid(relationId, relationList) + foreach_declared_oid(relationId, relationList) { if (!IsCitusTable(relationId)) { @@ -2155,7 +2155,7 @@ GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, List *restrictExprList = NIL; RestrictInfo *restrictInfo = NULL; - foreach_ptr(restrictInfo, baseRestrictInfo) + foreach_declared_ptr(restrictInfo, baseRestrictInfo) { Expr *restrictionClause = restrictInfo->clause; @@ -2199,7 +2199,7 @@ GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, Expr *copyOfRestrictClause = (Expr *) copyObject((Node *) restrictionClause); List *varClauses = pull_var_clause_default((Node *) copyOfRestrictClause); Var *column = NULL; - foreach_ptr(column, varClauses) + foreach_declared_ptr(column, varClauses) { column->varno = SINGLE_RTE_INDEX; column->varnosyn = SINGLE_RTE_INDEX; diff --git a/src/backend/distributed/planner/shard_pruning.c b/src/backend/distributed/planner/shard_pruning.c index e68ac72b0..2fd8ffdd6 100644 --- a/src/backend/distributed/planner/shard_pruning.c +++ b/src/backend/distributed/planner/shard_pruning.c @@ -1390,7 +1390,7 @@ DeepCopyShardIntervalList(List *originalShardIntervalList) List *copiedShardIntervalList = NIL; ShardInterval *originalShardInterval = NULL; - foreach_ptr(originalShardInterval, originalShardIntervalList) + foreach_declared_ptr(originalShardInterval, originalShardIntervalList) { ShardInterval *copiedShardInterval = CopyShardInterval(originalShardInterval); diff --git a/src/backend/distributed/progress/multi_progress.c b/src/backend/distributed/progress/multi_progress.c index 64e0a5b47..5d1550ddd 100644 --- a/src/backend/distributed/progress/multi_progress.c +++ b/src/backend/distributed/progress/multi_progress.c @@ -286,7 +286,7 @@ void DetachFromDSMSegments(List *dsmSegmentList) { dsm_segment *dsmSegment = NULL; - foreach_ptr(dsmSegment, dsmSegmentList) + foreach_declared_ptr(dsmSegment, dsmSegmentList) { dsm_detach(dsmSegment); } diff --git a/src/backend/distributed/relay/relay_event_utility.c b/src/backend/distributed/relay/relay_event_utility.c index d0267025b..630c783e5 100644 --- a/src/backend/distributed/relay/relay_event_utility.c +++ b/src/backend/distributed/relay/relay_event_utility.c @@ -150,7 +150,7 @@ RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId) AppendShardIdToName(relationName, shardId); AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { if (command->subtype == AT_AddConstraint) { @@ -162,7 +162,7 @@ RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId) { ColumnDef *columnDefinition = (ColumnDef *) command->def; Constraint *constraint = NULL; - foreach_ptr(constraint, columnDefinition->constraints) + foreach_declared_ptr(constraint, columnDefinition->constraints) { RelayEventExtendConstraintAndIndexNames(alterTableStmt, constraint, shardId); @@ -385,7 +385,7 @@ RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId) { List *shardStatisticsList = NIL; List *objectNameList = NULL; - foreach_ptr(objectNameList, dropStmt->objects) + foreach_declared_ptr(objectNameList, dropStmt->objects) { RangeVar *stat = makeRangeVarFromNameList(objectNameList); @@ -415,7 +415,7 @@ RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId) grantStmt->objtype == OBJECT_TABLE) { RangeVar *relation = NULL; - foreach_ptr(relation, grantStmt->objects) + foreach_declared_ptr(relation, grantStmt->objects) { char **relationName = &(relation->relname); char **relationSchemaName = &(relation->schemaname); @@ -673,7 +673,7 @@ RelayEventExtendNamesForInterShardCommands(Node *parseTree, uint64 leftShardId, List *commandList = alterTableStmt->cmds; AlterTableCmd *command = NULL; - foreach_ptr(command, commandList) + foreach_declared_ptr(command, commandList) { char **referencedTableName = NULL; char **relationSchemaName = NULL; @@ -693,7 +693,7 @@ RelayEventExtendNamesForInterShardCommands(Node *parseTree, uint64 leftShardId, List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; - foreach_ptr(constraint, columnConstraints) + foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { diff --git a/src/backend/distributed/replication/multi_logical_replication.c b/src/backend/distributed/replication/multi_logical_replication.c index ea61b77ca..cd34a787a 100644 --- a/src/backend/distributed/replication/multi_logical_replication.c +++ b/src/backend/distributed/replication/multi_logical_replication.c @@ -282,7 +282,7 @@ CreateGroupedLogicalRepTargetsHash(List *logicalRepTargetList) { HTAB *logicalRepTargetsHash = CreateSimpleHash(uint32, GroupedLogicalRepTargets); LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { bool found = false; GroupedLogicalRepTargets *groupedLogicalRepTargets = @@ -413,7 +413,7 @@ CreateShardMovePublicationInfoHash(WorkerNode *targetNode, List *shardIntervals) { HTAB *publicationInfoHash = CreateSimpleHash(NodeAndOwner, PublicationInfo); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervals) + foreach_declared_ptr(shardInterval, shardIntervals) { NodeAndOwner key; key.nodeId = targetNode->nodeId; @@ -474,7 +474,7 @@ CreateShardMoveLogicalRepTargetList(HTAB *publicationInfoHash, List *shardList) } ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { NodeAndOwner key; key.nodeId = nodeId; @@ -552,7 +552,7 @@ void CreateReplicaIdentities(List *logicalRepTargetList) { LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { MultiConnection *superuserConnection = target->superuserConnection; CreateReplicaIdentitiesOnNode( @@ -576,7 +576,7 @@ CreateReplicaIdentitiesOnNode(List *shardList, char *nodeName, int32 nodePort) MemoryContext oldContext = MemoryContextSwitchTo(localContext); ShardInterval *shardInterval; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { uint64 shardId = shardInterval->shardId; Oid relationId = shardInterval->relationId; @@ -725,10 +725,10 @@ ExecuteCreateIndexCommands(List *logicalRepTargetList) { List *taskList = NIL; LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; @@ -787,10 +787,10 @@ ExecuteCreateConstraintsBackedByIndexCommands(List *logicalRepTargetList) MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; @@ -873,10 +873,10 @@ ExecuteClusterOnCommands(List *logicalRepTargetList) { List *taskList = NIL; LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; @@ -925,10 +925,10 @@ ExecuteCreateIndexStatisticsCommands(List *logicalRepTargetList) MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; @@ -983,10 +983,10 @@ ExecuteRemainingPostLoadTableCommands(List *logicalRepTargetList) MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; @@ -1042,10 +1042,10 @@ CreatePartitioningHierarchy(List *logicalRepTargetList) MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { if (PartitionTable(shardInterval->relationId)) { @@ -1100,14 +1100,14 @@ CreateUncheckedForeignKeyConstraints(List *logicalRepTargetList) * Iterate over all the shards in the shard group. */ LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; /* * Iterate on split shards list for a given shard and create constraints. */ - foreach_ptr(shardInterval, target->newShards) + foreach_declared_ptr(shardInterval, target->newShards) { List *commandList = CopyShardForeignConstraintCommandList( shardInterval); @@ -1320,7 +1320,7 @@ CreatePublications(MultiConnection *connection, quote_identifier(entry->name)); ShardInterval *shard = NULL; - foreach_ptr(shard, entry->shardIntervals) + foreach_declared_ptr(shard, entry->shardIntervals) { char *shardName = ConstructQualifiedShardName(shard); @@ -1429,7 +1429,7 @@ CreateReplicationSlots(MultiConnection *sourceConnection, ReplicationSlotInfo *firstReplicationSlot = NULL; char *snapshot = NULL; LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ReplicationSlotInfo *replicationSlot = target->replicationSlot; @@ -1481,7 +1481,7 @@ CreateSubscriptions(MultiConnection *sourceConnection, List *logicalRepTargetList) { LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { int ownerId = target->tableOwnerId; @@ -1603,7 +1603,7 @@ void EnableSubscriptions(List *logicalRepTargetList) { LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { ExecuteCriticalRemoteCommand(target->superuserConnection, psprintf( "ALTER SUBSCRIPTION %s ENABLE", @@ -1737,7 +1737,7 @@ CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash, groupedLogicalRepTargets->superuserConnection = superuserConnection; LogicalRepTarget *target = NULL; - foreach_ptr(target, groupedLogicalRepTargets->logicalRepTargetList) + foreach_declared_ptr(target, groupedLogicalRepTargets->logicalRepTargetList) { target->superuserConnection = superuserConnection; } @@ -1774,7 +1774,7 @@ SubscriptionNamesValueList(List *logicalRepTargetList) bool first = true; LogicalRepTarget *target = NULL; - foreach_ptr(target, logicalRepTargetList) + foreach_declared_ptr(target, logicalRepTargetList) { if (!first) { diff --git a/src/backend/distributed/shardsplit/shardsplit_decoder.c b/src/backend/distributed/shardsplit/shardsplit_decoder.c index 51ab769f3..841fa89cd 100644 --- a/src/backend/distributed/shardsplit/shardsplit_decoder.c +++ b/src/backend/distributed/shardsplit/shardsplit_decoder.c @@ -291,7 +291,7 @@ FindTargetRelationOid(Relation sourceShardRelation, shardSplitInfo->distributedTableOid); shardSplitInfo = NULL; - foreach_ptr(shardSplitInfo, entry->shardSplitInfoList) + foreach_declared_ptr(shardSplitInfo, entry->shardSplitInfoList) { if (shardSplitInfo->shardMinValue <= hashValue && shardSplitInfo->shardMaxValue >= hashValue) diff --git a/src/backend/distributed/shardsplit/shardsplit_logical_replication.c b/src/backend/distributed/shardsplit/shardsplit_logical_replication.c index 328dc9af9..a18135372 100644 --- a/src/backend/distributed/shardsplit/shardsplit_logical_replication.c +++ b/src/backend/distributed/shardsplit/shardsplit_logical_replication.c @@ -154,7 +154,7 @@ AddPublishableShardEntryInMap(uint32 targetNodeId, ShardInterval *shardInterval, /* Check if parent is already added */ ShardInterval *existingShardInterval = NULL; - foreach_ptr(existingShardInterval, publicationInfo->shardIntervals) + foreach_declared_ptr(existingShardInterval, publicationInfo->shardIntervals) { if (existingShardInterval->shardId == shardInterval->shardId) { @@ -204,7 +204,7 @@ PopulateShardSplitSubscriptionsMetadataList(HTAB *shardSplitInfoHashMap, } List *shardIntervalList = NIL; - foreach_ptr(shardIntervalList, shardGroupSplitIntervalListList) + foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; @@ -256,7 +256,7 @@ CreateLogicalRepTarget(Oid tableOwnerId, uint32 nodeId, * table owner and node. */ ReplicationSlotInfo *replicationSlot = NULL; - foreach_ptr(replicationSlot, replicationSlotInfoList) + foreach_declared_ptr(replicationSlot, replicationSlotInfoList) { if (nodeId == replicationSlot->targetNodeId && tableOwnerId == replicationSlot->tableOwnerId) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index ada4399a2..ff16fa2df 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -2805,7 +2805,7 @@ ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource sou } char *appNamePrefix = NULL; - foreach_ptr(appNamePrefix, prefixList) + foreach_declared_ptr(appNamePrefix, prefixList) { int prefixLength = strlen(appNamePrefix); if (prefixLength >= NAMEDATALEN) diff --git a/src/backend/distributed/test/colocation_utils.c b/src/backend/distributed/test/colocation_utils.c index 6a87539c4..d1a738b80 100644 --- a/src/backend/distributed/test/colocation_utils.c +++ b/src/backend/distributed/test/colocation_utils.c @@ -93,7 +93,7 @@ get_colocated_table_array(PG_FUNCTION_ARGS) int colocatedTableIndex = 0; Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { Datum colocatedTableDatum = ObjectIdGetDatum(colocatedTableId); diff --git a/src/backend/distributed/test/create_shards.c b/src/backend/distributed/test/create_shards.c index 4ef13f1cb..d92a76059 100644 --- a/src/backend/distributed/test/create_shards.c +++ b/src/backend/distributed/test/create_shards.c @@ -46,7 +46,7 @@ sort_names(PG_FUNCTION_ARGS) StringInfo sortedNames = makeStringInfo(); const char *name = NULL; - foreach_ptr(name, nameList) + foreach_declared_ptr(name, nameList) { appendStringInfo(sortedNames, "%s\n", name); } diff --git a/src/backend/distributed/test/deparse_shard_query.c b/src/backend/distributed/test/deparse_shard_query.c index a9b4ced1d..1af5945cf 100644 --- a/src/backend/distributed/test/deparse_shard_query.c +++ b/src/backend/distributed/test/deparse_shard_query.c @@ -49,14 +49,14 @@ deparse_shard_query_test(PG_FUNCTION_ARGS) List *parseTreeList = pg_parse_query(queryStringChar); Node *parsetree = NULL; - foreach_ptr(parsetree, parseTreeList) + foreach_declared_ptr(parsetree, parseTreeList) { List *queryTreeList = pg_analyze_and_rewrite_fixedparams((RawStmt *) parsetree, queryStringChar, NULL, 0, NULL); Query *query = NULL; - foreach_ptr(query, queryTreeList) + foreach_declared_ptr(query, queryTreeList) { StringInfo buffer = makeStringInfo(); diff --git a/src/backend/distributed/test/dependency.c b/src/backend/distributed/test/dependency.c index 7afbfdec7..25a7ae6e4 100644 --- a/src/backend/distributed/test/dependency.c +++ b/src/backend/distributed/test/dependency.c @@ -50,7 +50,7 @@ citus_get_all_dependencies_for_object(PG_FUNCTION_ARGS) List *dependencies = GetAllSupportedDependenciesForObject(&address); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { Datum values[3]; bool isNulls[3]; @@ -95,7 +95,7 @@ citus_get_dependencies_for_object(PG_FUNCTION_ARGS) List *dependencies = GetDependenciesForObject(&address); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencies) + foreach_declared_ptr(dependency, dependencies) { Datum values[3]; bool isNulls[3]; diff --git a/src/backend/distributed/test/distributed_intermediate_results.c b/src/backend/distributed/test/distributed_intermediate_results.c index 843bda476..adbcbff89 100644 --- a/src/backend/distributed/test/distributed_intermediate_results.c +++ b/src/backend/distributed/test/distributed_intermediate_results.c @@ -90,7 +90,7 @@ partition_task_list_results(PG_FUNCTION_ARGS) Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); DistributedResultFragment *fragment = NULL; - foreach_ptr(fragment, fragmentList) + foreach_declared_ptr(fragment, fragmentList) { bool columnNulls[5] = { 0 }; Datum columnValues[5] = { @@ -169,7 +169,7 @@ redistribute_task_list_results(PG_FUNCTION_ARGS) const char *resultId = NULL; int resultIdIndex = 0; - foreach_ptr(resultId, sortedResultIds) + foreach_declared_ptr(resultId, sortedResultIds) { resultIdValues[resultIdIndex++] = CStringGetTextDatum(resultId); } diff --git a/src/backend/distributed/test/distribution_metadata.c b/src/backend/distributed/test/distribution_metadata.c index 01117922e..e75d3110b 100644 --- a/src/backend/distributed/test/distribution_metadata.c +++ b/src/backend/distributed/test/distribution_metadata.c @@ -74,7 +74,7 @@ load_shard_id_array(PG_FUNCTION_ARGS) Datum *shardIdDatumArray = palloc0(shardIdCount * sizeof(Datum)); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { Datum shardIdDatum = Int64GetDatum(shardInterval->shardId); @@ -144,7 +144,7 @@ load_shard_placement_array(PG_FUNCTION_ARGS) Datum *placementDatumArray = palloc0(placementCount * sizeof(Datum)); ShardPlacement *placement = NULL; - foreach_ptr(placement, placementList) + foreach_declared_ptr(placement, placementList) { appendStringInfo(placementInfo, "%s:%d", placement->nodeName, placement->nodePort); @@ -263,14 +263,14 @@ relation_count_in_query(PG_FUNCTION_ARGS) List *parseTreeList = pg_parse_query(queryStringChar); Node *parsetree = NULL; - foreach_ptr(parsetree, parseTreeList) + foreach_declared_ptr(parsetree, parseTreeList) { List *queryTreeList = pg_analyze_and_rewrite_fixedparams((RawStmt *) parsetree, queryStringChar, NULL, 0, NULL); Query *query = NULL; - foreach_ptr(query, queryTreeList) + foreach_declared_ptr(query, queryTreeList) { List *rangeTableList = NIL; diff --git a/src/backend/distributed/test/foreign_key_relationship_query.c b/src/backend/distributed/test/foreign_key_relationship_query.c index af187111a..8f96f5e3a 100644 --- a/src/backend/distributed/test/foreign_key_relationship_query.c +++ b/src/backend/distributed/test/foreign_key_relationship_query.c @@ -205,7 +205,7 @@ get_foreign_key_connected_relations(PG_FUNCTION_ARGS) Oid connectedRelationId; List *fkeyConnectedRelationIdList = GetForeignKeyConnectedRelationIdList(relationId); - foreach_oid(connectedRelationId, fkeyConnectedRelationIdList) + foreach_declared_oid(connectedRelationId, fkeyConnectedRelationIdList) { Datum values[GET_FKEY_CONNECTED_RELATIONS_COLUMNS]; bool nulls[GET_FKEY_CONNECTED_RELATIONS_COLUMNS]; diff --git a/src/backend/distributed/test/metadata_sync.c b/src/backend/distributed/test/metadata_sync.c index 4a5779b63..744183968 100644 --- a/src/backend/distributed/test/metadata_sync.c +++ b/src/backend/distributed/test/metadata_sync.c @@ -73,7 +73,7 @@ activate_node_snapshot(PG_FUNCTION_ARGS) sizeof(Datum)); const char *activateNodeSnapshotCommand = NULL; - foreach_ptr(activateNodeSnapshotCommand, activateNodeCommandList) + foreach_declared_ptr(activateNodeSnapshotCommand, activateNodeCommandList) { Datum activateNodeSnapshotCommandDatum = CStringGetTextDatum( activateNodeSnapshotCommand); @@ -105,7 +105,7 @@ wait_until_metadata_sync(PG_FUNCTION_ARGS) bool waitNotifications = false; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerList) + foreach_declared_ptr(workerNode, workerList) { /* if already has metadata, no need to do it again */ if (workerNode->hasMetadata && !workerNode->metadataSynced) diff --git a/src/backend/distributed/test/partitioning_utils.c b/src/backend/distributed/test/partitioning_utils.c index be9163561..f1e186bad 100644 --- a/src/backend/distributed/test/partitioning_utils.c +++ b/src/backend/distributed/test/partitioning_utils.c @@ -85,7 +85,7 @@ print_partitions(PG_FUNCTION_ARGS) partitionList = SortList(partitionList, CompareOids); Oid partitionOid = InvalidOid; - foreach_oid(partitionOid, partitionList) + foreach_declared_oid(partitionOid, partitionList) { /* at least one table is already added, add comma */ if (resultRelationNames->len > 0) diff --git a/src/backend/distributed/test/progress_utils.c b/src/backend/distributed/test/progress_utils.c index e1ea09e3d..7c335ce8a 100644 --- a/src/backend/distributed/test/progress_utils.c +++ b/src/backend/distributed/test/progress_utils.c @@ -95,7 +95,7 @@ show_progress(PG_FUNCTION_ARGS) Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); ProgressMonitorData *monitor = NULL; - foreach_ptr(monitor, monitorList) + foreach_declared_ptr(monitor, monitorList) { uint64 *steps = ProgressMonitorSteps(monitor); diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index f972281ec..f5bb9c979 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -224,7 +224,7 @@ PrunedShardIdsForTable(Oid distributedTableId, List *whereClauseList) Datum *shardIdDatumArray = palloc0(shardIdCount * sizeof(Datum)); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardList) + foreach_declared_ptr(shardInterval, shardList) { Datum shardIdDatum = Int64GetDatum(shardInterval->shardId); diff --git a/src/backend/distributed/test/shard_rebalancer.c b/src/backend/distributed/test/shard_rebalancer.c index 32bfd9f46..1b79fc27a 100644 --- a/src/backend/distributed/test/shard_rebalancer.c +++ b/src/backend/distributed/test/shard_rebalancer.c @@ -128,13 +128,13 @@ shard_placement_rebalance_array(PG_FUNCTION_ARGS) pfree(shardPlacementJsonArray); /* map workerTestInfoList to a list of its WorkerNodes */ - foreach_ptr(workerTestInfo, context.workerTestInfoList) + foreach_declared_ptr(workerTestInfo, context.workerTestInfoList) { workerNodeList = lappend(workerNodeList, workerTestInfo->node); } /* map shardPlacementTestInfoList to a list of list of its ShardPlacements */ - foreach_ptr(shardPlacementTestInfo, context.shardPlacementTestInfoList) + foreach_declared_ptr(shardPlacementTestInfo, context.shardPlacementTestInfoList) { if (shardPlacementTestInfo->nextColocationGroup) { @@ -197,7 +197,7 @@ ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *voidContext) RebalancePlacementContext *context = voidContext; WorkerTestInfo *workerTestInfo = NULL; uint64 *disallowedShardIdPtr = NULL; - foreach_ptr(workerTestInfo, context->workerTestInfoList) + foreach_declared_ptr(workerTestInfo, context->workerTestInfoList) { if (workerTestInfo->node == workerNode) { @@ -206,7 +206,7 @@ ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *voidContext) } Assert(workerTestInfo != NULL); - foreach_ptr(disallowedShardIdPtr, workerTestInfo->disallowedShardIds) + foreach_declared_ptr(disallowedShardIdPtr, workerTestInfo->disallowedShardIds) { if (shardId == *disallowedShardIdPtr) { @@ -226,7 +226,7 @@ NodeCapacity(WorkerNode *workerNode, void *voidContext) { RebalancePlacementContext *context = voidContext; WorkerTestInfo *workerTestInfo = NULL; - foreach_ptr(workerTestInfo, context->workerTestInfoList) + foreach_declared_ptr(workerTestInfo, context->workerTestInfoList) { if (workerTestInfo->node == workerNode) { @@ -251,7 +251,7 @@ GetShardCost(uint64 shardId, void *voidContext) shardCost.shardId = shardId; ShardPlacementTestInfo *shardPlacementTestInfo = NULL; - foreach_ptr(shardPlacementTestInfo, context->shardPlacementTestInfoList) + foreach_declared_ptr(shardPlacementTestInfo, context->shardPlacementTestInfoList) { if (shardPlacementTestInfo->placement->shardId == shardId) { @@ -300,12 +300,12 @@ shard_placement_replication_array(PG_FUNCTION_ARGS) pfree(workerNodeJsonArray); pfree(shardPlacementJsonArray); - foreach_ptr(workerTestInfo, workerTestInfoList) + foreach_declared_ptr(workerTestInfo, workerTestInfoList) { workerNodeList = lappend(workerNodeList, workerTestInfo->node); } - foreach_ptr(shardPlacementTestInfo, shardPlacementTestInfoList) + foreach_declared_ptr(shardPlacementTestInfo, shardPlacementTestInfoList) { shardPlacementList = lappend(shardPlacementList, shardPlacementTestInfo->placement); diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index 5f868f548..a0584bde8 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -267,7 +267,7 @@ get_global_active_transactions(PG_FUNCTION_ARGS) /* open connections in parallel */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -289,7 +289,7 @@ get_global_active_transactions(PG_FUNCTION_ARGS) /* send commands in parallel */ MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommand(connection, queryToSend->data); if (querySent == 0) @@ -299,7 +299,7 @@ get_global_active_transactions(PG_FUNCTION_ARGS) } /* receive query results */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; Datum values[ACTIVE_TRANSACTION_COLUMN_COUNT]; diff --git a/src/backend/distributed/transaction/distributed_deadlock_detection.c b/src/backend/distributed/transaction/distributed_deadlock_detection.c index 27bb48ee3..834fcbe4d 100644 --- a/src/backend/distributed/transaction/distributed_deadlock_detection.c +++ b/src/backend/distributed/transaction/distributed_deadlock_detection.c @@ -177,7 +177,7 @@ CheckForDistributedDeadlocks(void) * this node. */ TransactionNode *currentNode = NULL; - foreach_ptr(currentNode, deadlockPath) + foreach_declared_ptr(currentNode, deadlockPath) { bool transactionAssociatedWithProc = AssociateDistributedTransactionWithBackendProc(currentNode); @@ -305,7 +305,7 @@ PrependOutgoingNodesToQueue(TransactionNode *transactionNode, int currentStackDe /* prepend to the list to continue depth-first search */ TransactionNode *waitForTransaction = NULL; - foreach_ptr(waitForTransaction, transactionNode->waitsFor) + foreach_declared_ptr(waitForTransaction, transactionNode->waitsFor) { QueuedTransactionNode *queuedNode = palloc0(sizeof(QueuedTransactionNode)); @@ -672,7 +672,7 @@ WaitsForToString(List *waitsFor) StringInfo transactionIdStr = makeStringInfo(); TransactionNode *waitingNode = NULL; - foreach_ptr(waitingNode, waitsFor) + foreach_declared_ptr(waitingNode, waitsFor) { if (transactionIdStr->len != 0) { diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index 82f936243..a224c29e1 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -149,7 +149,7 @@ BuildGlobalWaitGraph(bool onlyDistributedTx) /* open connections in parallel */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -172,7 +172,7 @@ BuildGlobalWaitGraph(bool onlyDistributedTx) /* send commands in parallel */ MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { StringInfo queryString = makeStringInfo(); @@ -203,7 +203,7 @@ BuildGlobalWaitGraph(bool onlyDistributedTx) } /* receive dump_local_wait_edges results */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index 5044941c4..0ffa68d95 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -367,7 +367,7 @@ RecordRelationParallelSelectAccessForTask(Task *task) List *relationShardList = task->relationShardList; RelationShard *relationShard = NULL; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; @@ -412,7 +412,7 @@ RecordRelationParallelModifyAccessForTask(Task *task) { relationShardList = task->relationShardList; RelationShard *relationShard = NULL; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; @@ -446,7 +446,7 @@ RecordRelationParallelDDLAccessForTask(Task *task) Oid lastRelationId = InvalidOid; RelationShard *relationShard = NULL; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; @@ -534,7 +534,7 @@ RecordParallelRelationAccess(Oid relationId, ShardPlacementAccessType placementA List *partitionList = PartitionList(relationId); Oid partitionOid = InvalidOid; - foreach_oid(partitionOid, partitionList) + foreach_declared_oid(partitionOid, partitionList) { /* recursively record all relation accesses of its partitions */ RecordParallelRelationAccess(partitionOid, placementAccess); @@ -926,7 +926,7 @@ HoldsConflictingLockWithReferencedRelations(Oid relationId, ShardPlacementAccess CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); Oid referencedRelation = InvalidOid; - foreach_oid(referencedRelation, cacheEntry->referencedRelationsViaForeignKey) + foreach_declared_oid(referencedRelation, cacheEntry->referencedRelationsViaForeignKey) { /* * We're only interested in foreign keys to reference tables and citus @@ -997,7 +997,8 @@ HoldsConflictingLockWithReferencingRelations(Oid relationId, ShardPlacementAcces Assert(!IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE)); Oid referencingRelation = InvalidOid; - foreach_oid(referencingRelation, cacheEntry->referencingRelationsViaForeignKey) + foreach_declared_oid(referencingRelation, + cacheEntry->referencingRelationsViaForeignKey) { /* * We're only interested in foreign keys to reference tables from diff --git a/src/backend/distributed/transaction/remote_transaction.c b/src/backend/distributed/transaction/remote_transaction.c index 3dc89c995..d85b19842 100644 --- a/src/backend/distributed/transaction/remote_transaction.c +++ b/src/backend/distributed/transaction/remote_transaction.c @@ -100,7 +100,7 @@ StartRemoteTransactionBegin(struct MultiConnection *connection) transaction->lastQueuedSubXact = TopSubTransactionId; SubXactContext *subXactState = NULL; - foreach_ptr(subXactState, activeSubXacts) + foreach_declared_ptr(subXactState, activeSubXacts) { /* append SET LOCAL state from when SAVEPOINT was encountered... */ if (subXactState->setLocalCmds != NULL) @@ -311,13 +311,13 @@ RemoteTransactionListBegin(List *connectionList) MultiConnection *connection = NULL; /* send BEGIN to all nodes */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { StartRemoteTransactionBegin(connection); } /* wait for BEGIN to finish on all nodes */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { FinishRemoteTransactionBegin(connection); } @@ -724,7 +724,7 @@ RemoteTransactionsBeginIfNecessary(List *connectionList) } /* issue BEGIN to all connections needing it */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { RemoteTransaction *transaction = &connection->remoteTransaction; @@ -748,7 +748,7 @@ RemoteTransactionsBeginIfNecessary(List *connectionList) WaitForAllConnections(connectionList, raiseInterrupts); /* get result of all the BEGINs */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { RemoteTransaction *transaction = &connection->remoteTransaction; diff --git a/src/backend/distributed/transaction/transaction_management.c b/src/backend/distributed/transaction/transaction_management.c index d133d7be6..9fd0d1a81 100644 --- a/src/backend/distributed/transaction/transaction_management.c +++ b/src/backend/distributed/transaction/transaction_management.c @@ -1118,7 +1118,7 @@ TrackPropagatedTableAndSequences(Oid relationId) /* track its sequences */ List *ownedSeqIdList = getOwnedSequences(relationId); Oid ownedSeqId = InvalidOid; - foreach_oid(ownedSeqId, ownedSeqIdList) + foreach_declared_oid(ownedSeqId, ownedSeqIdList) { ObjectAddress *seqAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*seqAddress, RelationRelationId, ownedSeqId); @@ -1147,7 +1147,7 @@ HasAnyDependencyInPropagatedObjects(const ObjectAddress *objectAddress) { List *dependencyList = GetAllSupportedDependenciesForObject(objectAddress); ObjectAddress *dependency = NULL; - foreach_ptr(dependency, dependencyList) + foreach_declared_ptr(dependency, dependencyList) { /* first search in root transaction */ if (DependencyInPropagatedObjectsHash(PropagatedObjectsInTx, dependency)) @@ -1161,7 +1161,7 @@ HasAnyDependencyInPropagatedObjects(const ObjectAddress *objectAddress) continue; } SubXactContext *state = NULL; - foreach_ptr(state, activeSubXactContexts) + foreach_declared_ptr(state, activeSubXactContexts) { if (DependencyInPropagatedObjectsHash(state->propagatedObjects, dependency)) { diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index 0ec5ba0a3..f25823b30 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -124,7 +124,7 @@ RecoverTwoPhaseCommits(void) List *workerList = ActivePrimaryNodeList(NoLock); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerList) + foreach_declared_ptr(workerNode, workerList) { recoveredTransactionCount += RecoverWorkerTransactions(workerNode); } diff --git a/src/backend/distributed/transaction/worker_transaction.c b/src/backend/distributed/transaction/worker_transaction.c index f008c2974..e0a40ba49 100644 --- a/src/backend/distributed/transaction/worker_transaction.c +++ b/src/backend/distributed/transaction/worker_transaction.c @@ -72,7 +72,7 @@ SendCommandToWorkersAsUser(TargetWorkerSet targetWorkerSet, const char *nodeUser /* run commands serially */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -145,7 +145,7 @@ void SendCommandListToWorkersWithMetadata(List *commands) { char *command = NULL; - foreach_ptr(command, commands) + foreach_declared_ptr(command, commands) { SendCommandToWorkersWithMetadata(command); } @@ -171,7 +171,7 @@ TargetWorkerSetNodeList(TargetWorkerSet targetWorkerSet, LOCKMODE lockMode) List *result = NIL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if ((targetWorkerSet == NON_COORDINATOR_METADATA_NODES || targetWorkerSet == METADATA_NODES) && @@ -205,7 +205,7 @@ SendBareCommandListToMetadataWorkers(List *commandList) /* run commands serially */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -218,7 +218,7 @@ SendBareCommandListToMetadataWorkers(List *commandList) /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; - foreach_ptr(commandString, commandList) + foreach_declared_ptr(commandString, commandList) { ExecuteCriticalRemoteCommand(workerConnection, commandString); } @@ -271,7 +271,7 @@ SendCommandToWorkersParamsInternal(TargetWorkerSet targetWorkerSet, const char * /* open connections in parallel */ WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -299,7 +299,7 @@ SendCommandToWorkersParamsInternal(TargetWorkerSet targetWorkerSet, const char * /* send commands in parallel */ MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommandParams(connection, command, parameterCount, parameterTypes, parameterValues, false); @@ -310,7 +310,7 @@ SendCommandToWorkersParamsInternal(TargetWorkerSet targetWorkerSet, const char * } /* get results */ - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { PGresult *result = GetRemoteCommandResult(connection, true); if (!IsResponseOK(result)) @@ -381,7 +381,7 @@ SendCommandListToWorkerOutsideTransactionWithConnection(MultiConnection *workerC /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; - foreach_ptr(commandString, commandList) + foreach_declared_ptr(commandString, commandList) { ExecuteCriticalRemoteCommand(workerConnection, commandString); } @@ -422,7 +422,7 @@ SendCommandListToWorkerListWithBareConnections(List *workerConnectionList, /* send commands in parallel */ MultiConnection *connection = NULL; - foreach_ptr(connection, workerConnectionList) + foreach_declared_ptr(connection, workerConnectionList) { int querySent = SendRemoteCommand(connection, stringToSend); if (querySent == 0) @@ -432,7 +432,7 @@ SendCommandListToWorkerListWithBareConnections(List *workerConnectionList, } bool failOnError = true; - foreach_ptr(connection, workerConnectionList) + foreach_declared_ptr(connection, workerConnectionList) { ClearResults(connection, failOnError); } @@ -462,7 +462,7 @@ SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList List *connectionList = NIL; WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; @@ -499,7 +499,7 @@ SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList /* send commands in parallel */ bool failOnError = true; MultiConnection *connection = NULL; - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommand(connection, stringToSend); if (querySent == 0) @@ -508,7 +508,7 @@ SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList } } - foreach_ptr(connection, connectionList) + foreach_declared_ptr(connection, connectionList) { ClearResults(connection, failOnError); } @@ -537,7 +537,7 @@ SendOptionalCommandListToWorkerOutsideTransactionWithConnection( /* iterate over the commands and execute them in the same connection */ bool failed = false; const char *commandString = NULL; - foreach_ptr(commandString, commandList) + foreach_declared_ptr(commandString, commandList) { if (ExecuteOptionalRemoteCommand(workerConnection, commandString, NULL) != 0) { @@ -613,7 +613,7 @@ SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(const char *node /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; - foreach_ptr(commandString, commandList) + foreach_declared_ptr(commandString, commandList) { if (ExecuteOptionalRemoteCommand(workerConnection, commandString, NULL) != RESPONSE_OKAY) @@ -648,7 +648,7 @@ static void ErrorIfAnyMetadataNodeOutOfSync(List *metadataNodeList) { WorkerNode *metadataNode = NULL; - foreach_ptr(metadataNode, metadataNodeList) + foreach_declared_ptr(metadataNode, metadataNodeList) { Assert(metadataNode->hasMetadata); diff --git a/src/backend/distributed/utils/background_jobs.c b/src/backend/distributed/utils/background_jobs.c index a7a124c74..73a635f21 100644 --- a/src/backend/distributed/utils/background_jobs.c +++ b/src/backend/distributed/utils/background_jobs.c @@ -158,7 +158,7 @@ citus_job_cancel(PG_FUNCTION_ARGS) /* send cancellation to any running backends */ int pid = 0; - foreach_int(pid, pids) + foreach_declared_int(pid, pids) { Datum pidDatum = Int32GetDatum(pid); Datum signalSuccessDatum = DirectFunctionCall1(pg_cancel_backend, pidDatum); @@ -891,7 +891,7 @@ IncrementParallelTaskCountForNodesInvolved(BackgroundTask *task) int node; /* first check whether we have reached the limit for any of the nodes */ - foreach_int(node, task->nodesInvolved) + foreach_declared_int(node, task->nodesInvolved) { bool found; ParallelTasksPerNodeEntry *hashEntry = hash_search( @@ -908,7 +908,7 @@ IncrementParallelTaskCountForNodesInvolved(BackgroundTask *task) } /* then, increment the parallel task count per each node */ - foreach_int(node, task->nodesInvolved) + foreach_declared_int(node, task->nodesInvolved) { ParallelTasksPerNodeEntry *hashEntry = hash_search( ParallelTasksPerNode, &(node), HASH_FIND, NULL); @@ -934,7 +934,7 @@ DecrementParallelTaskCountForNodesInvolved(BackgroundTask *task) if (task->nodesInvolved) { int node; - foreach_int(node, task->nodesInvolved) + foreach_declared_int(node, task->nodesInvolved) { ParallelTasksPerNodeEntry *hashEntry = hash_search(ParallelTasksPerNode, &(node), @@ -1278,7 +1278,7 @@ CitusBackgroundTaskQueueMonitorMain(Datum arg) /* iterate over all handle entries and monitor each task's output */ BackgroundExecutorHashEntry *handleEntry = NULL; - foreach_ptr(handleEntry, runningTaskEntries) + foreach_declared_ptr(handleEntry, runningTaskEntries) { /* create task execution context and assign it to queueMonitorExecutionContext */ TaskExecutionContext taskExecutionContext = { @@ -1916,7 +1916,7 @@ ExecuteSqlString(const char *sql) * analysis on the next one, since there may be interdependencies. */ RawStmt *parsetree = NULL; - foreach_ptr(parsetree, raw_parsetree_list) + foreach_declared_ptr(parsetree, raw_parsetree_list) { /* * We don't allow transaction-control commands like COMMIT and ABORT diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index abbb62839..697b71655 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -78,7 +78,7 @@ CitusSetTag(Node *node, int tag) do { \ char *curString = NULL; \ List *newList = NIL; \ - foreach_ptr(curString, from->fldname) { \ + foreach_declared_ptr(curString, from->fldname) { \ char *newString = curString ? pstrdup(curString) : (char *) NULL; \ newList = lappend(newList, newString); \ } \ diff --git a/src/backend/distributed/utils/citus_depended_object.c b/src/backend/distributed/utils/citus_depended_object.c index 7588f8594..3babf76f0 100644 --- a/src/backend/distributed/utils/citus_depended_object.c +++ b/src/backend/distributed/utils/citus_depended_object.c @@ -138,7 +138,7 @@ HideCitusDependentObjectsOnQueriesOfPgMetaTables(Node *node, void *context) int varno = 0; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, query->rtable) + foreach_declared_ptr(rangeTableEntry, query->rtable) { varno++; @@ -376,7 +376,7 @@ DistOpsValidityState(Node *node, const DistributeObjectOps *ops) bool isPostprocess = false; List *objectAddresses = ops->address(node, missingOk, isPostprocess); ObjectAddress *objectAddress = NULL; - foreach_ptr(objectAddress, objectAddresses) + foreach_declared_ptr(objectAddress, objectAddresses) { if (OidIsValid(objectAddress->objectId)) { @@ -478,7 +478,7 @@ AnyObjectViolatesOwnership(DropStmt *dropStmt) PG_TRY(); { Node *object = NULL; - foreach_ptr(object, dropStmt->objects) + foreach_declared_ptr(object, dropStmt->objects) { Relation rel = NULL; objectAddress = get_object_address(objectType, object, diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index b8db3c80f..c87db3b95 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -204,7 +204,7 @@ get_colocated_shard_array(PG_FUNCTION_ARGS) int colocatedShardIndex = 0; ShardInterval *colocatedShardInterval = NULL; - foreach_ptr(colocatedShardInterval, colocatedShardList) + foreach_declared_ptr(colocatedShardInterval, colocatedShardList) { uint64 colocatedShardId = colocatedShardInterval->shardId; @@ -1065,7 +1065,7 @@ ColocatedShardIntervalList(ShardInterval *shardInterval) Assert(shardIntervalIndex >= 0); Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { CitusTableCacheEntry *colocatedTableCacheEntry = GetCitusTableCacheEntry(colocatedTableId); @@ -1131,7 +1131,7 @@ ColocatedNonPartitionShardIntervalList(ShardInterval *shardInterval) Assert(shardIntervalIndex >= 0); Oid colocatedTableId = InvalidOid; - foreach_oid(colocatedTableId, colocatedTableList) + foreach_declared_oid(colocatedTableId, colocatedTableList) { if (PartitionTable(colocatedTableId)) { diff --git a/src/backend/distributed/utils/distribution_column_map.c b/src/backend/distributed/utils/distribution_column_map.c index 43f9939b1..380a5f98f 100644 --- a/src/backend/distributed/utils/distribution_column_map.c +++ b/src/backend/distributed/utils/distribution_column_map.c @@ -81,7 +81,7 @@ AddDistributionColumnForRelation(DistributionColumnMap *distributionColumnMap, List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { AddDistributionColumnForRelation(distributionColumnMap, partitionRelationId, distributionColumnName); diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index 1abb7ae07..0025becb4 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -190,7 +190,7 @@ GetRelationshipNodesForFKeyConnectedRelations( { List *allNeighboursList = GetAllNeighboursList(currentNode); ForeignConstraintRelationshipNode *neighbourNode = NULL; - foreach_ptr(neighbourNode, allNeighboursList) + foreach_declared_ptr(neighbourNode, allNeighboursList) { Oid neighbourRelationId = neighbourNode->relationId; if (OidVisited(oidVisitedMap, neighbourRelationId)) @@ -437,7 +437,7 @@ GetConnectedListHelper(ForeignConstraintRelationshipNode *node, bool isReferenci List *neighbourList = GetNeighbourList(currentNode, isReferencing); ForeignConstraintRelationshipNode *neighbourNode = NULL; - foreach_ptr(neighbourNode, neighbourList) + foreach_declared_ptr(neighbourNode, neighbourList) { Oid neighbourRelationId = neighbourNode->relationId; if (!OidVisited(oidVisitedMap, neighbourRelationId)) @@ -508,7 +508,7 @@ GetRelationIdsFromRelationshipNodeList(List *fKeyRelationshipNodeList) List *relationIdList = NIL; ForeignConstraintRelationshipNode *fKeyRelationshipNode = NULL; - foreach_ptr(fKeyRelationshipNode, fKeyRelationshipNodeList) + foreach_declared_ptr(fKeyRelationshipNode, fKeyRelationshipNodeList) { Oid relationId = fKeyRelationshipNode->relationId; relationIdList = lappend_oid(relationIdList, relationId); @@ -561,7 +561,7 @@ PopulateAdjacencyLists(void) frelEdgeList = SortList(frelEdgeList, CompareForeignConstraintRelationshipEdges); ForeignConstraintRelationshipEdge *currentFConstraintRelationshipEdge = NULL; - foreach_ptr(currentFConstraintRelationshipEdge, frelEdgeList) + foreach_declared_ptr(currentFConstraintRelationshipEdge, frelEdgeList) { /* we just saw this edge, no need to add it twice */ if (currentFConstraintRelationshipEdge->referencingRelationOID == diff --git a/src/backend/distributed/utils/listutils.c b/src/backend/distributed/utils/listutils.c index eddef1fea..6f3c73e55 100644 --- a/src/backend/distributed/utils/listutils.c +++ b/src/backend/distributed/utils/listutils.c @@ -43,7 +43,7 @@ SortList(List *pointerList, int (*comparisonFunction)(const void *, const void * void **array = (void **) palloc0(arraySize * sizeof(void *)); void *pointer = NULL; - foreach_ptr(pointer, pointerList) + foreach_declared_ptr(pointer, pointerList) { array[arrayIndex] = pointer; @@ -82,7 +82,7 @@ PointerArrayFromList(List *pointerList) int pointerIndex = 0; void *pointer = NULL; - foreach_ptr(pointer, pointerList) + foreach_declared_ptr(pointer, pointerList) { pointerArray[pointerIndex] = pointer; pointerIndex += 1; @@ -130,7 +130,7 @@ ListToHashSet(List *itemList, Size keySize, bool isStringList) HTAB *itemSet = hash_create("ListToHashSet", capacity, &info, flags); void *item = NULL; - foreach_ptr(item, itemList) + foreach_declared_ptr(item, itemList) { bool foundInSet = false; @@ -188,7 +188,7 @@ StringJoinParams(List *stringList, char delimiter, char *prefix, char *postfix) const char *command = NULL; int curIndex = 0; - foreach_ptr(command, stringList) + foreach_declared_ptr(command, stringList) { if (curIndex > 0) { @@ -219,7 +219,7 @@ ListTake(List *pointerList, int size) int listIndex = 0; void *pointer = NULL; - foreach_ptr(pointer, pointerList) + foreach_declared_ptr(pointer, pointerList) { result = lappend(result, pointer); listIndex++; @@ -279,7 +279,7 @@ list_filter_oid(List *list, bool (*keepElement)(Oid element)) { List *result = NIL; Oid element = InvalidOid; - foreach_oid(element, list) + foreach_declared_oid(element, list) { if (keepElement(element)) { diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index ede2008ca..063465beb 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -259,7 +259,7 @@ worker_fix_partition_shard_index_names(PG_FUNCTION_ARGS) List *partitionShardIndexIds = find_inheritance_children(parentShardIndexId, ShareRowExclusiveLock); Oid partitionShardIndexId = InvalidOid; - foreach_oid(partitionShardIndexId, partitionShardIndexIds) + foreach_declared_oid(partitionShardIndexId, partitionShardIndexIds) { if (IndexGetRelation(partitionShardIndexId, false) == partitionShardId) { @@ -372,7 +372,7 @@ CreateFixPartitionConstraintsTaskList(Oid relationId) LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; @@ -458,7 +458,7 @@ WorkerFixPartitionConstraintCommandList(Oid relationId, uint64 shardId, char *quotedShardName = quote_qualified_identifier(schemaName, shardRelationName); char *constraintName = NULL; - foreach_ptr(constraintName, checkConstraintList) + foreach_declared_ptr(constraintName, checkConstraintList) { StringInfo shardQueryString = makeStringInfo(); appendStringInfo(shardQueryString, @@ -543,7 +543,7 @@ CreateFixPartitionShardIndexNames(Oid parentRelationId, Oid partitionRelationId, else { Oid partitionId = InvalidOid; - foreach_oid(partitionId, partitionList) + foreach_declared_oid(partitionId, partitionList) { List *partitionShardIntervalList = LoadShardIntervalList(partitionId); LockShardListMetadata(partitionShardIntervalList, ShareLock); @@ -563,7 +563,7 @@ CreateFixPartitionShardIndexNames(Oid parentRelationId, Oid partitionRelationId, int taskId = 1; ShardInterval *parentShardInterval = NULL; - foreach_ptr(parentShardInterval, parentShardIntervalList) + foreach_declared_ptr(parentShardInterval, parentShardIntervalList) { uint64 parentShardId = parentShardInterval->shardId; @@ -615,7 +615,7 @@ WorkerFixPartitionShardIndexNamesCommandList(uint64 parentShardId, { List *commandList = NIL; Oid parentIndexId = InvalidOid; - foreach_oid(parentIndexId, parentIndexIdList) + foreach_declared_oid(parentIndexId, parentIndexIdList) { if (!has_subclass(parentIndexId)) { @@ -666,7 +666,7 @@ WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex( bool addAllPartitions = (partitionRelationId == InvalidOid); Oid partitionIndexId = InvalidOid; - foreach_oid(partitionIndexId, partitionIndexIds) + foreach_declared_oid(partitionIndexId, partitionIndexIds) { Oid partitionId = IndexGetRelation(partitionIndexId, false); if (addAllPartitions || partitionId == partitionRelationId) @@ -701,7 +701,7 @@ WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(Oid partitionIndex List *partitionShardIntervalList = LoadShardIntervalList(partitionId); ShardInterval *partitionShardInterval = NULL; - foreach_ptr(partitionShardInterval, partitionShardIntervalList) + foreach_declared_ptr(partitionShardInterval, partitionShardIntervalList) { /* * Prepare commands for each shard of current partition @@ -1044,7 +1044,7 @@ PartitionWithLongestNameRelationId(Oid parentRelationId) List *partitionList = PartitionList(parentRelationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { char *partitionName = get_rel_name(partitionRelationId); int partitionNameLength = strnlen(partitionName, NAMEDATALEN); @@ -1130,7 +1130,7 @@ GenerateDetachPartitionCommandRelationIdList(List *relationIds) { List *detachPartitionCommands = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIds) + foreach_declared_oid(relationId, relationIds) { Assert(PartitionTable(relationId)); char *detachCommand = GenerateDetachPartitionCommand(relationId); @@ -1246,7 +1246,7 @@ GenerateAttachPartitionCommandRelationIdList(List *relationIds) { List *attachPartitionCommands = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationIds) + foreach_declared_oid(relationId, relationIds) { char *attachCommand = GenerateAlterTableAttachPartitionCommand(relationId); attachPartitionCommands = lappend(attachPartitionCommands, attachCommand); @@ -1318,7 +1318,7 @@ ListShardsUnderParentRelation(Oid relationId) List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) + foreach_declared_oid(partitionRelationId, partitionList) { List *childShardList = ListShardsUnderParentRelation(partitionRelationId); shardList = list_concat(shardList, childShardList); diff --git a/src/backend/distributed/utils/reference_table_utils.c b/src/backend/distributed/utils/reference_table_utils.c index b1710c1d6..8f0d89fc9 100644 --- a/src/backend/distributed/utils/reference_table_utils.c +++ b/src/backend/distributed/utils/reference_table_utils.c @@ -228,7 +228,7 @@ EnsureReferenceTablesExistOnAllNodesExtended(char transferMode) } WorkerNode *newWorkerNode = NULL; - foreach_ptr(newWorkerNode, newWorkersList) + foreach_declared_ptr(newWorkerNode, newWorkersList) { ereport(NOTICE, (errmsg("replicating reference table '%s' to %s:%d ...", referenceTableName, newWorkerNode->workerName, @@ -360,7 +360,7 @@ AnyRelationsModifiedInTransaction(List *relationIdList) { Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) + foreach_declared_oid(relationId, relationIdList) { if (GetRelationDDLAccessMode(relationId) != RELATION_NOT_ACCESSED || GetRelationDMLAccessMode(relationId) != RELATION_NOT_ACCESSED) @@ -389,7 +389,7 @@ WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode) workerNodeList = SortList(workerNodeList, CompareWorkerNodes); WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { char *nodeName = workerNode->workerName; uint32 nodePort = workerNode->workerPort; @@ -538,7 +538,7 @@ ReplicatedPlacementsForNodeGroup(int32 groupId) List *replicatedPlacementsForNodeGroup = NIL; Oid replicatedTableId = InvalidOid; - foreach_oid(replicatedTableId, replicatedTableList) + foreach_declared_oid(replicatedTableId, replicatedTableList) { List *placements = GroupShardPlacementsForTableOnGroup(replicatedTableId, groupId); @@ -591,7 +591,7 @@ DeleteAllReplicatedTablePlacementsFromNodeGroup(int32 groupId, bool localOnly) } GroupShardPlacement *placement = NULL; - foreach_ptr(placement, replicatedPlacementListForGroup) + foreach_declared_ptr(placement, replicatedPlacementListForGroup) { LockShardDistributionMetadata(placement->shardId, ExclusiveLock); @@ -627,7 +627,7 @@ DeleteAllReplicatedTablePlacementsFromNodeGroupViaMetadataContext( MemoryContext oldContext = MemoryContextSwitchTo(context->context); GroupShardPlacement *placement = NULL; - foreach_ptr(placement, replicatedPlacementListForGroup) + foreach_declared_ptr(placement, replicatedPlacementListForGroup) { LockShardDistributionMetadata(placement->shardId, ExclusiveLock); @@ -663,7 +663,7 @@ ReplicatedMetadataSyncedDistributedTableList(void) List *replicatedHashDistributedTableList = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, distributedRelationList) + foreach_declared_oid(relationId, distributedRelationList) { if (ShouldSyncTableMetadata(relationId) && !SingleReplicatedTable(relationId)) { @@ -707,7 +707,7 @@ ErrorIfNotAllNodesHaveReferenceTableReplicas(List *workerNodeList) { WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { if (!NodeHasAllReferenceTableReplicas(workerNode)) { @@ -763,7 +763,7 @@ NodeHasAllReferenceTableReplicas(WorkerNode *workerNode) List *shardPlacementList = ActiveShardPlacementList(shardInterval->shardId); ShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { if (placement->groupId == workerNode->groupId) { diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index 8ac269e43..3f50b682e 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -299,7 +299,7 @@ LockShardListResourcesOnFirstWorker(LOCKMODE lockmode, List *shardIntervalList) appendStringInfo(lockCommand, "SELECT lock_shard_resources(%d, ARRAY[", lockmode); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; @@ -388,7 +388,7 @@ LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList) appendStringInfo(lockCommand, "SELECT lock_shard_metadata(%d, ARRAY[", lockmode); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; @@ -529,7 +529,7 @@ LockReferencedReferenceShardDistributionMetadata(uint64 shardId, LOCKMODE lockMo } ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { LockShardDistributionMetadata(shardInterval->shardId, lockMode); } @@ -573,7 +573,7 @@ LockReferencedReferenceShardResources(uint64 shardId, LOCKMODE lockMode) } ShardInterval *referencedShardInterval = NULL; - foreach_ptr(referencedShardInterval, referencedShardIntervalList) + foreach_declared_ptr(referencedShardInterval, referencedShardIntervalList) { LockShardResource(referencedShardInterval->shardId, lockMode); } @@ -590,7 +590,7 @@ GetSortedReferenceShardIntervals(List *relationList) List *shardIntervalList = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationList) + foreach_declared_oid(relationId, relationList) { if (!IsCitusTableType(relationId, REFERENCE_TABLE)) { @@ -652,7 +652,7 @@ LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode) shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; @@ -673,7 +673,7 @@ LockShardsInPlacementListMetadata(List *shardPlacementList, LOCKMODE lockMode) SortList(shardPlacementList, CompareShardPlacementsByShardId); GroupShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { int64 shardId = placement->shardId; @@ -760,7 +760,7 @@ AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList) List *localList = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; @@ -797,7 +797,7 @@ LockShardListResources(List *shardIntervalList, LOCKMODE lockMode) shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; @@ -820,7 +820,7 @@ LockRelationShardResources(List *relationShardList, LOCKMODE lockMode) List *shardIntervalList = NIL; RelationShard *relationShard = NULL; - foreach_ptr(relationShard, relationShardList) + foreach_declared_ptr(relationShard, relationShardList) { uint64 shardId = relationShard->shardId; @@ -846,7 +846,7 @@ LockParentShardResourceIfPartition(List *shardIntervalList, LOCKMODE lockMode) List *parentShardIntervalList = NIL; ShardInterval *shardInterval = NULL; - foreach_ptr(shardInterval, shardIntervalList) + foreach_declared_ptr(shardInterval, shardIntervalList) { Oid relationId = shardInterval->relationId; @@ -1092,7 +1092,7 @@ static bool LockRelationRecordListMember(List *lockRelationRecordList, Oid relationId) { LockRelationRecord *record = NULL; - foreach_ptr(record, lockRelationRecordList) + foreach_declared_ptr(record, lockRelationRecordList) { if (record->relationId == relationId) { @@ -1131,7 +1131,7 @@ ConcatLockRelationRecordList(List *lockRelationRecordList, List *relationOidList List *constructedList = NIL; Oid relationId = InvalidOid; - foreach_oid(relationId, relationOidList) + foreach_declared_oid(relationId, relationOidList) { if (!LockRelationRecordListMember(lockRelationRecordList, relationId)) { @@ -1178,7 +1178,7 @@ AcquireDistributedLockOnRelations_Internal(List *lockRelationRecordList, int lockedRelations = 0; LockRelationRecord *lockRelationRecord; - foreach_ptr(lockRelationRecord, lockRelationRecordList) + foreach_declared_ptr(lockRelationRecord, lockRelationRecordList) { Oid relationId = lockRelationRecord->relationId; bool lockDescendants = lockRelationRecord->inh; @@ -1251,7 +1251,7 @@ AcquireDistributedLockOnRelations_Internal(List *lockRelationRecordList, WorkerNode *workerNode = NULL; const char *currentUser = CurrentUserName(); - foreach_ptr(workerNode, workerNodeList) + foreach_declared_ptr(workerNode, workerNodeList) { /* if local node is one of the targets, acquire the lock locally */ if (workerNode->groupId == localGroupId) @@ -1294,7 +1294,7 @@ AcquireDistributedLockOnRelations(List *relationList, LOCKMODE lockMode, uint32 bool nowait = (configs & DIST_LOCK_NOWAIT) > 0; RangeVar *rangeVar = NULL; - foreach_ptr(rangeVar, relationList) + foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); diff --git a/src/backend/distributed/utils/shardinterval_utils.c b/src/backend/distributed/utils/shardinterval_utils.c index 16d43ffdc..aa45c50cb 100644 --- a/src/backend/distributed/utils/shardinterval_utils.c +++ b/src/backend/distributed/utils/shardinterval_utils.c @@ -472,7 +472,7 @@ SingleReplicatedTable(Oid relationId) List *shardIntervalList = LoadShardList(relationId); uint64 *shardIdPointer = NULL; - foreach_ptr(shardIdPointer, shardIntervalList) + foreach_declared_ptr(shardIdPointer, shardIntervalList) { uint64 shardId = *shardIdPointer; shardPlacementList = ShardPlacementListSortedByWorker(shardId); diff --git a/src/backend/distributed/utils/statistics_collection.c b/src/backend/distributed/utils/statistics_collection.c index 1cadea968..649c9dc82 100644 --- a/src/backend/distributed/utils/statistics_collection.c +++ b/src/backend/distributed/utils/statistics_collection.c @@ -184,7 +184,7 @@ DistributedTablesSize(List *distTableOids) uint64 totalSize = 0; Oid relationId = InvalidOid; - foreach_oid(relationId, distTableOids) + foreach_declared_oid(relationId, distTableOids) { /* * Relations can get dropped after getting the Oid list and before we diff --git a/src/backend/distributed/worker/worker_create_or_replace.c b/src/backend/distributed/worker/worker_create_or_replace.c index 2fab84ac6..451649969 100644 --- a/src/backend/distributed/worker/worker_create_or_replace.c +++ b/src/backend/distributed/worker/worker_create_or_replace.c @@ -85,7 +85,7 @@ WrapCreateOrReplaceList(List *sqls) appendStringInfoString(&textArrayLitteral, "ARRAY["); const char *sql = NULL; bool first = true; - foreach_ptr(sql, sqls) + foreach_declared_ptr(sql, sqls) { if (!first) { @@ -251,7 +251,7 @@ WorkerCreateOrReplaceObject(List *sqlStatements) /* apply all statement locally */ char *sqlStatement = NULL; - foreach_ptr(sqlStatement, sqlStatements) + foreach_declared_ptr(sqlStatement, sqlStatements) { parseTree = ParseTreeNode(sqlStatement); ProcessUtilityParseTree(parseTree, sqlStatement, PROCESS_UTILITY_QUERY, NULL, diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index f51d9c80c..d2b60aa50 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -377,7 +377,7 @@ check_log_statement(List *statementList) /* else we have to inspect the statement(s) to see whether to log */ Node *statement = NULL; - foreach_ptr(statement, statementList) + foreach_declared_ptr(statement, statementList) { if (GetCommandLogLevel(statement) <= log_statement) { @@ -480,7 +480,7 @@ void SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg) { DefElem *defElem = NULL; - foreach_ptr(defElem, statement->options) + foreach_declared_ptr(defElem, statement->options) { if (strcmp(defElem->defname, name) == 0) { diff --git a/src/backend/distributed/worker/worker_drop_protocol.c b/src/backend/distributed/worker/worker_drop_protocol.c index 6d7b5326a..1103d6c90 100644 --- a/src/backend/distributed/worker/worker_drop_protocol.c +++ b/src/backend/distributed/worker/worker_drop_protocol.c @@ -93,7 +93,7 @@ worker_drop_distributed_table(PG_FUNCTION_ARGS) */ List *partitionList = PartitionList(relationId); Oid partitionOid = InvalidOid; - foreach_oid(partitionOid, partitionList) + foreach_declared_oid(partitionOid, partitionList) { WorkerDropDistributedTable(partitionOid); } @@ -128,7 +128,7 @@ WorkerDropDistributedTable(Oid relationId) List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; - foreach_oid(ownedSequenceOid, ownedSequences) + foreach_declared_oid(ownedSequenceOid, ownedSequences) { ObjectAddress ownedSequenceAddress = { 0 }; ObjectAddressSet(ownedSequenceAddress, RelationRelationId, ownedSequenceOid); @@ -144,13 +144,13 @@ WorkerDropDistributedTable(Oid relationId) */ List *shardList = LoadShardList(relationId); uint64 *shardIdPointer = NULL; - foreach_ptr(shardIdPointer, shardList) + foreach_declared_ptr(shardIdPointer, shardList) { uint64 shardId = *shardIdPointer; List *shardPlacementList = ShardPlacementList(shardId); ShardPlacement *placement = NULL; - foreach_ptr(placement, shardPlacementList) + foreach_declared_ptr(placement, shardPlacementList) { /* delete the row from pg_dist_placement */ DeleteShardPlacementRow(placement->placementId); @@ -240,7 +240,7 @@ worker_drop_shell_table(PG_FUNCTION_ARGS) List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; - foreach_oid(ownedSequenceOid, ownedSequences) + foreach_declared_oid(ownedSequenceOid, ownedSequences) { ObjectAddress ownedSequenceAddress = { 0 }; ObjectAddressSet(ownedSequenceAddress, RelationRelationId, ownedSequenceOid); @@ -288,7 +288,7 @@ worker_drop_sequence_dependency(PG_FUNCTION_ARGS) List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; - foreach_oid(ownedSequenceOid, ownedSequences) + foreach_declared_oid(ownedSequenceOid, ownedSequences) { /* the caller doesn't want to drop the sequence, so break the dependency */ deleteDependencyRecordsForSpecific(RelationRelationId, ownedSequenceOid, diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index 3725800c3..f783d514d 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -382,7 +382,7 @@ ShouldHideShardsInternal(void) } char *appNamePrefix = NULL; - foreach_ptr(appNamePrefix, prefixList) + foreach_declared_ptr(appNamePrefix, prefixList) { /* never hide shards when one of the prefixes is * */ if (strcmp(appNamePrefix, "*") == 0) @@ -446,7 +446,7 @@ FilterShardsFromPgclass(Node *node, void *context) int varno = 0; RangeTblEntry *rangeTableEntry = NULL; - foreach_ptr(rangeTableEntry, query->rtable) + foreach_declared_ptr(rangeTableEntry, query->rtable) { varno++; diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index 2a52cbc75..db9ea7ce7 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -36,7 +36,7 @@ typedef struct ListCellAndListWrapper } ListCellAndListWrapper; /* - * foreach_ptr - + * foreach_declared_ptr - * a convenience macro which loops through a pointer list without needing a * ListCell, just a declared pointer variable to store the pointer of the * cell in. @@ -50,7 +50,7 @@ typedef struct ListCellAndListWrapper * - || true is used to always enter the loop when cell is not null even if * var is NULL. */ -#define foreach_ptr(var, l) \ +#define foreach_declared_ptr(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst(var ## CellDoNotUse)) || true); \ @@ -58,12 +58,12 @@ typedef struct ListCellAndListWrapper /* - * foreach_int - + * foreach_declared_int - * a convenience macro which loops through an int list without needing a * ListCell, just a declared int variable to store the int of the cell in. - * For explanation of how it works see foreach_ptr. + * For explanation of how it works see foreach_declared_ptr. */ -#define foreach_int(var, l) \ +#define foreach_declared_int(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_int(var ## CellDoNotUse)) || true); \ @@ -71,12 +71,12 @@ typedef struct ListCellAndListWrapper /* - * foreach_oid - + * foreach_declared_oid - * a convenience macro which loops through an oid list without needing a * ListCell, just a declared Oid variable to store the oid of the cell in. - * For explanation of how it works see foreach_ptr. + * For explanation of how it works see foreach_declared_ptr. */ -#define foreach_oid(var, l) \ +#define foreach_declared_oid(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_oid(var ## CellDoNotUse)) || true); \ From da2624cee82acd1ab53435b215fbb67c190a9e96 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:37:13 +0300 Subject: [PATCH 054/155] PG17 compatibility: Resolve compilation issues (#7699) This PR provides successful compilation against PG17.0. - Remove ExecFreeExprContext call Relevant PG commit d060e921ea5aa47b6265174c32e1128cebdbc3df https://github.com/postgres/postgres/commit/d060e921ea5aa47b6265174c32e1128cebdbc3df - PG17 uses streaming IO in analyze, fix scan_analyze_next_block function Relevant PG commit 041b96802efa33d2bc9456f2ad946976b92b5ae1 https://github.com/postgres/postgres/commit/041b96802efa33d2bc9456f2ad946976b92b5ae1 - Define ObjectClass for PG17+ only since it's removed Relevant PG commit: 89e5ef7e21812916c9cf9fcf56e45f0f74034656 https://github.com/postgres/postgres/commit/89e5ef7e21812916c9cf9fcf56e45f0f74034656 - Remove ReorderBufferTupleBuf structure. Relevant PG commit: 08e6344fd6423210b339e92c069bb979ba4e7cd6 https://github.com/postgres/postgres/commit/08e6344fd6423210b339e92c069bb979ba4e7cd6 - Define colliculocale and daticulocale since they have been renamed Relevant PG commit: f696c0cd5f299f1b51e214efc55a22a782cc175d https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d - makeStringConst defined in PG17 Relevant PG commit: de3600452b61d1bc3967e9e37e86db8956c8f577 https://github.com/postgres/postgres/commit/de3600452b61d1bc3967e9e37e86db8956c8f577 - RangeVarCallbackOwnsTable was replaced by RangeVarCallbackMaintainsTable Relevant PG commit: ecb0fd33720fab91df1207e85704f382f55e1eb7 https://github.com/postgres/postgres/commit/ecb0fd33720fab91df1207e85704f382f55e1eb7 - attstattarget is nullable, define pg compatible functions for it Relevant PG commit: 4f622503d6de975ac87448aea5cea7de4bc140d5 https://github.com/postgres/postgres/commit/4f622503d6de975ac87448aea5cea7de4bc140d5 - stxstattarget is nullable in PG17, write compat functions for it Relevant PG commit: 012460ee93c304fbc7220e5b55d9d0577fc766ab https://github.com/postgres/postgres/commit/012460ee93c304fbc7220e5b55d9d0577fc766ab - Use ResourceOwner to track WaitEventSet in PG17 Relevant PG commit: 50c67c2019ab9ade8aa8768bfe604cd802fe8591 https://github.com/postgres/postgres/commit/50c67c2019ab9ade8aa8768bfe604cd802fe8591 - getIdentitySequence now uses Relation instead of relation_id Relevant PG commit: 509199587df73f06eda898ae13284292f4ae573a https://github.com/postgres/postgres/commit/509199587df73f06eda898ae13284292f4ae573a - Remove no-op tuplestore_donestoring function Relevant PG commit: 75680c3d805e2323cd437ac567f0677fdfc7b680 https://github.com/postgres/postgres/commit/75680c3d805e2323cd437ac567f0677fdfc7b680 - MergeAction can have 3 merge kinds (now enum) in PG17, write compat Relevant PG commit: 0294df2f1f842dfb0eed79007b21016f486a3c6c https://github.com/postgres/postgres/commit/0294df2f1f842dfb0eed79007b21016f486a3c6c - EXPLAIN (MEMORY) is added, make changes to ExplainOnePlan Relevant PG commit: 5de890e3610d5a12cdaea36413d967cf5c544e20 https://github.com/postgres/postgres/commit/5de890e3610d5a12cdaea36413d967cf5c544e20 - LIMIT_OPTION_DEFAULT has been removed as it's useless, use LIMIT_OPTION_COUNT Relevant PG commit: a6be0600ac3b71dda8277ab0fcbe59ee101ac1ce https://github.com/postgres/postgres/commit/a6be0600ac3b71dda8277ab0fcbe59ee101ac1ce - write compat for create_foreignscan_path bcs of more arguments in PG17 Relevant PG commit: 9e9931d2bf40e2fea447d779c2e133c2c1256ef3 https://github.com/postgres/postgres/commit/9e9931d2bf40e2fea447d779c2e133c2c1256ef3 - pgprocno and lxid have been combined into a struct in PGPROC Relevant PG commits: 28f3915b73f75bd1b50ba070f56b34241fe53fd1 https://github.com/postgres/postgres/commit/28f3915b73f75bd1b50ba070f56b34241fe53fd1 ab355e3a88de745607f6dd4c21f0119b5c68f2ad https://github.com/postgres/postgres/commit/ab355e3a88de745607f6dd4c21f0119b5c68f2ad 024c521117579a6d356050ad3d78fdc95e44eefa https://github.com/postgres/postgres/commit/024c521117579a6d356050ad3d78fdc95e44eefa - Simplify CitusNewNode (#7434) postgres refactored newNode() in PG 17, the main point for doing this is the original tricks is no longer neccessary for modern compilers[1]. This does the same for Citus. This should have no backward compatibility issues since it just replaces palloc0fast with palloc0. This is good for forward compatibility since palloc0fast no longer exists in PG 17. [1] https://www.postgresql.org/message-id/b51f1fa7-7e6a-4ecc-936d-90a8a1659e7c@iki.fi (cherry picked from commit 4b295cc) --- src/backend/columnar/columnar_customscan.c | 5 - src/backend/columnar/columnar_tableam.c | 7 +- src/backend/distributed/cdc/cdc_decoder.c | 71 ++++++++++ src/backend/distributed/commands/collation.c | 16 +-- src/backend/distributed/commands/role.c | 7 + src/backend/distributed/commands/statistics.c | 11 +- .../connection/connection_management.c | 3 +- .../distributed/connection/remote_commands.c | 2 +- .../distributed/deparser/citus_ruleutils.c | 18 ++- .../deparser/deparse_statistics_stmts.c | 5 +- .../distributed/executor/adaptive_executor.c | 2 +- .../distributed/executor/query_stats.c | 3 - .../distributed/planner/merge_planner.c | 2 +- .../distributed/planner/multi_explain.c | 83 +++++++++++ .../planner/multi_physical_planner.c | 2 +- .../shardsplit/shardsplit_decoder.c | 97 +++++++++++++ src/backend/distributed/test/fake_am.c | 7 +- src/backend/distributed/test/fake_fdw.c | 10 +- .../distributed/transaction/backend_data.c | 13 +- .../distributed/transaction/lock_graph.c | 6 +- .../distributed/utils/citus_nodefuncs.c | 3 - .../worker/worker_data_fetch_protocol.c | 3 +- src/include/distributed/citus_nodes.h | 39 ++--- src/include/pg_version_compat.h | 133 ++++++++++++++++++ 24 files changed, 472 insertions(+), 76 deletions(-) diff --git a/src/backend/columnar/columnar_customscan.c b/src/backend/columnar/columnar_customscan.c index 698734bc9..c78485c66 100644 --- a/src/backend/columnar/columnar_customscan.c +++ b/src/backend/columnar/columnar_customscan.c @@ -1924,11 +1924,6 @@ ColumnarScan_EndCustomScan(CustomScanState *node) */ TableScanDesc scanDesc = node->ss.ss_currentScanDesc; - /* - * Free the exprcontext - */ - ExecFreeExprContext(&node->ss.ps); - /* * clean out the tuple table */ diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 6d31c2779..0e6c423c2 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -1424,8 +1424,13 @@ ConditionalLockRelationWithTimeout(Relation rel, LOCKMODE lockMode, int timeout, static bool -columnar_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, +columnar_scan_analyze_next_block(TableScanDesc scan, +#if PG_VERSION_NUM >= PG_VERSION_17 + ReadStream *stream) +#else + BlockNumber blockno, BufferAccessStrategy bstrategy) +#endif { /* * Our access method is not pages based, i.e. tuples are not confined diff --git a/src/backend/distributed/cdc/cdc_decoder.c b/src/backend/distributed/cdc/cdc_decoder.c index cf9f4963b..1e71a82a1 100644 --- a/src/backend/distributed/cdc/cdc_decoder.c +++ b/src/backend/distributed/cdc/cdc_decoder.c @@ -22,6 +22,8 @@ #include "utils/rel.h" #include "utils/typcache.h" +#include "pg_version_constants.h" + PG_MODULE_MAGIC; extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); @@ -435,6 +437,74 @@ TranslateChangesIfSchemaChanged(Relation sourceRelation, Relation targetRelation return; } +#if PG_VERSION_NUM >= PG_VERSION_17 + + /* Check the ReorderBufferChange's action type and handle them accordingly.*/ + switch (change->action) + { + case REORDER_BUFFER_CHANGE_INSERT: + { + /* For insert action, only new tuple should always be translated*/ + HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; + HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( + sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); + change->data.tp.newtuple = targetRelationNewTuple; + break; + } + + /* + * For update changes both old and new tuples need to be translated for target relation + * if the REPLICA IDENTITY is set to FULL. Otherwise, only the new tuple needs to be + * translated for target relation. + */ + case REORDER_BUFFER_CHANGE_UPDATE: + { + /* For update action, new tuple should always be translated*/ + /* Get the new tuple from the ReorderBufferChange, and translate it to target relation. */ + HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; + HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( + sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); + change->data.tp.newtuple = targetRelationNewTuple; + + /* + * Format oldtuple according to the target relation. If the column values of replica + * identiy change, then the old tuple is non-null and needs to be formatted according + * to the target relation schema. + */ + if (change->data.tp.oldtuple != NULL) + { + HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; + HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( + sourceRelationOldTuple, + sourceRelationDesc, + targetRelationDesc); + + change->data.tp.oldtuple = targetRelationOldTuple; + } + break; + } + + case REORDER_BUFFER_CHANGE_DELETE: + { + /* For delete action, only old tuple should be translated*/ + HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; + HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( + sourceRelationOldTuple, + sourceRelationDesc, + targetRelationDesc); + + change->data.tp.oldtuple = targetRelationOldTuple; + break; + } + + default: + { + /* Do nothing for other action types. */ + break; + } + } +#else + /* Check the ReorderBufferChange's action type and handle them accordingly.*/ switch (change->action) { @@ -499,4 +569,5 @@ TranslateChangesIfSchemaChanged(Relation sourceRelation, Relation targetRelation break; } } +#endif } diff --git a/src/backend/distributed/commands/collation.c b/src/backend/distributed/commands/collation.c index 5ce3d1436..1a8c211f9 100644 --- a/src/backend/distributed/commands/collation.c +++ b/src/backend/distributed/commands/collation.c @@ -77,7 +77,7 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati * ICU-related field. Only the libc-related fields or the ICU-related field * is set, never both. */ - char *colliculocale; + char *colllocale; bool isnull; Datum datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collcollate, @@ -101,17 +101,17 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati collctype = NULL; } - datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_colliculocale, &isnull); + datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_colllocale, &isnull); if (!isnull) { - colliculocale = TextDatumGetCString(datum); + colllocale = TextDatumGetCString(datum); } else { - colliculocale = NULL; + colllocale = NULL; } - Assert((collcollate && collctype) || colliculocale); + Assert((collcollate && collctype) || colllocale); #else /* @@ -147,12 +147,12 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati *quotedCollationName, providerString); #if PG_VERSION_NUM >= PG_VERSION_15 - if (colliculocale) + if (colllocale) { appendStringInfo(&collationNameDef, ", locale = %s", - quote_literal_cstr(colliculocale)); - pfree(colliculocale); + quote_literal_cstr(colllocale)); + pfree(colllocale); } else { diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 05959ca3e..dd45c98d8 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -71,7 +71,9 @@ static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); static char * GetDatabaseNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); +#if PG_VERSION_NUM < PG_VERSION_17 static Node * makeStringConst(char *str, int location); +#endif static Node * makeIntConst(int val, int location); static Node * makeFloatConst(char *str, int location); static const char * WrapQueryInAlterRoleIfExistsCall(const char *query, RoleSpec *role); @@ -949,6 +951,8 @@ PreprocessCreateRoleStmt(Node *node, const char *queryString, } +#if PG_VERSION_NUM < PG_VERSION_17 + /* * makeStringConst creates a Const Node that stores a given string * @@ -972,6 +976,9 @@ makeStringConst(char *str, int location) } +#endif + + /* * makeIntConst creates a Const Node that stores a given integer * diff --git a/src/backend/distributed/commands/statistics.c b/src/backend/distributed/commands/statistics.c index 45d79afe4..b43f6335e 100644 --- a/src/backend/distributed/commands/statistics.c +++ b/src/backend/distributed/commands/statistics.c @@ -651,14 +651,15 @@ GetAlterIndexStatisticsCommands(Oid indexOid) } Form_pg_attribute targetAttr = (Form_pg_attribute) GETSTRUCT(attTuple); - if (targetAttr->attstattarget != DEFAULT_STATISTICS_TARGET) + int32 targetAttstattarget = getAttstattarget_compat(attTuple); + if (targetAttstattarget != DEFAULT_STATISTICS_TARGET) { char *indexNameWithSchema = generate_qualified_relation_name(indexOid); char *command = GenerateAlterIndexColumnSetStatsCommand(indexNameWithSchema, targetAttr->attnum, - targetAttr->attstattarget); + targetAttstattarget); alterIndexStatisticsCommandList = lappend(alterIndexStatisticsCommandList, @@ -773,9 +774,10 @@ CreateAlterCommandIfTargetNotDefault(Oid statsOid) } Form_pg_statistic_ext statisticsForm = (Form_pg_statistic_ext) GETSTRUCT(tup); + int16 currentStxstattarget = getStxstattarget_compat(tup); ReleaseSysCache(tup); - if (statisticsForm->stxstattarget == -1) + if (currentStxstattarget == -1) { return NULL; } @@ -785,7 +787,8 @@ CreateAlterCommandIfTargetNotDefault(Oid statsOid) char *schemaName = get_namespace_name(statisticsForm->stxnamespace); char *statName = NameStr(statisticsForm->stxname); - alterStatsStmt->stxstattarget = statisticsForm->stxstattarget; + alterStatsStmt->stxstattarget = getAlterStatsStxstattarget_compat( + currentStxstattarget); alterStatsStmt->defnames = list_make2(makeString(schemaName), makeString(statName)); return DeparseAlterStatisticsStmt((Node *) alterStatsStmt); diff --git a/src/backend/distributed/connection/connection_management.c b/src/backend/distributed/connection/connection_management.c index a8d8bad8a..4787d8f2f 100644 --- a/src/backend/distributed/connection/connection_management.c +++ b/src/backend/distributed/connection/connection_management.c @@ -866,7 +866,8 @@ WaitEventSetFromMultiConnectionStates(List *connections, int *waitCount) *waitCount = 0; } - WaitEventSet *waitEventSet = CreateWaitEventSet(CurrentMemoryContext, eventSetSize); + WaitEventSet *waitEventSet = CreateWaitEventSet(WaitEventSetTracker_compat, + eventSetSize); EnsureReleaseResource((MemoryContextCallbackFunction) (&FreeWaitEventSet), waitEventSet); diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index 7a9e0601d..c9860c061 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -1130,7 +1130,7 @@ BuildWaitEventSet(MultiConnection **allConnections, int totalConnectionCount, /* allocate pending connections + 2 for the signal latch and postmaster death */ /* (CreateWaitEventSet makes room for pgwin32_signal_event automatically) */ - WaitEventSet *waitEventSet = CreateWaitEventSet(CurrentMemoryContext, + WaitEventSet *waitEventSet = CreateWaitEventSet(WaitEventSetTracker_compat, pendingConnectionCount + 2); for (int connectionIndex = 0; connectionIndex < pendingConnectionCount; diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 3b387799b..530f6e720 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -395,7 +395,8 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults if (attributeForm->attidentity && includeIdentityDefaults) { bool missing_ok = false; - Oid seqOid = getIdentitySequence(RelationGetRelid(relation), + Oid seqOid = getIdentitySequence(identitySequenceRelation_compat( + relation), attributeForm->attnum, missing_ok); if (includeIdentityDefaults == INCLUDE_IDENTITY) @@ -738,7 +739,18 @@ pg_get_tablecolumnoptionsdef_string(Oid tableRelationId) * If the user changed the column's statistics target, create * alter statement and add statement to a list for later processing. */ - if (attributeForm->attstattarget >= 0) + HeapTuple atttuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(tableRelationId), + Int16GetDatum(attributeForm->attnum)); + if (!HeapTupleIsValid(atttuple)) + { + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + attributeForm->attnum, tableRelationId); + } + + int32 targetAttstattarget = getAttstattarget_compat(atttuple); + ReleaseSysCache(atttuple); + if (targetAttstattarget >= 0) { StringInfoData statement = { NULL, 0, 0, 0 }; initStringInfo(&statement); @@ -746,7 +758,7 @@ pg_get_tablecolumnoptionsdef_string(Oid tableRelationId) appendStringInfo(&statement, "ALTER COLUMN %s ", quote_identifier(attributeName)); appendStringInfo(&statement, "SET STATISTICS %d", - attributeForm->attstattarget); + targetAttstattarget); columnOptionList = lappend(columnOptionList, statement.data); } diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 4d7211939..79be835b9 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -177,8 +177,9 @@ AppendAlterStatisticsSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) static void AppendAlterStatisticsStmt(StringInfo buf, AlterStatsStmt *stmt) { - appendStringInfo(buf, "ALTER STATISTICS %s SET STATISTICS %d", NameListToQuotedString( - stmt->defnames), stmt->stxstattarget); + appendStringInfo(buf, "ALTER STATISTICS %s SET STATISTICS %d", + NameListToQuotedString(stmt->defnames), + getIntStxstattarget_compat(stmt->stxstattarget)); } diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c index f7e7c2337..9606cd724 100644 --- a/src/backend/distributed/executor/adaptive_executor.c +++ b/src/backend/distributed/executor/adaptive_executor.c @@ -4740,7 +4740,7 @@ BuildWaitEventSet(List *sessionList) int eventSetSize = GetEventSetSize(sessionList); WaitEventSet *waitEventSet = - CreateWaitEventSet(CurrentMemoryContext, eventSetSize); + CreateWaitEventSet(WaitEventSetTracker_compat, eventSetSize); WorkerSession *session = NULL; foreach_declared_ptr(session, sessionList) diff --git a/src/backend/distributed/executor/query_stats.c b/src/backend/distributed/executor/query_stats.c index f37a99bbf..ce6179b96 100644 --- a/src/backend/distributed/executor/query_stats.c +++ b/src/backend/distributed/executor/query_stats.c @@ -759,9 +759,6 @@ citus_query_stats(PG_FUNCTION_ARGS) LWLockRelease(queryStats->lock); - /* clean up and return the tuplestore */ - tuplestore_donestoring(tupstore); - return (Datum) 0; } diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 66eaf71da..8048002e0 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -1475,7 +1475,7 @@ FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) foreach_declared_ptr(action, query->mergeActionList) { /* Skip MATCHED clause as INSERTS are not allowed in it */ - if (action->matched) + if (matched_compat(action)) { continue; } diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 86a9f2a64..3f0120dae 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -375,6 +375,21 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) BufferUsage bufusage_start, bufusage; +#if PG_VERSION_NUM >= PG_VERSION_17 + MemoryContextCounters mem_counters; + MemoryContext planner_ctx = NULL; + MemoryContext saved_ctx = NULL; + + if (es->memory) + { + /* copy paste from postgres code */ + planner_ctx = AllocSetContextCreate(CurrentMemoryContext, + "explain analyze planner context", + ALLOCSET_DEFAULT_SIZES); + saved_ctx = MemoryContextSwitchTo(planner_ctx); + } +#endif + if (es->buffers) { bufusage_start = pgBufferUsage; @@ -432,8 +447,20 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) ExplainOpenGroup("PlannedStmt", "PlannedStmt", false, es); +#if PG_VERSION_NUM >= PG_VERSION_17 + if (es->memory) + { + MemoryContextSwitchTo(saved_ctx); + MemoryContextMemConsumed(planner_ctx, &mem_counters); + } + + ExplainOnePlan(plan, into, es, queryString, params, NULL, &planduration, + (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL)); +#else ExplainOnePlan(plan, into, es, queryString, params, NULL, &planduration, (es->buffers ? &bufusage : NULL)); +#endif ExplainCloseGroup("PlannedStmt", "PlannedStmt", false, es); ExplainCloseGroup("Subplan", NULL, true, es); @@ -1253,6 +1280,21 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, BufferUsage bufusage_start, bufusage; +#if PG_VERSION_NUM >= PG_VERSION_17 + MemoryContextCounters mem_counters; + MemoryContext planner_ctx = NULL; + MemoryContext saved_ctx = NULL; + + if (es->memory) + { + /* copy paste from postgres code */ + planner_ctx = AllocSetContextCreate(CurrentMemoryContext, + "explain analyze planner context", + ALLOCSET_DEFAULT_SIZES); + saved_ctx = MemoryContextSwitchTo(planner_ctx); + } +#endif + if (es->buffers) { bufusage_start = pgBufferUsage; @@ -1286,9 +1328,23 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } +#if PG_VERSION_NUM >= PG_VERSION_17 + if (es->memory) + { + MemoryContextSwitchTo(saved_ctx); + MemoryContextMemConsumed(planner_ctx, &mem_counters); + } + + /* run it (if needed) and produce output */ + ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + &planduration, (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL)); +#else + /* run it (if needed) and produce output */ ExplainOnePlan(plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL)); +#endif } @@ -1701,6 +1757,21 @@ ExplainOneQuery(Query *query, int cursorOptions, BufferUsage bufusage_start, bufusage; +#if PG_VERSION_NUM >= PG_VERSION_17 + MemoryContextCounters mem_counters; + MemoryContext planner_ctx = NULL; + MemoryContext saved_ctx = NULL; + + if (es->memory) + { + /* copy paste from postgres code */ + planner_ctx = AllocSetContextCreate(CurrentMemoryContext, + "explain analyze planner context", + ALLOCSET_DEFAULT_SIZES); + saved_ctx = MemoryContextSwitchTo(planner_ctx); + } +#endif + if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); @@ -1718,9 +1789,21 @@ ExplainOneQuery(Query *query, int cursorOptions, BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } +#if PG_VERSION_NUM >= PG_VERSION_17 + if (es->memory) + { + MemoryContextSwitchTo(saved_ctx); + MemoryContextMemConsumed(planner_ctx, &mem_counters); + } + /* run it (if needed) and produce output */ + ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + &planduration, (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL)); +#else /* run it (if needed) and produce output */ ExplainOnePlan(plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL)); +#endif } } diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 2fb5b26e3..dee3464cf 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -547,7 +547,7 @@ BuildJobQuery(MultiNode *multiNode, List *dependentJobList) List *sortClauseList = NIL; Node *limitCount = NULL; Node *limitOffset = NULL; - LimitOption limitOption = LIMIT_OPTION_DEFAULT; + LimitOption limitOption = LIMIT_OPTION_COUNT; Node *havingQual = NULL; bool hasDistinctOn = false; List *distinctClause = NIL; diff --git a/src/backend/distributed/shardsplit/shardsplit_decoder.c b/src/backend/distributed/shardsplit/shardsplit_decoder.c index 841fa89cd..bcd25ce2e 100644 --- a/src/backend/distributed/shardsplit/shardsplit_decoder.c +++ b/src/backend/distributed/shardsplit/shardsplit_decoder.c @@ -14,6 +14,8 @@ #include "utils/lsyscache.h" #include "utils/typcache.h" +#include "pg_version_constants.h" + #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/shardinterval_utils.h" @@ -134,6 +136,43 @@ shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, } Oid targetRelationOid = InvalidOid; + +#if PG_VERSION_NUM >= PG_VERSION_17 + switch (change->action) + { + case REORDER_BUFFER_CHANGE_INSERT: + { + HeapTuple newTuple = change->data.tp.newtuple; + targetRelationOid = FindTargetRelationOid(relation, newTuple, + replicationSlotName); + break; + } + + /* updating non-partition column value */ + case REORDER_BUFFER_CHANGE_UPDATE: + { + HeapTuple newTuple = change->data.tp.newtuple; + targetRelationOid = FindTargetRelationOid(relation, newTuple, + replicationSlotName); + break; + } + + case REORDER_BUFFER_CHANGE_DELETE: + { + HeapTuple oldTuple = change->data.tp.oldtuple; + targetRelationOid = FindTargetRelationOid(relation, oldTuple, + replicationSlotName); + + break; + } + + /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ + default: + ereport(ERROR, errmsg( + "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", + change->action)); + } +#else switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: @@ -168,6 +207,7 @@ shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } +#endif /* Current replication slot is not responsible for handling the change */ if (targetRelationOid == InvalidOid) @@ -185,6 +225,62 @@ shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, TupleDesc targetRelationDesc = RelationGetDescr(targetRelation); if (sourceRelationDesc->natts > targetRelationDesc->natts) { +#if PG_VERSION_NUM >= PG_VERSION_17 + switch (change->action) + { + case REORDER_BUFFER_CHANGE_INSERT: + { + HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; + HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( + sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); + + change->data.tp.newtuple = targetRelationNewTuple; + break; + } + + case REORDER_BUFFER_CHANGE_UPDATE: + { + HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; + HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( + sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); + + change->data.tp.newtuple = targetRelationNewTuple; + + /* + * Format oldtuple according to the target relation. If the column values of replica + * identiy change, then the old tuple is non-null and needs to be formatted according + * to the target relation schema. + */ + if (change->data.tp.oldtuple != NULL) + { + HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; + HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( + sourceRelationOldTuple, + sourceRelationDesc, + targetRelationDesc); + + change->data.tp.oldtuple = targetRelationOldTuple; + } + break; + } + + case REORDER_BUFFER_CHANGE_DELETE: + { + HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; + HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( + sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); + + change->data.tp.oldtuple = targetRelationOldTuple; + break; + } + + /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ + default: + ereport(ERROR, errmsg( + "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", + change->action)); + } +#else switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: @@ -239,6 +335,7 @@ shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } +#endif } pgOutputPluginChangeCB(ctx, txn, targetRelation, change); diff --git a/src/backend/distributed/test/fake_am.c b/src/backend/distributed/test/fake_am.c index cff124961..928051942 100644 --- a/src/backend/distributed/test/fake_am.c +++ b/src/backend/distributed/test/fake_am.c @@ -372,8 +372,13 @@ fake_vacuum(Relation onerel, VacuumParams *params, static bool -fake_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, +fake_scan_analyze_next_block(TableScanDesc scan, +#if PG_VERSION_NUM >= PG_VERSION_17 + ReadStream *stream) +#else + BlockNumber blockno, BufferAccessStrategy bstrategy) +#endif { /* we don't support analyze, so return false */ return false; diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index 585e61d41..90b205b1e 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -29,7 +29,7 @@ #include "optimizer/restrictinfo.h" #include "utils/palloc.h" -#include "pg_version_constants.h" +#include "pg_version_compat.h" /* local function forward declarations */ static void FakeGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, @@ -91,9 +91,11 @@ FakeGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) Cost startup_cost = 0; Cost total_cost = startup_cost + baserel->rows; - add_path(baserel, (Path *) create_foreignscan_path(root, baserel, NULL, baserel->rows, - startup_cost, total_cost, NIL, - NULL, NULL, NIL)); + add_path(baserel, (Path *) create_foreignscan_path_compat(root, baserel, NULL, + baserel->rows, + startup_cost, total_cost, + NIL, + NULL, NULL, NIL, NIL)); } diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index a0584bde8..f23d58bd0 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -33,7 +33,7 @@ #include "storage/spin.h" #include "utils/timestamp.h" -#include "pg_version_constants.h" +#include "pg_version_compat.h" #include "distributed/backend_data.h" #include "distributed/connection_management.h" @@ -700,7 +700,7 @@ InitializeBackendData(const char *applicationName) uint64 gpid = ExtractGlobalPID(applicationName); - MyBackendData = &backendManagementShmemData->backends[MyProc->pgprocno]; + MyBackendData = &backendManagementShmemData->backends[getProcNo_compat(MyProc)]; Assert(MyBackendData); @@ -1174,11 +1174,11 @@ CurrentDistributedTransactionNumber(void) void GetBackendDataForProc(PGPROC *proc, BackendData *result) { - int pgprocno = proc->pgprocno; + int pgprocno = getProcNo_compat(proc); if (proc->lockGroupLeader != NULL) { - pgprocno = proc->lockGroupLeader->pgprocno; + pgprocno = getProcNo_compat(proc->lockGroupLeader); } BackendData *backendData = &backendManagementShmemData->backends[pgprocno]; @@ -1198,7 +1198,8 @@ GetBackendDataForProc(PGPROC *proc, BackendData *result) void CancelTransactionDueToDeadlock(PGPROC *proc) { - BackendData *backendData = &backendManagementShmemData->backends[proc->pgprocno]; + BackendData *backendData = &backendManagementShmemData->backends[getProcNo_compat( + proc)]; /* backend might not have used citus yet and thus not initialized backend data */ if (!backendData) @@ -1330,7 +1331,7 @@ ActiveDistributedTransactionNumbers(void) LocalTransactionId GetMyProcLocalTransactionId(void) { - return MyProc->lxid; + return getLxid_compat(MyProc); } diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index a224c29e1..dadcfe0a4 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -23,6 +23,8 @@ #include "utils/hsearch.h" #include "utils/timestamp.h" +#include "pg_version_compat.h" + #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" @@ -993,7 +995,7 @@ AllocWaitEdge(WaitGraph *waitGraph) static void AddProcToVisit(PROCStack *remaining, PGPROC *proc) { - if (remaining->procAdded[proc->pgprocno]) + if (remaining->procAdded[getProcNo_compat(proc)]) { return; } @@ -1001,7 +1003,7 @@ AddProcToVisit(PROCStack *remaining, PGPROC *proc) Assert(remaining->procCount < TotalProcCount()); remaining->procs[remaining->procCount++] = proc; - remaining->procAdded[proc->pgprocno] = true; + remaining->procAdded[getProcNo_compat(proc)] = true; } diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index 0b03926f8..076e8ce6a 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -53,9 +53,6 @@ static const char *CitusNodeTagNamesD[] = { const char **CitusNodeTagNames = CitusNodeTagNamesD; -/* support for CitusNewNode() macro */ -CitusNode *newCitusNodeMacroHolder; - /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_extradata_container); diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index d2b60aa50..0370001ee 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -170,7 +170,8 @@ worker_adjust_identity_column_seq_ranges(PG_FUNCTION_ARGS) if (attributeForm->attidentity) { - Oid sequenceOid = getIdentitySequence(tableRelationId, + Oid sequenceOid = getIdentitySequence(identitySequenceRelation_compat( + tableRelation), attributeForm->attnum, missingSequenceOk); diff --git a/src/include/distributed/citus_nodes.h b/src/include/distributed/citus_nodes.h index 888133a89..16df367aa 100644 --- a/src/include/distributed/citus_nodes.h +++ b/src/include/distributed/citus_nodes.h @@ -92,38 +92,21 @@ CitusNodeTagI(Node *node) return ((CitusNode*)(node))->citus_tag; } -/* - * Postgres's nodes/nodes.h has more information on why we do this. - */ -#ifdef __GNUC__ /* Citus variant of newNode(), don't use directly. */ -#define CitusNewNode(size, tag) \ -({ CitusNode *_result; \ - AssertMacro((size) >= sizeof(CitusNode)); /* need the tag, at least */ \ - _result = (CitusNode *) palloc0fast(size); \ - _result->extensible.type = T_ExtensibleNode; \ - _result->extensible.extnodename = CitusNodeTagNames[tag - CITUS_NODE_TAG_START]; \ - _result->citus_tag =(int) (tag); \ - _result; \ -}) +static inline CitusNode * +CitusNewNode(size_t size, CitusNodeTag tag) +{ + CitusNode *result; -#else - -extern CitusNode *newCitusNodeMacroHolder; - -#define CitusNewNode(size, tag) \ -( \ - AssertMacro((size) >= sizeof(CitusNode)), /* need the tag, at least */ \ - newCitusNodeMacroHolder = (CitusNode *) palloc0fast(size), \ - newCitusNodeMacroHolder->extensible.type = T_ExtensibleNode, \ - newCitusNodeMacroHolder->extensible.extnodename = CitusNodeTagNames[tag - CITUS_NODE_TAG_START], \ - newCitusNodeMacroHolder->citus_tag =(int) (tag), \ - newCitusNodeMacroHolder \ -) - -#endif + Assert(size >= sizeof(CitusNode)); /* need the ExtensibleNode and the tag, at least */ + result = (CitusNode *) palloc0(size); + result->extensible.type = T_ExtensibleNode; + result->extensible.extnodename = CitusNodeTagNames[tag - CITUS_NODE_TAG_START]; + result->citus_tag = (int) (tag); + return result; +} /* * IsA equivalent that compares node tags, including Citus-specific nodes. diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 4e874e2ee..80c4e9d3d 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -13,6 +13,139 @@ #include "pg_version_constants.h" +#if PG_VERSION_NUM >= PG_VERSION_17 + +#include "catalog/pg_am.h" +#include "catalog/pg_auth_members.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_class.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_database.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_parameter_acl.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_publication.h" +#include "catalog/pg_tablespace.h" +#include "catalog/pg_transform.h" +#include "catalog/pg_ts_config.h" +#include "catalog/pg_ts_dict.h" +#include "catalog/pg_ts_template.h" +#include "catalog/pg_type.h" + +typedef int ObjectClass; +#define getObjectClass(a) a->classId +#define LAST_OCLASS TransformRelationId +#define OCLASS_ROLE AuthIdRelationId +#define OCLASS_DATABASE DatabaseRelationId +#define OCLASS_TBLSPACE TableSpaceRelationId +#define OCLASS_PARAMETER_ACL ParameterAclRelationId +#define OCLASS_ROLE_MEMBERSHIP AuthMemRelationId +#define OCLASS_CLASS RelationRelationId +#define OCLASS_COLLATION CollationRelationId +#define OCLASS_CONSTRAINT ConstraintRelationId +#define OCLASS_PROC ProcedureRelationId +#define OCLASS_PUBLICATION PublicationRelationId +#define OCLASS_SCHEMA NamespaceRelationId +#define OCLASS_TSCONFIG TSConfigRelationId +#define OCLASS_TSDICT TSDictionaryRelationId +#define OCLASS_TYPE TypeRelationId +#define OCLASS_EXTENSION ExtensionRelationId +#define OCLASS_FOREIGN_SERVER ForeignServerRelationId +#define OCLASS_AM AccessMethodRelationId +#define OCLASS_TSTEMPLATE TSTemplateRelationId + +#include "commands/tablecmds.h" + +static inline void +RangeVarCallbackOwnsTable(const RangeVar *relation, + Oid relId, Oid oldRelId, void *arg) +{ + return RangeVarCallbackMaintainsTable(relation, relId, oldRelId, arg); +} + + +#include "catalog/pg_attribute.h" +#include "utils/syscache.h" + +static inline int +getAttstattarget_compat(HeapTuple attTuple) +{ + bool isnull; + Datum dat = SysCacheGetAttr(ATTNUM, attTuple, + Anum_pg_attribute_attstattarget, &isnull); + return (isnull ? -1 : DatumGetInt16(dat)); +} + + +#include "catalog/pg_statistic_ext.h" + +static inline int +getStxstattarget_compat(HeapTuple tup) +{ + bool isnull; + Datum dat = SysCacheGetAttr(STATEXTOID, tup, + Anum_pg_statistic_ext_stxstattarget, &isnull); + return (isnull ? -1 : DatumGetInt16(dat)); +} + + +#define getAlterStatsStxstattarget_compat(a) ((Node *) makeInteger(a)) +#define getIntStxstattarget_compat(a) (intVal(a)) + +#define WaitEventSetTracker_compat CurrentResourceOwner + +#define identitySequenceRelation_compat(a) (a) + +#define matched_compat(a) (a->matchKind == MERGE_WHEN_MATCHED) + +#define create_foreignscan_path_compat(a, b, c, d, e, f, g, h, i, j, \ + k) create_foreignscan_path(a, b, c, d, e, f, g, h, \ + i, j, k) + +#define getProcNo_compat(a) (a->vxid.procNumber) +#define getLxid_compat(a) (a->vxid.lxid) + +#else + +#define Anum_pg_collation_colllocale Anum_pg_collation_colliculocale + +#include "access/htup_details.h" +static inline int +getAttstattarget_compat(HeapTuple attTuple) +{ + return ((Form_pg_attribute) GETSTRUCT(attTuple))->attstattarget; +} + + +#include "catalog/pg_statistic_ext.h" +static inline int +getStxstattarget_compat(HeapTuple tup) +{ + return ((Form_pg_statistic_ext) GETSTRUCT(tup))->stxstattarget; +} + + +#define getAlterStatsStxstattarget_compat(a) (a) +#define getIntStxstattarget_compat(a) (a) + +#define WaitEventSetTracker_compat CurrentMemoryContext + +#define identitySequenceRelation_compat(a) (RelationGetRelid(a)) + +#define matched_compat(a) (a->matched) + +#define create_foreignscan_path_compat(a, b, c, d, e, f, g, h, i, j, \ + k) create_foreignscan_path(a, b, c, d, e, f, g, h, \ + i, k) + +#define getProcNo_compat(a) (a->pgprocno) +#define getLxid_compat(a) (a->lxid) + +#endif + #if PG_VERSION_NUM >= PG_VERSION_16 #include "utils/guc_tables.h" From c0a5f5c78c051bc622c75ea43dd0db36e274c8ef Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:55:10 +0300 Subject: [PATCH 055/155] PG17 compatibility: ruleutils (#7725) PG17 compatibility - Part 2 https://github.com/citusdata/citus/pull/7699 was the first PG17 compatibility PR merged to main branch, which provided ONLY successful Citus compilation with PG17.0. This PR, consider it as Part 2, provides ruleutils changes for PG17. Ruleutils changes is the first thing we should merge, after successful build. It's the core for deparsing logic in Citus. # Question: How do we add ruleutils changes? - We add a new ruleutils file specific to PG17. - We keep track of the changes in Postgres's ruleutils file from here https://github.com/postgres/postgres/commits/REL_17_0/src/backend/utils/adt/ruleutils.c - Per each commit in that history that belongs only to 17.0, we add the relevant changes to static functions to our ruleutils file for PG17. It's like a manual commit copying. # Check the PR's commits for detailed steps https://github.com/citusdata/citus/pull/7725/commits --- .gitattributes | 1 + .../distributed/deparser/ruleutils_17.c | 9922 +++++++++++++++++ src/include/pg_version_constants.h | 1 + 3 files changed, 9924 insertions(+) create mode 100644 src/backend/distributed/deparser/ruleutils_17.c diff --git a/.gitattributes b/.gitattributes index 42f42cd25..c7c03e1ef 100644 --- a/.gitattributes +++ b/.gitattributes @@ -29,6 +29,7 @@ src/backend/distributed/deparser/ruleutils_13.c -citus-style src/backend/distributed/deparser/ruleutils_14.c -citus-style src/backend/distributed/deparser/ruleutils_15.c -citus-style src/backend/distributed/deparser/ruleutils_16.c -citus-style +src/backend/distributed/deparser/ruleutils_17.c -citus-style src/backend/distributed/commands/index_pg_source.c -citus-style src/include/distributed/citus_nodes.h -citus-style diff --git a/src/backend/distributed/deparser/ruleutils_17.c b/src/backend/distributed/deparser/ruleutils_17.c new file mode 100644 index 000000000..f0710e684 --- /dev/null +++ b/src/backend/distributed/deparser/ruleutils_17.c @@ -0,0 +1,9922 @@ +/*------------------------------------------------------------------------- + * + * ruleutils_16.c + * Functions to convert stored expressions/querytrees back to + * source text + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/distributed/deparser/ruleutils_16.c + * + * This needs to be closely in sync with the core code. + *------------------------------------------------------------------------- + */ +#include "pg_version_constants.h" + +#include "pg_config.h" + +#if (PG_VERSION_NUM >= PG_VERSION_17) && (PG_VERSION_NUM < PG_VERSION_18) + +#include "postgres.h" + +#include +#include +#include + +#include "access/amapi.h" +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/table.h" +#include "catalog/pg_aggregate.h" +#include "catalog/pg_am.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_language.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_partitioned_table.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/extension.h" +#include "commands/tablespace.h" +#include "common/keywords.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_ruleutils.h" +#include "distributed/multi_router_planner.h" +#include "distributed/namespace_utils.h" +#include "executor/spi.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#include "parser/parse_node.h" +#include "parser/parse_agg.h" +#include "parser/parse_func.h" +#include "parser/parse_oper.h" +#include "parser/parse_relation.h" +#include "parser/parser.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rewriteSupport.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "utils/varlena.h" +#include "utils/xml.h" + + +/* ---------- + * Pretty formatting constants + * ---------- + */ + +/* Indent counts */ +#define PRETTYINDENT_STD 8 +#define PRETTYINDENT_JOIN 4 +#define PRETTYINDENT_VAR 4 + +#define PRETTYINDENT_LIMIT 40 /* wrap limit */ + +/* Pretty flags */ +#define PRETTYFLAG_PAREN 0x0001 +#define PRETTYFLAG_INDENT 0x0002 + +/* Default line length for pretty-print wrapping: 0 means wrap always */ +#define WRAP_COLUMN_DEFAULT 0 + +/* macros to test if pretty action needed */ +#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) +#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) + + +/* ---------- + * Local data types + * ---------- + */ + +/* Context info needed for invoking a recursive querytree display routine */ +typedef struct +{ + StringInfo buf; /* output buffer to append to */ + List *namespaces; /* List of deparse_namespace nodes */ + TupleDesc resultDesc; /* if top level of a view, the view's tupdesc */ + List *targetList; /* Current query level's SELECT targetlist */ + List *windowClause; /* Current query level's WINDOW clause */ + int prettyFlags; /* enabling of pretty-print functions */ + int wrapColumn; /* max line length, or -1 for no limit */ + int indentLevel; /* current indent level for prettyprint */ + bool varprefix; /* true to print prefixes on Vars */ + Oid distrelid; /* the distributed table being modified, if valid */ + int64 shardid; /* a distributed table's shardid, if positive */ + bool colNamesVisible; /* do we care about output column names? */ + bool inGroupBy; /* deparsing GROUP BY clause? */ + bool varInOrderBy; /* deparsing simple Var in ORDER BY? */ + Bitmapset *appendparents; /* if not null, map child Vars of these relids + * back to the parent rel */ +} deparse_context; + +/* + * Each level of query context around a subtree needs a level of Var namespace. + * A Var having varlevelsup=N refers to the N'th item (counting from 0) in + * the current context's namespaces list. + * + * The rangetable is the list of actual RTEs from the query tree, and the + * cte list is the list of actual CTEs. + * + * rtable_names holds the alias name to be used for each RTE (either a C + * string, or NULL for nameless RTEs such as unnamed joins). + * rtable_columns holds the column alias names to be used for each RTE. + * + * In some cases we need to make names of merged JOIN USING columns unique + * across the whole query, not only per-RTE. If so, unique_using is true + * and using_names is a list of C strings representing names already assigned + * to USING columns. + * + * When deparsing plan trees, there is always just a single item in the + * deparse_namespace list (since a plan tree never contains Vars with + * varlevelsup > 0). We store the PlanState node that is the immediate + * parent of the expression to be deparsed, as well as a list of that + * PlanState's ancestors. In addition, we store its outer and inner subplan + * state nodes, as well as their plan nodes' targetlists, and the index tlist + * if the current plan node might contain INDEX_VAR Vars. (These fields could + * be derived on-the-fly from the current PlanState, but it seems notationally + * clearer to set them up as separate fields.) + */ +typedef struct +{ + List *rtable; /* List of RangeTblEntry nodes */ + List *rtable_names; /* Parallel list of names for RTEs */ + List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *subplans; /* List of Plan trees for SubPlans */ + List *ctes; /* List of CommonTableExpr nodes */ + AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ + /* Workspace for column alias assignment: */ + bool unique_using; /* Are we making USING names globally unique */ + List *using_names; /* List of assigned names for USING columns */ + /* Remaining fields are used only when deparsing a Plan tree: */ + Plan *plan; /* immediate parent of current expression */ + List *ancestors; /* ancestors of planstate */ + Plan *outer_plan; /* outer subnode, or NULL if none */ + Plan *inner_plan; /* inner subnode, or NULL if none */ + List *outer_tlist; /* referent for OUTER_VAR Vars */ + List *inner_tlist; /* referent for INNER_VAR Vars */ + List *index_tlist; /* referent for INDEX_VAR Vars */ + /* Special namespace representing a function signature: */ + char *funcname; + int numargs; + char **argnames; +} deparse_namespace; + +/* Callback signature for resolve_special_varno() */ +typedef void (*rsv_callback) (Node *node, deparse_context *context, + void *callback_arg); + +/* + * Per-relation data about column alias names. + * + * Selecting aliases is unreasonably complicated because of the need to dump + * rules/views whose underlying tables may have had columns added, deleted, or + * renamed since the query was parsed. We must nonetheless print the rule/view + * in a form that can be reloaded and will produce the same results as before. + * + * For each RTE used in the query, we must assign column aliases that are + * unique within that RTE. SQL does not require this of the original query, + * but due to factors such as *-expansion we need to be able to uniquely + * reference every column in a decompiled query. As long as we qualify all + * column references, per-RTE uniqueness is sufficient for that. + * + * However, we can't ensure per-column name uniqueness for unnamed join RTEs, + * since they just inherit column names from their input RTEs, and we can't + * rename the columns at the join level. Most of the time this isn't an issue + * because we don't need to reference the join's output columns as such; we + * can reference the input columns instead. That approach can fail for merged + * JOIN USING columns, however, so when we have one of those in an unnamed + * join, we have to make that column's alias globally unique across the whole + * query to ensure it can be referenced unambiguously. + * + * Another problem is that a JOIN USING clause requires the columns to be + * merged to have the same aliases in both input RTEs, and that no other + * columns in those RTEs or their children conflict with the USING names. + * To handle that, we do USING-column alias assignment in a recursive + * traversal of the query's jointree. When descending through a JOIN with + * USING, we preassign the USING column names to the child columns, overriding + * other rules for column alias assignment. We also mark each RTE with a list + * of all USING column names selected for joins containing that RTE, so that + * when we assign other columns' aliases later, we can avoid conflicts. + * + * Another problem is that if a JOIN's input tables have had columns added or + * deleted since the query was parsed, we must generate a column alias list + * for the join that matches the current set of input columns --- otherwise, a + * change in the number of columns in the left input would throw off matching + * of aliases to columns of the right input. Thus, positions in the printable + * column alias list are not necessarily one-for-one with varattnos of the + * JOIN, so we need a separate new_colnames[] array for printing purposes. + */ +typedef struct +{ + /* + * colnames is an array containing column aliases to use for columns that + * existed when the query was parsed. Dropped columns have NULL entries. + * This array can be directly indexed by varattno to get a Var's name. + * + * Non-NULL entries are guaranteed unique within the RTE, *except* when + * this is for an unnamed JOIN RTE. In that case we merely copy up names + * from the two input RTEs. + * + * During the recursive descent in set_using_names(), forcible assignment + * of a child RTE's column name is represented by pre-setting that element + * of the child's colnames array. So at that stage, NULL entries in this + * array just mean that no name has been preassigned, not necessarily that + * the column is dropped. + */ + int num_cols; /* length of colnames[] array */ + char **colnames; /* array of C strings and NULLs */ + + /* + * new_colnames is an array containing column aliases to use for columns + * that would exist if the query was re-parsed against the current + * definitions of its base tables. This is what to print as the column + * alias list for the RTE. This array does not include dropped columns, + * but it will include columns added since original parsing. Indexes in + * it therefore have little to do with current varattno values. As above, + * entries are unique unless this is for an unnamed JOIN RTE. (In such an + * RTE, we never actually print this array, but we must compute it anyway + * for possible use in computing column names of upper joins.) The + * parallel array is_new_col marks which of these columns are new since + * original parsing. Entries with is_new_col false must match the + * non-NULL colnames entries one-for-one. + */ + int num_new_cols; /* length of new_colnames[] array */ + char **new_colnames; /* array of C strings */ + bool *is_new_col; /* array of bool flags */ + + /* This flag tells whether we should actually print a column alias list */ + bool printaliases; + + /* This list has all names used as USING names in joins above this RTE */ + List *parentUsing; /* names assigned to parent merged columns */ + + /* + * If this struct is for a JOIN RTE, we fill these fields during the + * set_using_names() pass to describe its relationship to its child RTEs. + * + * leftattnos and rightattnos are arrays with one entry per existing + * output column of the join (hence, indexable by join varattno). For a + * simple reference to a column of the left child, leftattnos[i] is the + * child RTE's attno and rightattnos[i] is zero; and conversely for a + * column of the right child. But for merged columns produced by JOIN + * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. + * Also, if the column has been dropped, both are zero. + * + * If it's a JOIN USING, usingNames holds the alias names selected for the + * merged columns (these might be different from the original USING list, + * if we had to modify names to achieve uniqueness). + */ + int leftrti; /* rangetable index of left child */ + int rightrti; /* rangetable index of right child */ + int *leftattnos; /* left-child varattnos of join cols, or 0 */ + int *rightattnos; /* right-child varattnos of join cols, or 0 */ + List *usingNames; /* names assigned to merged columns */ +} deparse_columns; + +/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ +#define deparse_columns_fetch(rangetable_index, dpns) \ + ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) + +/* + * Entry in set_rtable_names' hash table + */ +typedef struct +{ + char name[NAMEDATALEN]; /* Hash key --- must be first */ + int counter; /* Largest addition used so far for name */ +} NameHashEntry; + + +/* ---------- + * Local functions + * + * Most of these functions used to use fixed-size buffers to build their + * results. Now, they take an (already initialized) StringInfo object + * as a parameter, and append their text output to its contents. + * ---------- + */ +static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used); +static void set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces); +static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); +static void set_using_names(deparse_namespace *dpns, Node *jtnode, + List *parentUsing); +static void set_relation_column_names(deparse_namespace *dpns, + RangeTblEntry *rte, + deparse_columns *colinfo); +static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo); +static bool colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static char *make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static void expand_colnames_array_to(deparse_columns *colinfo, int n); +static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo); +static char *get_rtable_name(int rtindex, deparse_context *context); +static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); +static Plan *find_recursive_union(deparse_namespace *dpns, + WorkTableScan *wtscan); +static void push_child_plan(deparse_namespace *dpns, Plan *plan, + deparse_namespace *save_dpns); +static void pop_child_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns); +static void pop_ancestor_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, bool colNamesVisible, + int prettyFlags, int wrapColumn, int startIndent); +static void get_query_def_extended(Query *query, StringInfo buf, + List *parentnamespace, Oid distrelid, int64 shardid, + TupleDesc resultDesc, bool colNamesVisible, + int prettyFlags, int wrapColumn, + int startIndent); +static void get_values_def(List *values_lists, deparse_context *context); +static void get_with_clause(Query *query, deparse_context *context); +static void get_select_query_def(Query *query, deparse_context *context); +static void get_insert_query_def(Query *query, deparse_context *context); +static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); +static void get_delete_query_def(Query *query, deparse_context *context); +static void get_merge_query_def(Query *query, deparse_context *context); +static void get_utility_query_def(Query *query, deparse_context *context); +static void get_basic_select_query(Query *query, deparse_context *context); +static void get_target_list(List *targetList, deparse_context *context); +static void get_setop_query(Node *setOp, Query *query, + deparse_context *context); +static Node *get_rule_sortgroupclause(Index ref, List *tlist, + bool force_colno, + deparse_context *context); +static void get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context); +static void get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context); +static void get_rule_windowclause(Query *query, deparse_context *context); +static void get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context); +static char *get_variable(Var *var, int levelsup, bool istoplevel, + deparse_context *context); +static void get_special_variable(Node *node, deparse_context *context, + void *callback_arg); +static void resolve_special_varno(Node *node, deparse_context *context, + rsv_callback callback, void *callback_arg); +static Node *find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p); +static SubPlan *find_param_generator(Param *param, deparse_context *context, + int *column_p); +static SubPlan *find_param_generator_initplan(Param *param, Plan *plan, + int *column_p); +static void get_parameter(Param *param, deparse_context *context); +static const char *get_simple_binary_op_name(OpExpr *expr); +static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); +static void appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus); +static void removeStringInfoSpaces(StringInfo str); +static void get_rule_expr(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_list_toplevel(List *lst, deparse_context *context, + bool showimplicit); +static void get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit); +static bool looks_like_function(Node *node); +static void get_oper_expr(OpExpr *expr, deparse_context *context); +static void get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit); +static void get_proc_expr(CallStmt *stmt, deparse_context *context, + bool showimplicit); +static void get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref); +static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, + Aggref *original_aggref, const char *funcname, + const char *options, bool is_json_objectagg); +static void get_agg_combine_expr(Node *node, deparse_context *context, + void *callback_arg); +static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, + const char *funcname, const char *options, + bool is_json_objectagg); +static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); +static void get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode); +static void get_const_expr(Const *constval, deparse_context *context, + int showtype); +static void get_const_collation(Const *constval, deparse_context *context); +static void get_json_format(JsonFormat *format, StringInfo buf); +static void get_json_returning(JsonReturning *returning, StringInfo buf, + bool json_format_by_default); +static void get_json_constructor(JsonConstructorExpr *ctor, + deparse_context *context, bool showimplicit); +static void get_json_constructor_options(JsonConstructorExpr *ctor, + StringInfo buf); +static void get_json_agg_constructor(JsonConstructorExpr *ctor, + deparse_context *context, + const char *funcname, + bool is_json_objectagg); +static void simple_quote_literal(StringInfo buf, const char *val); +static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); +static void get_from_clause(Query *query, const char *prefix, + deparse_context *context); +static void get_from_clause_item(Node *jtnode, Query *query, + deparse_context *context); +static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, + deparse_context *context); +static void get_column_alias_list(deparse_columns *colinfo, + deparse_context *context); +static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context); +static void get_tablesample_def(TableSampleClause *tablesample, + deparse_context *context); +static void get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf); +static Node *processIndirection(Node *node, deparse_context *context); +static void printSubscripts(SubscriptingRef *aref, deparse_context *context); +static char *get_relation_name(Oid relid); +static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, + int64 shardid, List *namespaces); +static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); +static char *generate_fragment_name(char *schemaName, char *tableName); +static char *generate_function_name(Oid funcid, int nargs, + List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + bool inGroupBy); +static List *get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte); +static void get_json_path_spec(Node *path_spec, deparse_context *context, + bool showimplicit); +static void get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, + deparse_context *context, + bool showimplicit); +static void get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, + deparse_context *context, + bool showimplicit, + bool needcomma); + +#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") + + + +/* + * pg_get_query_def parses back one query tree, and outputs the resulting query + * string into given buffer. + */ +void +pg_get_query_def(Query *query, StringInfo buffer) +{ + get_query_def(query, buffer, NIL, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); +} + +/* + * get_merged_argument_list merges both the IN and OUT arguments lists into one and + * also eliminates the INOUT duplicates(present in both the lists). After merging both + * the lists, it returns all the named-arguments in a list(mergedNamedArgList) along + * with their types(mergedNamedArgTypes), final argument list(mergedArgumentList), and + * the total number of arguments(totalArguments). + */ +bool +get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, + Oid **mergedNamedArgTypes, + List **mergedArgumentList, + int *totalArguments) +{ + + Oid functionOid = stmt->funcexpr->funcid; + List *namedArgList = NIL; + List *finalArgumentList = NIL; + Oid *finalArgTypes; + Oid *argTypes = NULL; + char *argModes = NULL; + char **argNames = NULL; + int argIndex = 0; + + HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); + if (!HeapTupleIsValid(proctup)) + { + elog(ERROR, "cache lookup failed for function %u", functionOid); + } + + int defArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); + ReleaseSysCache(proctup); + + if (argModes == NULL) + { + /* No OUT arguments */ + return false; + } + + /* + * Passed arguments Includes IN, OUT, INOUT (in both the lists) and VARIADIC arguments, + * which means INOUT arguments are double counted. + */ + int numberOfArgs = list_length(stmt->funcexpr->args) + list_length(stmt->outargs); + int totalInoutArgs = 0; + + /* Let's count INOUT arguments from the defined number of arguments */ + for (argIndex=0; argIndex < defArgs; ++argIndex) + { + if (argModes[argIndex] == PROARGMODE_INOUT) + totalInoutArgs++; + } + + /* Remove the duplicate INOUT counting */ + numberOfArgs = numberOfArgs - totalInoutArgs; + finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs); + + ListCell *inArgCell = list_head(stmt->funcexpr->args); + ListCell *outArgCell = list_head(stmt->outargs); + + for (argIndex=0; argIndex < numberOfArgs; ++argIndex) + { + switch (argModes[argIndex]) + { + case PROARGMODE_IN: + case PROARGMODE_VARIADIC: + { + Node *arg = (Node *) lfirst(inArgCell); + + if (IsA(arg, NamedArgExpr)) + namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); + finalArgTypes[argIndex] = exprType(arg); + finalArgumentList = lappend(finalArgumentList, arg); + inArgCell = lnext(stmt->funcexpr->args, inArgCell); + break; + } + + case PROARGMODE_OUT: + { + Node *arg = (Node *) lfirst(outArgCell); + + if (IsA(arg, NamedArgExpr)) + namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); + finalArgTypes[argIndex] = exprType(arg); + finalArgumentList = lappend(finalArgumentList, arg); + outArgCell = lnext(stmt->outargs, outArgCell); + break; + } + + case PROARGMODE_INOUT: + { + Node *arg = (Node *) lfirst(inArgCell); + + if (IsA(arg, NamedArgExpr)) + namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); + finalArgTypes[argIndex] = exprType(arg); + finalArgumentList = lappend(finalArgumentList, arg); + inArgCell = lnext(stmt->funcexpr->args, inArgCell); + outArgCell = lnext(stmt->outargs, outArgCell); + break; + } + + case PROARGMODE_TABLE: + default: + { + elog(ERROR, "Unhandled procedure argument mode[%d]", argModes[argIndex]); + break; + } + } + } + + /* + * After eliminating INOUT duplicates and merging OUT arguments, we now + * have the final list of arguments. + */ + if (defArgs != list_length(finalArgumentList)) + { + elog(ERROR, "Insufficient number of args passed[%d] for function[%s]", + list_length(finalArgumentList), + get_func_name(functionOid)); + } + + if (list_length(finalArgumentList) > FUNC_MAX_ARGS) + { + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments[%d] for function[%s]", + list_length(finalArgumentList), + get_func_name(functionOid)))); + } + + *mergedNamedArgList = namedArgList; + *mergedNamedArgTypes = finalArgTypes; + *mergedArgumentList = finalArgumentList; + *totalArguments = numberOfArgs; + + return true; +} + +/* + * pg_get_rule_expr deparses an expression and returns the result as a string. + */ +char * +pg_get_rule_expr(Node *expression) +{ + bool showImplicitCasts = true; + deparse_context context; + StringInfo buffer = makeStringInfo(); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushEmptySearchPath(). + */ + int saveNestLevel = PushEmptySearchPath(); + + context.buf = buffer; + context.namespaces = NIL; + context.resultDesc = NULL; + context.targetList = NIL; + context.windowClause = NIL; + context.varprefix = false; + context.prettyFlags = 0; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = 0; + context.colNamesVisible = true; + context.inGroupBy = false; + context.varInOrderBy = false; + context.distrelid = InvalidOid; + context.shardid = INVALID_SHARD_ID; + + get_rule_expr(expression, &context, showImplicitCasts); + + /* revert back to original search_path */ + PopEmptySearchPath(saveNestLevel); + + return buffer->data; +} + +/* + * set_rtable_names: select RTE aliases to be used in printing a query + * + * We fill in dpns->rtable_names with a list of names that is one-for-one with + * the already-filled dpns->rtable list. Each RTE name is unique among those + * in the new namespace plus any ancestor namespaces listed in + * parent_namespaces. + * + * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. + * + * Note that this function is only concerned with relation names, not column + * names. + */ +static void +set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used) +{ + HASHCTL hash_ctl; + HTAB *names_hash; + NameHashEntry *hentry; + bool found; + int rtindex; + ListCell *lc; + + dpns->rtable_names = NIL; + /* nothing more to do if empty rtable */ + if (dpns->rtable == NIL) + return; + + /* + * We use a hash table to hold known names, so that this process is O(N) + * not O(N^2) for N names. + */ + hash_ctl.keysize = NAMEDATALEN; + hash_ctl.entrysize = sizeof(NameHashEntry); + hash_ctl.hcxt = CurrentMemoryContext; + names_hash = hash_create("set_rtable_names names", + list_length(dpns->rtable), + &hash_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + + /* Preload the hash table with names appearing in parent_namespaces */ + foreach(lc, parent_namespaces) + { + deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, olddpns->rtable_names) + { + char *oldname = (char *) lfirst(lc2); + + if (oldname == NULL) + continue; + hentry = (NameHashEntry *) hash_search(names_hash, + oldname, + HASH_ENTER, + &found); + /* we do not complain about duplicate names in parent namespaces */ + hentry->counter = 0; + } + } + + /* Now we can scan the rtable */ + rtindex = 1; + foreach(lc, dpns->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + char *refname; + + /* Just in case this takes an unreasonable amount of time ... */ + CHECK_FOR_INTERRUPTS(); + + if (rels_used && !bms_is_member(rtindex, rels_used)) + { + /* Ignore unreferenced RTE */ + refname = NULL; + } + else if (rte->alias) + { + /* If RTE has a user-defined alias, prefer that */ + refname = rte->alias->aliasname; + } + else if (rte->rtekind == RTE_RELATION) + { + /* Use the current actual name of the relation */ + refname = get_rel_name(rte->relid); + } + else if (rte->rtekind == RTE_JOIN) + { + /* Unnamed join has no refname */ + refname = NULL; + } + else + { + /* Otherwise use whatever the parser assigned */ + refname = rte->eref->aliasname; + } + + /* + * If the selected name isn't unique, append digits to make it so, and + * make a new hash entry for it once we've got a unique name. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (refname) + { + hentry = (NameHashEntry *) hash_search(names_hash, + refname, + HASH_ENTER, + &found); + if (found) + { + /* Name already in use, must choose a new one */ + int refnamelen = strlen(refname); + char *modname = (char *) palloc(refnamelen + 16); + NameHashEntry *hentry2; + + do + { + hentry->counter++; + for (;;) + { + memcpy(modname, refname, refnamelen); + sprintf(modname + refnamelen, "_%d", hentry->counter); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from refname to keep all the digits */ + refnamelen = pg_mbcliplen(refname, refnamelen, + refnamelen - 1); + } + hentry2 = (NameHashEntry *) hash_search(names_hash, + modname, + HASH_ENTER, + &found); + } while (found); + hentry2->counter = 0; /* init new hash entry */ + refname = modname; + } + else + { + /* Name not previously used, need only initialize hentry */ + hentry->counter = 0; + } + } + + dpns->rtable_names = lappend(dpns->rtable_names, refname); + rtindex++; + } + + hash_destroy(names_hash); +} + +/* + * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree + * + * For convenience, this is defined to initialize the deparse_namespace struct + * from scratch. + */ +static void +set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize *dpns and fill rtable/ctes links */ + memset(dpns, 0, sizeof(deparse_namespace)); + dpns->rtable = query->rtable; + dpns->subplans = NIL; + dpns->ctes = query->cteList; + dpns->appendrels = NULL; + + /* Assign a unique relation alias to each RTE */ + set_rtable_names(dpns, parent_namespaces, NULL); + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* If it's a utility query, it won't have a jointree */ + if (query->jointree) + { + /* Detect whether global uniqueness of USING names is needed */ + dpns->unique_using = + has_dangerous_join_using(dpns, (Node *) query->jointree); + + /* + * Select names for columns merged by USING, via a recursive pass over + * the query jointree. + */ + set_using_names(dpns, (Node *) query->jointree, NIL); + } + + /* + * Now assign remaining column aliases for each RTE. We do this in a + * linear scan of the rtable, so as to process RTEs whether or not they + * are in the jointree (we mustn't miss NEW.*, INSERT target relations, + * etc). JOIN RTEs must be processed after their children, but this is + * okay because they appear later in the rtable list than their children + * (cf Asserts in identify_join_columns()). + */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + if (rte->rtekind == RTE_JOIN) + set_join_column_names(dpns, rte, colinfo); + else + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * has_dangerous_join_using: search jointree for unnamed JOIN USING + * + * Merged columns of a JOIN USING may act differently from either of the input + * columns, either because they are merged with COALESCE (in a FULL JOIN) or + * because an implicit coercion of the underlying input column is required. + * In such a case the column must be referenced as a column of the JOIN not as + * a column of either input. And this is problematic if the join is unnamed + * (alias-less): we cannot qualify the column's name with an RTE name, since + * there is none. (Forcibly assigning an alias to the join is not a solution, + * since that will prevent legal references to tables below the join.) + * To ensure that every column in the query is unambiguously referenceable, + * we must assign such merged columns names that are globally unique across + * the whole query, aliasing other columns out of the way as necessary. + * + * Because the ensuing re-aliasing is fairly damaging to the readability of + * the query, we don't do this unless we have to. So, we must pre-scan + * the join tree to see if we have to, before starting set_using_names(). + */ +static bool +has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do here */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + { + if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) + return true; + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + /* Is it an unnamed JOIN with USING? */ + if (j->alias == NULL && j->usingClause) + { + /* + * Yes, so check each join alias var to see if any of them are not + * simple references to underlying columns. If so, we have a + * dangerous situation and must pick unique aliases. + */ + RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); + + /* We need only examine the merged columns */ + for (int i = 0; i < jrte->joinmergedcols; i++) + { + Node *aliasvar = list_nth(jrte->joinaliasvars, i); + + if (!IsA(aliasvar, Var)) + return true; + } + } + + /* Nope, but inspect children */ + if (has_dangerous_join_using(dpns, j->larg)) + return true; + if (has_dangerous_join_using(dpns, j->rarg)) + return true; + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + return false; +} + +/* + * set_using_names: select column aliases to be used for merged USING columns + * + * We do this during a recursive descent of the query jointree. + * dpns->unique_using must already be set to determine the global strategy. + * + * Column alias info is saved in the dpns->rtable_columns list, which is + * assumed to be filled with pre-zeroed deparse_columns structs. + * + * parentUsing is a list of all USING aliases assigned in parent joins of + * the current jointree node. (The passed-in list must not be modified.) + */ +static void +set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do now */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + set_using_names(dpns, (Node *) lfirst(lc), parentUsing); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + int *leftattnos; + int *rightattnos; + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + int i; + ListCell *lc; + + /* Get info about the shape of the join */ + identify_join_columns(j, rte, colinfo); + leftattnos = colinfo->leftattnos; + rightattnos = colinfo->rightattnos; + + /* Look up the not-yet-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * If this join is unnamed, then we cannot substitute new aliases at + * this level, so any name requirements pushed down to here must be + * pushed down again to the children. + */ + if (rte->alias == NULL) + { + for (i = 0; i < colinfo->num_cols; i++) + { + char *colname = colinfo->colnames[i]; + + if (colname == NULL) + continue; + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + } + } + + /* + * If there's a USING clause, select the USING column names and push + * those names down to the children. We have two strategies: + * + * If dpns->unique_using is true, we force all USING names to be + * unique across the whole query level. In principle we'd only need + * the names of dangerous USING columns to be globally unique, but to + * safely assign all USING names in a single pass, we have to enforce + * the same uniqueness rule for all of them. However, if a USING + * column's name has been pushed down from the parent, we should use + * it as-is rather than making a uniqueness adjustment. This is + * necessary when we're at an unnamed join, and it creates no risk of + * ambiguity. Also, if there's a user-written output alias for a + * merged column, we prefer to use that rather than the input name; + * this simplifies the logic and seems likely to lead to less aliasing + * overall. + * + * If dpns->unique_using is false, we only need USING names to be + * unique within their own join RTE. We still need to honor + * pushed-down names, though. + * + * Though significantly different in results, these two strategies are + * implemented by the same code, with only the difference of whether + * to put assigned names into dpns->using_names. + */ + if (j->usingClause) + { + /* Copy the input parentUsing list so we don't modify it */ + parentUsing = list_copy(parentUsing); + + /* USING names must correspond to the first join output columns */ + expand_colnames_array_to(colinfo, list_length(j->usingClause)); + i = 0; + foreach(lc, j->usingClause) + { + char *colname = strVal(lfirst(lc)); + + /* Assert it's a merged column */ + Assert(leftattnos[i] != 0 && rightattnos[i] != 0); + + /* Adopt passed-down name if any, else select unique name */ + if (colinfo->colnames[i] != NULL) + colname = colinfo->colnames[i]; + else + { + /* Prefer user-written output alias if any */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + /* Make it appropriately unique */ + colname = make_colname_unique(colname, dpns, colinfo); + if (dpns->unique_using) + dpns->using_names = lappend(dpns->using_names, + colname); + /* Save it as output column name, too */ + colinfo->colnames[i] = colname; + } + + /* Remember selected names for use later */ + colinfo->usingNames = lappend(colinfo->usingNames, colname); + parentUsing = lappend(parentUsing, colname); + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + + i++; + } + } + + /* Mark child deparse_columns structs with correct parentUsing info */ + leftcolinfo->parentUsing = parentUsing; + rightcolinfo->parentUsing = parentUsing; + + /* Now recursively assign USING column names in children */ + set_using_names(dpns, j->larg, parentUsing); + set_using_names(dpns, j->rarg, parentUsing); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * set_relation_column_names: select column aliases for a non-join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. + */ +static void +set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + int ncolumns; + char **real_colnames; + bool changed_any; + bool has_anonymous; + int noldcolumns; + int i; + int j; + + /* + * Construct an array of the current "real" column names of the RTE. + * real_colnames[] will be indexed by physical column number, with NULL + * entries for dropped columns. + */ + if (rte->rtekind == RTE_RELATION || + GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* Relation --- look to the system catalogs for up-to-date info */ + Relation rel; + TupleDesc tupdesc; + + rel = relation_open(rte->relid, AccessShareLock); + tupdesc = RelationGetDescr(rel); + + ncolumns = tupdesc->natts; + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + for (i = 0; i < ncolumns; i++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + + if (attr->attisdropped) + real_colnames[i] = NULL; + else + real_colnames[i] = pstrdup(NameStr(attr->attname)); + } + relation_close(rel, AccessShareLock); + } + else + { + /* Otherwise get the column names from eref or expandRTE() */ + List *colnames; + ListCell *lc; + + /* + * Functions returning composites have the annoying property that some + * of the composite type's columns might have been dropped since the + * query was parsed. If possible, use expandRTE() to handle that + * case, since it has the tedious logic needed to find out about + * dropped columns. However, if we're explaining a plan, then we + * don't have rte->functions because the planner thinks that won't be + * needed later, and that breaks expandRTE(). So in that case we have + * to rely on rte->eref, which may lead us to report a dropped + * column's old name; that seems close enough for EXPLAIN's purposes. + * + * For non-RELATION, non-FUNCTION RTEs, we can just look at rte->eref, + * which should be sufficiently up-to-date: no other RTE types can + * have columns get dropped from under them after parsing. + */ + if (rte->rtekind == RTE_FUNCTION && rte->functions != NIL) + { + /* Since we're not creating Vars, rtindex etc. don't matter */ + expandRTE(rte, 1, 0, -1, true /* include dropped */ , + &colnames, NULL); + } + else + colnames = rte->eref->colnames; + + ncolumns = list_length(colnames); + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + i = 0; + foreach(lc, colnames) + { + /* + * If the column name we find here is an empty string, then it's a + * dropped column, so change to NULL. + */ + char *cname = strVal(lfirst(lc)); + + if (cname[0] == '\0') + cname = NULL; + real_colnames[i] = cname; + i++; + } + } + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that there are now more columns than there were when the + * query was parsed, ie colnames could be longer than rte->eref->colnames. + * We must assign unique aliases to the new columns too, else there could + * be unresolved conflicts when the view/rule is reloaded. + */ + expand_colnames_array_to(colinfo, ncolumns); + Assert(colinfo->num_cols == ncolumns); + + /* + * Make sufficiently large new_colnames and is_new_col arrays, too. + * + * Note: because we leave colinfo->num_new_cols zero until after the loop, + * colname_is_unique will not consult that array, which is fine because it + * would only be duplicate effort. + */ + colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); + + /* + * Scan the columns, select a unique alias for each one, and store it in + * colinfo->colnames and colinfo->new_colnames. The former array has NULL + * entries for dropped columns, the latter omits them. Also mark + * new_colnames entries as to whether they are new since parse time; this + * is the case for entries beyond the length of rte->eref->colnames. + */ + noldcolumns = list_length(rte->eref->colnames); + changed_any = false; + has_anonymous = false; + j = 0; + for (i = 0; i < ncolumns; i++) + { + char *real_colname = real_colnames[i]; + char *colname = colinfo->colnames[i]; + + /* Skip dropped columns */ + if (real_colname == NULL) + { + Assert(colname == NULL); /* colnames[i] is already NULL */ + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Put names of non-dropped columns in new_colnames[] too */ + colinfo->new_colnames[j] = colname; + /* And mark them as new or not */ + colinfo->is_new_col[j] = (i >= noldcolumns); + j++; + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + + /* + * Remember if there is a reference to an anonymous column as named by + * char * FigureColname(Node *node) + */ + if (!has_anonymous && strcmp(real_colname, "?column?") == 0) + has_anonymous = true; + } + + /* + * Set correct length for new_colnames[] array. (Note: if columns have + * been added, colinfo->num_cols includes them, which is not really quite + * right but is harmless, since any new columns must be at the end where + * they won't affect varattnos of pre-existing columns.) + */ + colinfo->num_new_cols = j; + + /* + * For a relation RTE, we need only print the alias column names if any + * are different from the underlying "real" names. For a function RTE, + * always emit a complete column alias list; this is to protect against + * possible instability of the default column names (eg, from altering + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). + */ + if (rte->rtekind == RTE_RELATION) + colinfo->printaliases = changed_any; + else if (rte->rtekind == RTE_FUNCTION) + colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; + else if (rte->alias && rte->alias->colnames != NIL) + colinfo->printaliases = true; + else + colinfo->printaliases = changed_any || has_anonymous; +} + +/* + * set_join_column_names: select column aliases for a join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. Also, names for USING columns were already chosen by + * set_using_names(). We further expect that column alias selection has been + * completed for both input RTEs. + */ +static void +set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + bool changed_any; + int noldcolumns; + int nnewcolumns; + Bitmapset *leftmerged = NULL; + Bitmapset *rightmerged = NULL; + int i; + int j; + int ic; + int jc; + + /* Look up the previously-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that one or both inputs now have more columns than there + * were when the query was parsed, but we'll deal with that below. We + * only need entries in colnames for pre-existing columns. + */ + noldcolumns = list_length(rte->eref->colnames); + expand_colnames_array_to(colinfo, noldcolumns); + Assert(colinfo->num_cols == noldcolumns); + + /* + * Scan the join output columns, select an alias for each one, and store + * it in colinfo->colnames. If there are USING columns, set_using_names() + * already selected their names, so we can start the loop at the first + * non-merged column. + */ + changed_any = false; + for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) + { + char *colname = colinfo->colnames[i]; + char *real_colname; + + /* Join column must refer to at least one input column */ + Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); + + /* Get the child column name */ + if (colinfo->leftattnos[i] > 0) + real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; + else if (colinfo->rightattnos[i] > 0) + real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; + else + { + /* We're joining system columns --- use eref name */ + real_colname = strVal(list_nth(rte->eref->colnames, i)); + } + /* If child col has been dropped, no need to assign a join colname */ + if (real_colname == NULL) + { + colinfo->colnames[i] = NULL; + continue; + } + + /* In an unnamed join, just report child column names as-is */ + if (rte->alias == NULL) + { + colinfo->colnames[i] = real_colname; + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Calculate number of columns the join would have if it were re-parsed + * now, and create storage for the new_colnames and is_new_col arrays. + * + * Note: colname_is_unique will be consulting new_colnames[] during the + * loops below, so its not-yet-filled entries must be zeroes. + */ + nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - + list_length(colinfo->usingNames); + colinfo->num_new_cols = nnewcolumns; + colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); + + /* + * Generating the new_colnames array is a bit tricky since any new columns + * added since parse time must be inserted in the right places. This code + * must match the parser, which will order a join's columns as merged + * columns first (in USING-clause order), then non-merged columns from the + * left input (in attnum order), then non-merged columns from the right + * input (ditto). If one of the inputs is itself a join, its columns will + * be ordered according to the same rule, which means newly-added columns + * might not be at the end. We can figure out what's what by consulting + * the leftattnos and rightattnos arrays plus the input is_new_col arrays. + * + * In these loops, i indexes leftattnos/rightattnos (so it's join varattno + * less one), j indexes new_colnames/is_new_col, and ic/jc have similar + * meanings for the current child RTE. + */ + + /* Handle merged columns; they are first and can't be new */ + i = j = 0; + while (i < noldcolumns && + colinfo->leftattnos[i] != 0 && + colinfo->rightattnos[i] != 0) + { + /* column name is already determined and known unique */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + colinfo->is_new_col[j] = false; + + /* build bitmapsets of child attnums of merged columns */ + if (colinfo->leftattnos[i] > 0) + leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); + if (colinfo->rightattnos[i] > 0) + rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); + + i++, j++; + } + + /* Handle non-merged left-child columns */ + ic = 0; + for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) + { + char *child_colname = leftcolinfo->new_colnames[jc]; + + if (!leftcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of left child */ + while (ic < leftcolinfo->num_cols && + leftcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < leftcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, leftmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->leftattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; + j++; + } + + /* Handle non-merged right-child columns in exactly the same way */ + ic = 0; + for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) + { + char *child_colname = rightcolinfo->new_colnames[jc]; + + if (!rightcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of right child */ + while (ic < rightcolinfo->num_cols && + rightcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < rightcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, rightmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->rightattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; + j++; + } + + /* Assert we processed the right number of columns */ +#ifdef USE_ASSERT_CHECKING + for (int col_index = 0; col_index < colinfo->num_cols; col_index++) + { + /* + * In the above processing-loops, "i" advances only if + * the column is not new, check if this is a new column. + */ + if (colinfo->is_new_col[col_index]) + i++; + } + Assert(i == colinfo->num_cols); + Assert(j == nnewcolumns); +#endif + + /* + * For a named join, print column aliases if we changed any from the child + * names. Unnamed joins cannot print aliases. + */ + if (rte->alias != NULL) + colinfo->printaliases = changed_any; + else + colinfo->printaliases = false; +} + +/* + * colname_is_unique: is colname distinct from already-chosen column names? + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static bool +colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + int i; + ListCell *lc; + + /* Check against already-assigned column aliases within RTE */ + for (i = 0; i < colinfo->num_cols; i++) + { + char *oldname = colinfo->colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* + * If we're building a new_colnames array, check that too (this will be + * partially but not completely redundant with the previous checks) + */ + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *oldname = colinfo->new_colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against USING-column names that must be globally unique */ + foreach(lc, dpns->using_names) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against names already assigned for parent-join USING cols */ + foreach(lc, colinfo->parentUsing) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + return true; +} + +/* + * make_colname_unique: modify colname if necessary to make it unique + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static char * +make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + /* + * If the selected name isn't unique, append digits to make it so. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (!colname_is_unique(colname, dpns, colinfo)) + { + int colnamelen = strlen(colname); + char *modname = (char *) palloc(colnamelen + 16); + int i = 0; + + do + { + i++; + for (;;) + { + memcpy(modname, colname, colnamelen); + sprintf(modname + colnamelen, "_%d", i); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from colname to keep all the digits */ + colnamelen = pg_mbcliplen(colname, colnamelen, + colnamelen - 1); + } + } while (!colname_is_unique(modname, dpns, colinfo)); + colname = modname; + } + return colname; +} + +/* + * expand_colnames_array_to: make colinfo->colnames at least n items long + * + * Any added array entries are initialized to zero. + */ +static void +expand_colnames_array_to(deparse_columns *colinfo, int n) +{ + if (n > colinfo->num_cols) + { + if (colinfo->colnames == NULL) + colinfo->colnames = palloc0_array(char *, n); + else + { + colinfo->colnames = repalloc0_array(colinfo->colnames, char *, colinfo->num_cols, n); + } + colinfo->num_cols = n; + } +} + +/* + * identify_join_columns: figure out where columns of a join come from + * + * Fills the join-specific fields of the colinfo struct, except for + * usingNames which is filled later. + */ +static void +identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo) +{ + int numjoincols; + int jcolno; + int rcolno; + ListCell *lc; + + /* Extract left/right child RT indexes */ + if (IsA(j->larg, RangeTblRef)) + colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; + else if (IsA(j->larg, JoinExpr)) + colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->larg)); + if (IsA(j->rarg, RangeTblRef)) + colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; + else if (IsA(j->rarg, JoinExpr)) + colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->rarg)); + + /* Assert children will be processed earlier than join in second pass */ + Assert(colinfo->leftrti < j->rtindex); + Assert(colinfo->rightrti < j->rtindex); + + /* Initialize result arrays with zeroes */ + numjoincols = list_length(jrte->joinaliasvars); + Assert(numjoincols == list_length(jrte->eref->colnames)); + colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); + colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); + + /* + * Deconstruct RTE's joinleftcols/joinrightcols into desired format. + * Recall that the column(s) merged due to USING are the first column(s) + * of the join output. We need not do anything special while scanning + * joinleftcols, but while scanning joinrightcols we must distinguish + * merged from unmerged columns. + */ + jcolno = 0; + foreach(lc, jrte->joinleftcols) + { + int leftattno = lfirst_int(lc); + + colinfo->leftattnos[jcolno++] = leftattno; + } + rcolno = 0; + foreach(lc, jrte->joinrightcols) + { + int rightattno = lfirst_int(lc); + + if (rcolno < jrte->joinmergedcols) /* merged column? */ + colinfo->rightattnos[rcolno] = rightattno; + else + colinfo->rightattnos[jcolno++] = rightattno; + rcolno++; + } + Assert(jcolno == numjoincols); +} + +/* + * get_rtable_name: convenience function to get a previously assigned RTE alias + * + * The RTE must belong to the topmost namespace level in "context". + */ +static char * +get_rtable_name(int rtindex, deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); + return (char *) list_nth(dpns->rtable_names, rtindex - 1); +} + +/* + * set_deparse_plan: set up deparse_namespace to parse subexpressions + * of a given Plan node + * + * This sets the plan, outer_planstate, inner_planstate, outer_tlist, + * inner_tlist, and index_tlist fields. Caller is responsible for adjusting + * the ancestors list if necessary. Note that the rtable and ctes fields do + * not need to change when shifting attention to different plan nodes in a + * single plan tree. + */ +static void +set_deparse_plan(deparse_namespace *dpns, Plan *plan) +{ + dpns->plan = plan; + + /* + * We special-case Append and MergeAppend to pretend that the first child + * plan is the OUTER referent; we have to interpret OUTER Vars in their + * tlists according to one of the children, and the first one is the most + * natural choice. + */ + if (IsA(plan, Append)) + dpns->outer_plan = linitial(((Append *) plan)->appendplans); + else if (IsA(plan, MergeAppend)) + dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); + else + dpns->outer_plan = outerPlan(plan); + + if (dpns->outer_plan) + dpns->outer_tlist = dpns->outer_plan->targetlist; + else + dpns->outer_tlist = NIL; + + /* + * For a SubqueryScan, pretend the subplan is INNER referent. (We don't + * use OUTER because that could someday conflict with the normal meaning.) + * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For a WorkTableScan, locate the parent RecursiveUnion plan node and use + * that as INNER referent. + * + * For MERGE, pretend the ModifyTable's source plan (its outer plan) is + * INNER referent. This is the join from the target relation to the data + * source, and all INNER_VAR Vars in other parts of the query refer to its + * targetlist. + * + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). + */ + if (IsA(plan, SubqueryScan)) + dpns->inner_plan = ((SubqueryScan *) plan)->subplan; + else if (IsA(plan, CteScan)) + dpns->inner_plan = list_nth(dpns->subplans, + ((CteScan *) plan)->ctePlanId - 1); + else if (IsA(plan, WorkTableScan)) + dpns->inner_plan = find_recursive_union(dpns, + (WorkTableScan *) plan); + else if (IsA(plan, ModifyTable)) + { + if (((ModifyTable *) plan)->operation == CMD_MERGE) + dpns->inner_plan = outerPlan(plan); + else + dpns->inner_plan = plan; + } + else + dpns->inner_plan = innerPlan(plan); + + if (IsA(plan, ModifyTable) && ((ModifyTable *) plan)->operation == CMD_INSERT) + dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; + else if (dpns->inner_plan) + dpns->inner_tlist = dpns->inner_plan->targetlist; + else + dpns->inner_tlist = NIL; + + /* Set up referent for INDEX_VAR Vars, if needed */ + if (IsA(plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; + else if (IsA(plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; + else if (IsA(plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; + else + dpns->index_tlist = NIL; +} + +/* + * Locate the ancestor plan node that is the RecursiveUnion generating + * the WorkTableScan's work table. We can match on wtParam, since that + * should be unique within the plan tree. + */ +static Plan * +find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan) +{ + ListCell *lc; + + foreach(lc, dpns->ancestors) + { + Plan *ancestor = (Plan *) lfirst(lc); + + if (IsA(ancestor, RecursiveUnion) && + ((RecursiveUnion *) ancestor)->wtParam == wtscan->wtParam) + return ancestor; + } + elog(ERROR, "could not find RecursiveUnion for WorkTableScan with wtParam %d", + wtscan->wtParam); + return NULL; +} + +/* + * push_child_plan: temporarily transfer deparsing attention to a child plan + * + * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the + * deparse context in case the referenced expression itself uses + * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid + * affecting levelsup issues (although in a Plan tree there really shouldn't + * be any). + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_child_plan. + */ +static void +push_child_plan(deparse_namespace *dpns, Plan *plan, + deparse_namespace *save_dpns) +{ + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Link current plan node into ancestors list */ + dpns->ancestors = lcons(dpns->plan, dpns->ancestors); + + /* Set attention on selected child */ + set_deparse_plan(dpns, plan); +} + +/* + * pop_child_plan: undo the effects of push_child_plan + */ +static void +pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + List *ancestors; + + /* Get rid of ancestors list cell added by push_child_plan */ + ancestors = list_delete_first(dpns->ancestors); + + /* Restore fields changed by push_child_plan */ + *dpns = *save_dpns; + + /* Make sure dpns->ancestors is right (may be unnecessary) */ + dpns->ancestors = ancestors; +} + +/* + * push_ancestor_plan: temporarily transfer deparsing attention to an + * ancestor plan + * + * When expanding a Param reference, we must adjust the deparse context + * to match the plan node that contains the expression being printed; + * otherwise we'd fail if that expression itself contains a Param or + * OUTER_VAR/INNER_VAR/INDEX_VAR variable. + * + * The target ancestor is conveniently identified by the ListCell holding it + * in dpns->ancestors. + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_ancestor_plan. + */ +static void +push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns) +{ + Plan *plan = (Plan *) lfirst(ancestor_cell); + + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Build a new ancestor list with just this node's ancestors */ + dpns->ancestors = + list_copy_tail(dpns->ancestors, + list_cell_number(dpns->ancestors, ancestor_cell) + 1); + + /* Set attention on selected ancestor */ + set_deparse_plan(dpns, plan); +} + +/* + * pop_ancestor_plan: undo the effects of push_ancestor_plan + */ +static void +pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + /* Free the ancestor list made in push_ancestor_plan */ + list_free(dpns->ancestors); + + /* Restore fields changed by push_ancestor_plan */ + *dpns = *save_dpns; +} + +/* ---------- + * deparse_shard_query - Parse back a query for execution on a shard + * + * Builds an SQL string to perform the provided query on a specific shard and + * places this string into the provided buffer. + * ---------- + */ +void +deparse_shard_query(Query *query, Oid distrelid, int64 shardid, + StringInfo buffer) +{ + get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, + false, + 0, WRAP_COLUMN_DEFAULT, 0); +} + +/* ---------- + * get_query_def - Parse back one query parsetree + * + * query: parsetree to be displayed + * buf: output text is appended to buf + * parentnamespace: list (initially empty) of outer-level deparse_namespace's + * resultDesc: if not NULL, the output tuple descriptor for the view + * represented by a SELECT query. We use the column names from it + * to label SELECT output columns, in preference to names in the query + * colNamesVisible: true if the surrounding context cares about the output + * column names at all (as, for example, an EXISTS() context does not); + * when false, we can suppress dummy column labels such as "?column?" + * prettyFlags: bitmask of PRETTYFLAG_XXX options + * wrapColumn: maximum line length, or -1 to disable wrapping + * startIndent: initial indentation amount + * ---------- + */ +static void +get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, bool colNamesVisible, + int prettyFlags, int wrapColumn, int startIndent) +{ + get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, + colNamesVisible, + prettyFlags, wrapColumn, startIndent); +} + +/* ---------- + * get_query_def_extended - Parse back one query parsetree, optionally + * with extension using a shard identifier. + * + * If distrelid is valid and shardid is positive, the provided shardid is added + * any time the provided relid is deparsed, so that the query may be executed + * on a placement for the given shard. + * ---------- + */ +static void +get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, + Oid distrelid, int64 shardid, TupleDesc resultDesc, + bool colNamesVisible, + int prettyFlags, int wrapColumn, int startIndent) +{ + deparse_context context; + deparse_namespace dpns; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Before we begin to examine the query, acquire locks on referenced + * relations, and fix up deleted columns in JOIN RTEs. This ensures + * consistent results. Note we assume it's OK to scribble on the passed + * querytree! + * + * We are only deparsing the query (we are not about to execute it), so we + * only need AccessShareLock on the relations it mentions. + */ + AcquireRewriteLocks(query, false, false); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushEmptySearchPath(). + */ + int saveNestLevel = PushEmptySearchPath(); + + context.buf = buf; + context.namespaces = lcons(&dpns, list_copy(parentnamespace)); + context.resultDesc = NULL; + context.targetList = NIL; + context.windowClause = NIL; + context.varprefix = (parentnamespace != NIL || + list_length(query->rtable) != 1); + context.prettyFlags = prettyFlags; + context.wrapColumn = wrapColumn; + context.indentLevel = startIndent; + context.colNamesVisible = true; + context.inGroupBy = false; + context.varInOrderBy = false; + context.appendparents = NULL; + context.distrelid = distrelid; + context.shardid = shardid; + + set_deparse_for_query(&dpns, query, parentnamespace); + + switch (query->commandType) + { + case CMD_SELECT: + /* We set context.resultDesc only if it's a SELECT */ + context.resultDesc = resultDesc; + get_select_query_def(query, &context); + break; + + case CMD_UPDATE: + get_update_query_def(query, &context); + break; + + case CMD_INSERT: + get_insert_query_def(query, &context); + break; + + case CMD_DELETE: + get_delete_query_def(query, &context); + break; + + case CMD_MERGE: + get_merge_query_def(query, &context); + break; + + case CMD_NOTHING: + appendStringInfoString(buf, "NOTHING"); + break; + + case CMD_UTILITY: + get_utility_query_def(query, &context); + break; + + default: + elog(ERROR, "unrecognized query command type: %d", + query->commandType); + break; + } + + /* revert back to original search_path */ + PopEmptySearchPath(saveNestLevel); +} + +/* ---------- + * get_values_def - Parse back a VALUES list + * ---------- + */ +static void +get_values_def(List *values_lists, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first_list = true; + ListCell *vtl; + + appendStringInfoString(buf, "VALUES "); + + foreach(vtl, values_lists) + { + List *sublist = (List *) lfirst(vtl); + bool first_col = true; + ListCell *lc; + + if (first_list) + first_list = false; + else + appendStringInfoString(buf, ", "); + + appendStringInfoChar(buf, '('); + foreach(lc, sublist) + { + Node *col = (Node *) lfirst(lc); + + if (first_col) + first_col = false; + else + appendStringInfoChar(buf, ','); + + /* + * Print the value. Whole-row Vars need special treatment. + */ + get_rule_expr_toplevel(col, context, false); + } + appendStringInfoChar(buf, ')'); + } +} + +/* ---------- + * get_with_clause - Parse back a WITH clause + * ---------- + */ +static void +get_with_clause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + if (query->cteList == NIL) + return; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + if (query->hasRecursive) + sep = "WITH RECURSIVE "; + else + sep = "WITH "; + foreach(l, query->cteList) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); + + appendStringInfoString(buf, sep); + appendStringInfoString(buf, quote_identifier(cte->ctename)); + if (cte->aliascolnames) + { + bool first = true; + ListCell *col; + + appendStringInfoChar(buf, '('); + foreach(col, cte->aliascolnames) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(col)))); + } + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, " AS "); + switch (cte->ctematerialized) + { + case CTEMaterializeDefault: + break; + case CTEMaterializeAlways: + appendStringInfoString(buf, "MATERIALIZED "); + break; + case CTEMaterializeNever: + appendStringInfoString(buf, "NOT MATERIALIZED "); + break; + } + appendStringInfoChar(buf, '('); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, + true, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + appendStringInfoChar(buf, ')'); + + if (cte->search_clause) + { + bool first = true; + ListCell *lc; + + appendStringInfo(buf, " SEARCH %s FIRST BY ", + cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); + + foreach(lc, cte->search_clause->search_col_list) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(lc)))); + } + + appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); + } + + if (cte->cycle_clause) + { + bool first = true; + ListCell *lc; + + appendStringInfoString(buf, " CYCLE "); + + foreach(lc, cte->cycle_clause->cycle_col_list) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(lc)))); + } + + appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); + + { + Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); + Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); + + if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && + cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) + { + appendStringInfoString(buf, " TO "); + get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); + appendStringInfoString(buf, " DEFAULT "); + get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); + } + } + + appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); + } + + sep = ", "; + } + + if (PRETTY_INDENT(context)) + { + context->indentLevel -= PRETTYINDENT_STD; + appendContextKeyword(context, "", 0, 0, 0); + } + else + appendStringInfoChar(buf, ' '); +} + +/* ---------- + * get_select_query_def - Parse back a SELECT parsetree + * ---------- + */ +static void +get_select_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + bool force_colno; + ListCell *l; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* Subroutines may need to consult the SELECT targetlist and windowClause */ + context->targetList = query->targetList; + context->windowClause = query->windowClause; + + /* + * If the Query node has a setOperations tree, then it's the top level of + * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT + * fields are interesting in the top query itself. + */ + if (query->setOperations) + { + get_setop_query(query->setOperations, query, context); + /* ORDER BY clauses must be simple in this case */ + force_colno = true; + } + else + { + get_basic_select_query(query, context); + force_colno = false; + } + + /* Add the ORDER BY clause if given */ + if (query->sortClause != NIL) + { + appendContextKeyword(context, " ORDER BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_orderby(query->sortClause, query->targetList, + force_colno, context); + } + + /* + * Add the LIMIT/OFFSET clauses if given. If non-default options, use the + * standard spelling of LIMIT. + */ + if (query->limitOffset != NULL) + { + appendContextKeyword(context, " OFFSET ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitOffset, context, false); + } + if (query->limitCount != NULL) + { + if (query->limitOption == LIMIT_OPTION_WITH_TIES) + { + // had to add '(' and ')' here because it fails with casting + appendContextKeyword(context, " FETCH FIRST (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitCount, context, false); + appendStringInfoString(buf, ") ROWS WITH TIES"); + } + else + { + appendContextKeyword(context, " LIMIT ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + if (IsA(query->limitCount, Const) && + ((Const *) query->limitCount)->constisnull) + appendStringInfoString(buf, "ALL"); + else + get_rule_expr(query->limitCount, context, false); + } + } + + /* Add FOR [KEY] UPDATE/SHARE clauses if present */ + if (query->hasForUpdate) + { + foreach(l, query->rowMarks) + { + RowMarkClause *rc = (RowMarkClause *) lfirst(l); + + /* don't print implicit clauses */ + if (rc->pushedDown) + continue; + + switch (rc->strength) + { + case LCS_NONE: + /* we intentionally throw an error for LCS_NONE */ + elog(ERROR, "unrecognized LockClauseStrength %d", + (int) rc->strength); + break; + case LCS_FORKEYSHARE: + appendContextKeyword(context, " FOR KEY SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORSHARE: + appendContextKeyword(context, " FOR SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORNOKEYUPDATE: + appendContextKeyword(context, " FOR NO KEY UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORUPDATE: + appendContextKeyword(context, " FOR UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + } + + appendStringInfo(buf, " OF %s", + quote_identifier(get_rtable_name(rc->rti, + context))); + if (rc->waitPolicy == LockWaitError) + appendStringInfoString(buf, " NOWAIT"); + else if (rc->waitPolicy == LockWaitSkip) + appendStringInfoString(buf, " SKIP LOCKED"); + } + } +} + +/* + * Detect whether query looks like SELECT ... FROM VALUES(); + * if so, return the VALUES RTE. Otherwise return NULL. + */ +static RangeTblEntry * +get_simple_values_rte(Query *query, TupleDesc resultDesc) +{ + RangeTblEntry *result = NULL; + ListCell *lc; + int colno; + + /* + * We want to return true even if the Query also contains OLD or NEW rule + * RTEs. So the idea is to scan the rtable and see if there is only one + * inFromCl RTE that is a VALUES RTE. + */ + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + if (rte->rtekind == RTE_VALUES && rte->inFromCl) + { + if (result) + return NULL; /* multiple VALUES (probably not possible) */ + result = rte; + } + else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) + continue; /* ignore rule entries */ + else + return NULL; /* something else -> not simple VALUES */ + } + + /* + * We don't need to check the targetlist in any great detail, because + * parser/analyze.c will never generate a "bare" VALUES RTE --- they only + * appear inside auto-generated sub-queries with very restricted + * structure. However, DefineView might have modified the tlist by + * injecting new column aliases; so compare tlist resnames against the + * RTE's names to detect that. + */ + if (result) + { + ListCell *lcn; + + if (list_length(query->targetList) != list_length(result->eref->colnames)) + return NULL; /* this probably cannot happen */ + colno = 0; + forboth(lc, query->targetList, lcn, result->eref->colnames) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *cname = strVal(lfirst(lcn)); + char *colname; + + if (tle->resjunk) + return NULL; /* this probably cannot happen */ + /* compute name that get_target_list would use for column */ + colno++; + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* does it match the VALUES RTE? */ + if (colname == NULL || strcmp(colname, cname) != 0) + return NULL; /* column name has been changed */ + } + } + + return result; +} + +static void +get_basic_select_query(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *values_rte; + char *sep; + ListCell *l; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + /* + * If the query looks like SELECT * FROM (VALUES ...), then print just the + * VALUES part. This reverses what transformValuesClause() did at parse + * time. + */ + values_rte = get_simple_values_rte(query, context->resultDesc); + if (values_rte) + { + get_values_def(values_rte->values_lists, context); + return; + } + + /* + * Build up the query string - first we say SELECT + */ + if (query->isReturn) + appendStringInfoString(buf, "RETURN"); + else + appendStringInfoString(buf, "SELECT"); + + /* Add the DISTINCT clause if given */ + if (query->distinctClause != NIL) + { + if (query->hasDistinctOn) + { + appendStringInfoString(buf, " DISTINCT ON ("); + sep = ""; + foreach(l, query->distinctClause) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DISTINCT"); + } + + /* Then we tell what to select (the targetlist) */ + get_target_list(query->targetList, context); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add the WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add the GROUP BY clause if given */ + if (query->groupClause != NULL || query->groupingSets != NULL) + { + bool save_ingroupby; + + appendContextKeyword(context, " GROUP BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + if (query->groupDistinct) + appendStringInfoString(buf, "DISTINCT "); + + save_ingroupby = context->inGroupBy; + context->inGroupBy = true; + + if (query->groupingSets == NIL) + { + sep = ""; + foreach(l, query->groupClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + } + else + { + sep = ""; + foreach(l, query->groupingSets) + { + GroupingSet *grp = lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_groupingset(grp, query->targetList, true, context); + sep = ", "; + } + } + + context->inGroupBy = save_ingroupby; + } + + /* Add the HAVING clause if given */ + if (query->havingQual != NULL) + { + appendContextKeyword(context, " HAVING ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->havingQual, context, false); + } + + /* Add the WINDOW clause if needed */ + if (query->windowClause != NIL) + get_rule_windowclause(query, context); +} + +/* ---------- + * get_target_list - Parse back a SELECT target list + * + * This is also used for RETURNING lists in INSERT/UPDATE/DELETE/MERGE. + * ---------- + */ +static void +get_target_list(List *targetList, deparse_context *context) +{ + StringInfo buf = context->buf; + StringInfoData targetbuf; + bool last_was_multiline = false; + char *sep; + int colno; + ListCell *l; + + /* we use targetbuf to hold each TLE's text temporarily */ + initStringInfo(&targetbuf); + + sep = " "; + colno = 0; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + char *colname; + char *attname; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + colno++; + + /* + * Put the new field text into targetbuf so we can decide after we've + * got it whether or not it needs to go on a new line. + */ + resetStringInfo(&targetbuf); + context->buf = &targetbuf; + + /* + * We special-case Var nodes rather than using get_rule_expr. This is + * needed because get_rule_expr will display a whole-row Var as + * "foo.*", which is the preferred notation in most contexts, but at + * the top level of a SELECT list it's not right (the parser will + * expand that notation into multiple columns, yielding behavior + * different from a whole-row Var). We need to call get_variable + * directly so that we can tell it to do the right thing, and so that + * we can get the attribute name which is the default AS label. + */ + if (tle->expr && (IsA(tle->expr, Var))) + { + attname = get_variable((Var *) tle->expr, 0, true, context); + } + else + { + get_rule_expr((Node *) tle->expr, context, true); + + /* + * When colNamesVisible is true, we should always show the + * assigned column name explicitly. Otherwise, show it only if + * it's not FigureColname's fallback. + */ + attname = context->colNamesVisible ? NULL : "?column?"; + } + + /* + * Figure out what the result column should be called. In the context + * of a view, use the view's tuple descriptor (so as to pick up the + * effects of any column RENAME that's been done on the view). + * Otherwise, just use what we can find in the TLE. + */ + if (context->resultDesc && colno <= context->resultDesc->natts) + colname = NameStr(TupleDescAttr(context->resultDesc, + colno - 1)->attname); + else + colname = tle->resname; + + /* Show AS unless the column's name is correct as-is */ + if (colname) /* resname could be NULL */ + { + if (attname == NULL || strcmp(attname, colname) != 0) + appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); + } + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + int leading_nl_pos; + + /* Does the new field start with a new line? */ + if (targetbuf.len > 0 && targetbuf.data[0] == '\n') + leading_nl_pos = 0; + else + leading_nl_pos = -1; + + /* If so, we shouldn't add anything */ + if (leading_nl_pos >= 0) + { + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the output buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new field is + * not the first and either the new field would cause an + * overflow or the last field used more than one line. + */ + if (colno > 1 && + ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || + last_was_multiline)) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, PRETTYINDENT_VAR); + } + + /* Remember this field's multiline status for next iteration */ + last_was_multiline = + (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); + } + + /* Add the new field */ + appendStringInfoString(buf, targetbuf.data); + } + + /* clean up */ + pfree(targetbuf.data); +} + +static void +get_setop_query(Node *setOp, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + bool need_paren; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + if (IsA(setOp, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) setOp; + RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); + Query *subquery = rte->subquery; + + Assert(subquery != NULL); + Assert(subquery->setOperations == NULL); + /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ + need_paren = (subquery->cteList || + subquery->sortClause || + subquery->rowMarks || + subquery->limitOffset || + subquery->limitCount); + if (need_paren) + appendStringInfoChar(buf, '('); + get_query_def(subquery, buf, context->namespaces, + context->resultDesc, context->colNamesVisible, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (need_paren) + appendStringInfoChar(buf, ')'); + } + else if (IsA(setOp, SetOperationStmt)) + { + SetOperationStmt *op = (SetOperationStmt *) setOp; + int subindent; + bool save_colnamesvisible; + + /* + * We force parens when nesting two SetOperationStmts, except when the + * lefthand input is another setop of the same kind. Syntactically, + * we could omit parens in rather more cases, but it seems best to use + * parens to flag cases where the setop operator changes. If we use + * parens, we also increase the indentation level for the child query. + * + * There are some cases in which parens are needed around a leaf query + * too, but those are more easily handled at the next level down (see + * code above). + */ + if (IsA(op->larg, SetOperationStmt)) + { + SetOperationStmt *lop = (SetOperationStmt *) op->larg; + + if (op->op == lop->op && op->all == lop->all) + need_paren = false; + else + need_paren = true; + } + else + need_paren = false; + + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + appendContextKeyword(context, "", subindent, 0, 0); + } + else + subindent = 0; + + get_setop_query(op->larg, query, context); + + if (need_paren) + appendContextKeyword(context, ") ", -subindent, 0, 0); + else if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", -subindent, 0, 0); + else + appendStringInfoChar(buf, ' '); + + switch (op->op) + { + case SETOP_UNION: + appendStringInfoString(buf, "UNION "); + break; + case SETOP_INTERSECT: + appendStringInfoString(buf, "INTERSECT "); + break; + case SETOP_EXCEPT: + appendStringInfoString(buf, "EXCEPT "); + break; + default: + elog(ERROR, "unrecognized set op: %d", + (int) op->op); + } + if (op->all) + appendStringInfoString(buf, "ALL "); + + /* Always parenthesize if RHS is another setop */ + need_paren = IsA(op->rarg, SetOperationStmt); + + /* + * The indentation code here is deliberately a bit different from that + * for the lefthand input, because we want the line breaks in + * different places. + */ + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + } + else + subindent = 0; + appendContextKeyword(context, "", subindent, 0, 0); + + /* + * The output column names of the RHS sub-select don't matter. + */ + save_colnamesvisible = context->colNamesVisible; + context->colNamesVisible = false; + get_setop_query(op->rarg, query, context); + context->colNamesVisible = save_colnamesvisible; + + if (PRETTY_INDENT(context)) + context->indentLevel -= subindent; + if (need_paren) + appendContextKeyword(context, ")", 0, 0, 0); + } + else + { + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(setOp)); + } +} + +/* + * Display a sort/group clause. + * + * Also returns the expression tree, so caller need not find it again. + */ +static Node * +get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, + deparse_context *context) +{ + StringInfo buf = context->buf; + TargetEntry *tle; + Node *expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = (Node *) tle->expr; + + /* + * Use column-number form if requested by caller. Otherwise, if + * expression is a constant, force it to be dumped with an explicit cast + * as decoration --- this is because a simple integer constant is + * ambiguous (and will be misinterpreted by findTargetlistEntrySQL92()) if + * we dump it without any decoration. Similarly, if it's just a Var, + * there is risk of misinterpretation if the column name is reassigned in + * the SELECT list, so we may need to force table qualification. And, if + * it's anything more complex than a simple Var, then force extra parens + * around it, to ensure it can't be misinterpreted as a cube() or rollup() + * construct. + */ + if (force_colno) + { + Assert(!tle->resjunk); + appendStringInfo(buf, "%d", tle->resno); + } + else if (!expr) + /* do nothing, probably can't happen */ ; + else if (IsA(expr, Const)) + get_const_expr((Const *) expr, context, 1); + else if (IsA(expr, Var)) + { + /* Tell get_variable to check for name conflict */ + bool save_varinorderby = context->varInOrderBy; + context->varInOrderBy = true; + (void) get_variable((Var *) expr, 0, false, context); + context->varInOrderBy = save_varinorderby; + } + else + { + /* + * We must force parens for function-like expressions even if + * PRETTY_PAREN is off, since those are the ones in danger of + * misparsing. For other expressions we need to force them only if + * PRETTY_PAREN is on, since otherwise the expression will output them + * itself. (We can't skip the parens.) + */ + bool need_paren = (PRETTY_PAREN(context) + || IsA(expr, FuncExpr) + || IsA(expr, Aggref) + || IsA(expr, WindowFunc) + || IsA(expr, JsonConstructorExpr)); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + get_rule_expr(expr, context, true); + if (need_paren) + appendStringInfoChar(context->buf, ')'); + } + + return expr; +} + +/* + * Display a GroupingSet + */ +static void +get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context) +{ + ListCell *l; + StringInfo buf = context->buf; + bool omit_child_parens = true; + char *sep = ""; + + switch (gset->kind) + { + case GROUPING_SET_EMPTY: + appendStringInfoString(buf, "()"); + return; + + case GROUPING_SET_SIMPLE: + { + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, '('); + + foreach(l, gset->content) + { + Index ref = lfirst_int(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(ref, targetlist, + false, context); + sep = ", "; + } + + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, ')'); + } + return; + + case GROUPING_SET_ROLLUP: + appendStringInfoString(buf, "ROLLUP("); + break; + case GROUPING_SET_CUBE: + appendStringInfoString(buf, "CUBE("); + break; + case GROUPING_SET_SETS: + appendStringInfoString(buf, "GROUPING SETS ("); + omit_child_parens = false; + break; + } + + foreach(l, gset->content) + { + appendStringInfoString(buf, sep); + get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * Display an ORDER BY list. + */ +static void +get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = ""; + foreach(l, orderList) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + Node *sortexpr; + Oid sortcoltype; + TypeCacheEntry *typentry; + + appendStringInfoString(buf, sep); + sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, + force_colno, context); + sortcoltype = exprType(sortexpr); + /* See whether operator is default < or > for datatype */ + typentry = lookup_type_cache(sortcoltype, + TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + if (srt->sortop == typentry->lt_opr) + { + /* ASC is default, so emit nothing for it */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + } + else if (srt->sortop == typentry->gt_opr) + { + appendStringInfoString(buf, " DESC"); + /* DESC defaults to NULLS FIRST */ + if (!srt->nulls_first) + appendStringInfoString(buf, " NULLS LAST"); + } + else + { + appendStringInfo(buf, " USING %s", + generate_operator_name(srt->sortop, + sortcoltype, + sortcoltype)); + /* be specific to eliminate ambiguity */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + else + appendStringInfoString(buf, " NULLS LAST"); + } + sep = ", "; + } +} + +/* + * Display a WINDOW clause. + * + * Note that the windowClause list might contain only anonymous window + * specifications, in which case we should print nothing here. + */ +static void +get_rule_windowclause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = NULL; + foreach(l, query->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->name == NULL) + continue; /* ignore anonymous windows */ + + if (sep == NULL) + appendContextKeyword(context, " WINDOW ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + else + appendStringInfoString(buf, sep); + + appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); + + get_rule_windowspec(wc, query->targetList, context); + + sep = ", "; + } +} + +/* + * Display a window definition + */ +static void +get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context) +{ + StringInfo buf = context->buf; + bool needspace = false; + const char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + if (wc->refname) + { + appendStringInfoString(buf, quote_identifier(wc->refname)); + needspace = true; + } + /* partition clauses are always inherited, so only print if no refname */ + if (wc->partitionClause && !wc->refname) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "PARTITION BY "); + sep = ""; + foreach(l, wc->partitionClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, + false, context); + sep = ", "; + } + needspace = true; + } + /* print ordering clause only if not inherited */ + if (wc->orderClause && !wc->copiedOrder) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "ORDER BY "); + get_rule_orderby(wc->orderClause, targetList, false, context); + needspace = true; + } + /* framing clause is never inherited, so print unless it's default */ + if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) + { + if (needspace) + appendStringInfoChar(buf, ' '); + if (wc->frameOptions & FRAMEOPTION_RANGE) + appendStringInfoString(buf, "RANGE "); + else if (wc->frameOptions & FRAMEOPTION_ROWS) + appendStringInfoString(buf, "ROWS "); + else if (wc->frameOptions & FRAMEOPTION_GROUPS) + appendStringInfoString(buf, "GROUPS "); + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + appendStringInfoString(buf, "BETWEEN "); + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) + appendStringInfoString(buf, "UNBOUNDED PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) + { + get_rule_expr(wc->startOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + { + appendStringInfoString(buf, "AND "); + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) + appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); + else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) + { + get_rule_expr(wc->endOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + } + if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) + appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) + appendStringInfoString(buf, "EXCLUDE GROUP "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) + appendStringInfoString(buf, "EXCLUDE TIES "); + /* we will now have a trailing space; remove it */ + buf->len--; + } + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_insert_query_def - Parse back an INSERT parsetree + * ---------- + */ +static void +get_insert_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *select_rte = NULL; + RangeTblEntry *values_rte = NULL; + RangeTblEntry *rte; + ListCell *l; + List *strippedexprs = NIL; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * If it's an INSERT ... SELECT or multi-row VALUES, there will be a + * single RTE for the SELECT or VALUES. Plain VALUES has neither. + */ + foreach(l, query->rtable) + { + rte = (RangeTblEntry *) lfirst(l); + + if (rte->rtekind == RTE_SUBQUERY) + { + if (select_rte) + elog(ERROR, "too many subquery RTEs in INSERT"); + select_rte = rte; + } + + if (rte->rtekind == RTE_VALUES) + { + if (values_rte) + elog(ERROR, "too many values RTEs in INSERT"); + values_rte = rte; + } + } + if (select_rte && values_rte) + elog(ERROR, "both subquery and values RTEs in INSERT"); + + /* + * Start the query with INSERT INTO relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + appendStringInfo(buf, "INSERT INTO %s", + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + /* Print the relation alias, if needed; INSERT requires explicit AS */ + get_rte_alias(rte, query->resultRelation, true, context); + + /* always want a space here */ + appendStringInfoChar(buf, ' '); + + /* + * Add the insert-column-names list. Any indirection decoration needed on + * the column names can be inferred from the top targetlist. + */ + if (query->targetList) + { + strippedexprs = get_insert_column_names_list(query->targetList, + buf, context, rte); + } + + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + + if (select_rte) + { + /* Add the SELECT */ + get_query_def(select_rte->subquery, buf, context->namespaces, NULL, + false, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + } + else if (values_rte) + { + /* Add the multi-VALUES expression lists */ + get_values_def(values_rte->values_lists, context); + } + else if (strippedexprs) + { + /* Add the single-VALUES expression list */ + appendContextKeyword(context, "VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_list_toplevel(strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + { + /* No expressions, so it must be DEFAULT VALUES */ + appendStringInfoString(buf, "DEFAULT VALUES"); + } + + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + appendStringInfoString(buf, " ON CONFLICT"); + + if (confl->arbiterElems) + { + /* Add the single-VALUES expression list */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) confl->arbiterElems, context, false); + appendStringInfoChar(buf, ')'); + + /* Add a WHERE clause (for partial indexes) if given */ + if (confl->arbiterWhere != NULL) + { + bool save_varprefix; + + /* + * Force non-prefixing of Vars, since parser assumes that they + * belong to target relation. WHERE clause does not use + * InferenceElem, so this is separately required. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->arbiterWhere, context, false); + + context->varprefix = save_varprefix; + } + } + else if (OidIsValid(confl->constraint)) + { + char *constraint = get_constraint_name(confl->constraint); + int64 shardId = context->shardid; + + if (shardId > 0) + { + AppendShardIdToName(&constraint, shardId); + } + + if (!constraint) + elog(ERROR, "cache lookup failed for constraint %u", + confl->constraint); + appendStringInfo(buf, " ON CONSTRAINT %s", + quote_identifier(constraint)); + } + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " DO NOTHING"); + } + else + { + appendStringInfoString(buf, " DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context); + } +} + +/* ---------- + * get_update_query_def - Parse back an UPDATE parsetree + * ---------- + */ +static void +get_update_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with UPDATE relname SET + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + else + { + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + /* Print the relation alias, if needed */ + get_rte_alias(rte, query->resultRelation, false, context); + } + + appendStringInfoString(buf, " SET "); + + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context); + } +} + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + + /* + * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks + * into a list. We expect them to appear, in ID order, in resjunk tlist + * entries. + */ + ma_sublinks = NIL; + if (query->hasSubLinks) /* else there can't be any */ + { + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk && IsA(tle->expr, SubLink)) + { + SubLink *sl = (SubLink *) tle->expr; + + if (sl->subLinkType == MULTIEXPR_SUBLINK) + { + ma_sublinks = lappend(ma_sublinks, sl); + Assert(sl->subLinkId == list_length(ma_sublinks)); + } + } + } + } + next_ma_cell = list_head(ma_sublinks); + cur_ma_sublink = NULL; + remaining_ma_columns = 0; + + /* Add the comma separated list of 'attname = value' */ + sep = ""; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *expr; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + /* Emit separator (OK whether we're in multiassignment or not) */ + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Check to see if we're starting a multiassignment group: if so, + * output a left paren. + */ + if (next_ma_cell != NULL && cur_ma_sublink == NULL) + { + /* + * We must dig down into the expr to see if it's a PARAM_MULTIEXPR + * Param. That could be buried under FieldStores and + * SubscriptingRefs and CoerceToDomains (cf processIndirection()), + * and underneath those there could be an implicit type coercion. + * Because we would ignore implicit type coercions anyway, we + * don't need to be as careful as processIndirection() is about + * descending past implicit CoerceToDomains. + */ + expr = (Node *) tle->expr; + while (expr) + { + if (IsA(expr, FieldStore)) + { + FieldStore *fstore = (FieldStore *) expr; + + expr = (Node *) linitial(fstore->newvals); + } + else if (IsA(expr, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) expr; + + if (sbsref->refassgnexpr == NULL) + break; + expr = (Node *) sbsref->refassgnexpr; + } + else if (IsA(expr, CoerceToDomain)) + { + CoerceToDomain *cdomain = (CoerceToDomain *) expr; + + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + expr = (Node *) cdomain->arg; + } + else + break; + } + expr = strip_implicit_coercions(expr); + + if (expr && IsA(expr, Param) && + ((Param *) expr)->paramkind == PARAM_MULTIEXPR) + { + cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); + next_ma_cell = lnext(ma_sublinks, next_ma_cell); + remaining_ma_columns = count_nonjunk_tlist_entries( + ((Query *) cur_ma_sublink->subselect)->targetList); + Assert(((Param *) expr)->paramid == + ((cur_ma_sublink->subLinkId << 16) | 1)); + appendStringInfoChar(buf, '('); + } + } + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + */ + expr = processIndirection((Node *) tle->expr, context); + + /* + * If we're in a multiassignment, skip printing anything more, unless + * this is the last column; in which case, what we print should be the + * sublink, not the Param. + */ + if (cur_ma_sublink != NULL) + { + if (--remaining_ma_columns > 0) + continue; /* not the last column of multiassignment */ + appendStringInfoChar(buf, ')'); + expr = (Node *) cur_ma_sublink; + cur_ma_sublink = NULL; + } + + appendStringInfoString(buf, " = "); + + get_rule_expr(expr, context, false); + } +} + +/* ---------- + * get_delete_query_def - Parse back a DELETE parsetree + * ---------- + */ +static void +get_delete_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with DELETE FROM relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + else + { + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + /* Print the relation alias, if needed */ + get_rte_alias(rte, query->resultRelation, false, context); + } + + /* Add the USING clause if given */ + get_from_clause(query, " USING ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context); + } +} + + +/* ---------- + * get_merge_query_def - Parse back a MERGE parsetree + * ---------- + */ +static void +get_merge_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + ListCell *lc; + bool haveNotMatchedBySource; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with MERGE INTO relname + */ + rte = ExtractResultRelationRTE(query); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "MERGE INTO %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + else + { + appendStringInfo(buf, "MERGE INTO %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + + /* Print the source relation and join clause */ + get_from_clause(query, " USING ", context); + appendContextKeyword(context, " ON ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_expr(query->mergeJoinCondition, context, false); + + /* + * Test for any NOT MATCHED BY SOURCE actions. If there are none, then + * any NOT MATCHED BY TARGET actions are output as "WHEN NOT MATCHED", per + * SQL standard. Otherwise, we have a non-SQL-standard query, so output + * "BY SOURCE" / "BY TARGET" qualifiers for all NOT MATCHED actions, to be + * more explicit. + */ + haveNotMatchedBySource = false; + foreach(lc, query->mergeActionList) + { + MergeAction *action = lfirst_node(MergeAction, lc); + + if (action->matchKind == MERGE_WHEN_NOT_MATCHED_BY_SOURCE) + { + haveNotMatchedBySource = true; + break; + } + } + + /* Print each merge action */ + foreach(lc, query->mergeActionList) + { + MergeAction *action = lfirst_node(MergeAction, lc); + + appendContextKeyword(context, " WHEN ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + switch (action->matchKind) + { + case MERGE_WHEN_MATCHED: + appendStringInfoString(buf, "MATCHED"); + break; + case MERGE_WHEN_NOT_MATCHED_BY_SOURCE: + appendStringInfoString(buf, "NOT MATCHED BY SOURCE"); + break; + case MERGE_WHEN_NOT_MATCHED_BY_TARGET: + if (haveNotMatchedBySource) + appendStringInfoString(buf, "NOT MATCHED BY TARGET"); + else + appendStringInfoString(buf, "NOT MATCHED"); + break; + default: + elog(ERROR, "unrecognized matchKind: %d", + (int) action->matchKind); + } + + if (action->qual) + { + appendContextKeyword(context, " AND ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); + get_rule_expr(action->qual, context, false); + } + appendContextKeyword(context, " THEN ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); + + if (action->commandType == CMD_INSERT) + { + /* This generally matches get_insert_query_def() */ + List *strippedexprs = NIL; + const char *sep = ""; + ListCell *lc2; + + appendStringInfoString(buf, "INSERT"); + + if (action->targetList) + appendStringInfoString(buf, " ("); + foreach(lc2, action->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc2); + + Assert(!tle->resjunk); + + appendStringInfoString(buf, sep); + sep = ", "; + + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + if (action->targetList) + appendStringInfoChar(buf, ')'); + + if (action->override) + { + if (action->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, " OVERRIDING SYSTEM VALUE"); + else if (action->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, " OVERRIDING USER VALUE"); + } + + if (strippedexprs) + { + appendContextKeyword(context, " VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 4); + get_rule_list_toplevel(strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DEFAULT VALUES"); + } + else if (action->commandType == CMD_UPDATE) + { + appendStringInfoString(buf, "UPDATE SET "); + get_update_query_targetlist_def(query, action->targetList, + context, rte); + } + else if (action->commandType == CMD_DELETE) + appendStringInfoString(buf, "DELETE"); + else if (action->commandType == CMD_NOTHING) + appendStringInfoString(buf, "DO NOTHING"); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context); + } + + ereport(DEBUG1, (errmsg("", buf->data))); +} + + +/* ---------- + * get_utility_query_def - Parse back a UTILITY parsetree + * ---------- + */ +static void +get_utility_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) + { + NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + appendStringInfo(buf, "NOTIFY %s", + quote_identifier(stmt->conditionname)); + if (stmt->payload) + { + appendStringInfoString(buf, ", "); + simple_quote_literal(buf, stmt->payload); + } + } + else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) + { + TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; + List *relationList = stmt->relations; + ListCell *relationCell = NULL; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + + appendStringInfo(buf, "TRUNCATE TABLE"); + + foreach(relationCell, relationList) + { + RangeVar *relationVar = (RangeVar *) lfirst(relationCell); + Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); + char *relationName = generate_relation_or_shard_name(relationId, + context->distrelid, + context->shardid, NIL); + appendStringInfo(buf, " %s", relationName); + + if (lnext(relationList, relationCell) != NULL) + { + appendStringInfo(buf, ","); + } + } + + if (stmt->restart_seqs) + { + appendStringInfo(buf, " RESTART IDENTITY"); + } + + if (stmt->behavior == DROP_CASCADE) + { + appendStringInfo(buf, " CASCADE"); + } + } + else + { + /* Currently only NOTIFY utility commands can appear in rules */ + elog(ERROR, "unexpected utility statement type"); + } +} + +/* + * Display a Var appropriately. + * + * In some cases (currently only when recursing into an unnamed join) + * the Var's varlevelsup has to be interpreted with respect to a context + * above the current one; levelsup indicates the offset. + * + * If istoplevel is true, the Var is at the top level of a SELECT's + * targetlist, which means we need special treatment of whole-row Vars. + * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a + * dirty hack to prevent "tab.*" from being expanded into multiple columns. + * (The parser will strip the useless coercion, so no inefficiency is added in + * dump and reload.) We used to print just "tab" in such cases, but that is + * ambiguous and will yield the wrong result if "tab" is also a plain column + * name in the query. + * + * Returns the attname of the Var, or NULL if the Var has no attname (because + * it is a whole-row Var or a subplan output reference). + */ +static char * +get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + AttrNumber attnum; + int varno; + AttrNumber varattno; + int netlevelsup; + deparse_namespace *dpns; + deparse_columns *colinfo; + char *refname; + char *attname; + bool need_prefix; + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + varno = var->varno; + varattno = var->varattno; + + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. Also + * find the aliases previously assigned for this RTE. + */ + if (varno >= 1 && varno <= list_length(dpns->rtable)) + { + + /* + * We might have been asked to map child Vars to some parent relation. + */ + if (context->appendparents && dpns->appendrels) + { + + int pvarno = varno; + AttrNumber pvarattno = varattno; + AppendRelInfo *appinfo = dpns->appendrels[pvarno]; + bool found = false; + + /* Only map up to inheritance parents, not UNION ALL appendrels */ + while (appinfo && + rt_fetch(appinfo->parent_relid, + dpns->rtable)->rtekind == RTE_RELATION) + { + found = false; + if (pvarattno > 0) /* system columns stay as-is */ + { + if (pvarattno > appinfo->num_child_cols) + break; /* safety check */ + pvarattno = appinfo->parent_colnos[pvarattno - 1]; + if (pvarattno == 0) + break; /* Var is local to child */ + } + + pvarno = appinfo->parent_relid; + found = true; + + /* If the parent is itself a child, continue up. */ + Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); + appinfo = dpns->appendrels[pvarno]; + } + + /* + * If we found an ancestral rel, and that rel is included in + * appendparents, print that column not the original one. + */ + if (found && bms_is_member(pvarno, context->appendparents)) + { + varno = pvarno; + varattno = pvarattno; + } + } + + rte = rt_fetch(varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, varno - 1); + colinfo = deparse_columns_fetch(varno, dpns); + attnum = varattno; + } + else + { + resolve_special_varno((Node *) var, context, get_special_variable, + NULL); + return NULL; + } + + /* + * The planner will sometimes emit Vars referencing resjunk elements of a + * subquery's target list (this is currently only possible if it chooses + * to generate a "physical tlist" for a SubqueryScan or CteScan node). + * Although we prefer to print subquery-referencing Vars using the + * subquery's alias, that's not possible for resjunk items since they have + * no alias. So in that case, drill down to the subplan and print the + * contents of the referenced tlist item. This works because in a plan + * tree, such Vars can only occur in a SubqueryScan or CteScan node, and + * we'll have set dpns->inner_plan to reference the child plan node. + */ + if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && + attnum > list_length(rte->eref->colnames) && + dpns->inner_plan) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + /* + * Force parentheses because our caller probably assumed a Var is a + * simple expression. + */ + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tle->expr, context, true); + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, ')'); + + pop_child_plan(dpns, &save_dpns); + return NULL; + } + + /* + * If it's an unnamed join, look at the expansion of the alias variable. + * If it's a simple reference to one of the input vars, then recursively + * print the name of that var instead. When it's not a simple reference, + * we have to just print the unqualified join column name. (This can only + * happen with "dangerous" merged columns in a JOIN USING; we took pains + * previously to make the unqualified column name unique in such cases.) + * + * This wouldn't work in decompiling plan trees, because we don't store + * joinaliasvars lists after planning; but a plan tree should never + * contain a join alias variable. + */ + if (rte->rtekind == RTE_JOIN && rte->alias == NULL) + { + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + if (attnum > 0) + { + Var *aliasvar; + + aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); + /* we intentionally don't strip implicit coercions here */ + if (aliasvar && IsA(aliasvar, Var)) + { + return get_variable(aliasvar, var->varlevelsup + levelsup, + istoplevel, context); + } + } + + /* + * Unnamed join has no refname. (Note: since it's unnamed, there is + * no way the user could have referenced it to create a whole-row Var + * for it. So we don't have to cover that case below.) + */ + Assert(refname == NULL); + } + + if (attnum == InvalidAttrNumber) + attname = NULL; + else if (attnum > 0) + { + /* Get column name to use from the colinfo struct */ + if (attnum > colinfo->num_cols) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + attname = colinfo->colnames[attnum - 1]; + + /* + * If we find a Var referencing a dropped column, it seems better to + * print something (anything) than to fail. In general this should + * not happen, but it used to be possible for some cases involving + * functions returning named composite types, and perhaps there are + * still bugs out there. + */ + if (attname == NULL) + attname = "?dropped?column?"; + } + else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* System column on a Citus shard */ + attname = get_attname(rte->relid, attnum, false); + } + else + { + /* System column - name is fixed, get it from the catalog */ + attname = get_rte_attribute_name(rte, attnum); + } + + need_prefix = (context->varprefix || attname == NULL); + /* + * If we're considering a plain Var in an ORDER BY (but not GROUP BY) + * clause, we may need to add a table-name prefix to prevent + * findTargetlistEntrySQL92 from misinterpreting the name as an + * output-column name. To avoid cluttering the output with unnecessary + * prefixes, do so only if there is a name match to a SELECT tlist item + * that is different from the Var. + */ + if (context->varInOrderBy && !context->inGroupBy && !need_prefix) + { + int colno = 0; + foreach_node(TargetEntry, tle, context->targetList) + { + char *colname; + if (tle->resjunk) + continue; /* ignore junk entries */ + colno++; + /* This must match colname-choosing logic in get_target_list() */ + if (context->resultDesc && colno <= context->resultDesc->natts) + colname = NameStr(TupleDescAttr(context->resultDesc, + colno - 1)->attname); + else + colname = tle->resname; + if (colname && strcmp(colname, attname) == 0 && + !equal(var, tle->expr)) + { + need_prefix = true; + break; + } + } + } + + if (refname && need_prefix) + { + appendStringInfoString(buf, quote_identifier(refname)); + appendStringInfoChar(buf, '.'); + } + if (attname) + appendStringInfoString(buf, quote_identifier(attname)); + else + { + appendStringInfoChar(buf, '*'); + + if (istoplevel) + { + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* use rel.*::shard_name instead of rel.*::table_name */ + appendStringInfo(buf, "::%s", + generate_rte_shard_name(rte)); + } + else + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(var->vartype, + var->vartypmod)); + } + } + } + + return attname; +} + +/* + * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This + * routine is actually a callback for get_special_varno, which handles finding + * the correct TargetEntry. We get the expression contained in that + * TargetEntry and just need to deparse it, a job we can throw back on + * get_rule_expr. + */ +static void +get_special_variable(Node *node, deparse_context *context, void *callback_arg) +{ + StringInfo buf = context->buf; + + /* + * For a non-Var referent, force parentheses because our caller probably + * assumed a Var is a simple expression. + */ + if (!IsA(node, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr(node, context, true); + if (!IsA(node, Var)) + appendStringInfoChar(buf, ')'); +} + +/* + * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, + * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, + * invoke the callback provided. + */ +static void +resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) +{ + Var *var; + deparse_namespace *dpns; + + /* This function is recursive, so let's be paranoid. */ + check_stack_depth(); + + /* If it's not a Var, invoke the callback. */ + if (!IsA(node, Var)) + { + (*callback) (node, context, callback_arg); + return; + } + + /* Find appropriate nesting depth */ + var = (Var *) node; + dpns = (deparse_namespace *) list_nth(context->namespaces, + var->varlevelsup); + + /* + * It's a special RTE, so recurse. + */ + if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + Bitmapset *save_appendparents; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + /* If we're descending to the first child of an Append or MergeAppend, + * update appendparents. This will affect deparsing of all Vars + * appearing within the eventually-resolved subexpression. + */ + save_appendparents = context->appendparents; + + if (IsA(dpns->plan, Append)) + context->appendparents = bms_union(context->appendparents, + ((Append *) dpns->plan)->apprelids); + else if (IsA(dpns->plan, MergeAppend)) + context->appendparents = bms_union(context->appendparents, + ((MergeAppend *) dpns->plan)->apprelids); + + push_child_plan(dpns, dpns->outer_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, + callback, callback_arg); + pop_child_plan(dpns, &save_dpns); + context->appendparents = save_appendparents; + return; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); + return; + } + else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) + elog(ERROR, "bogus varno: %d", var->varno); + + /* Not special. Just invoke the callback. */ + (*callback) (node, context, callback_arg); +} + +/* + * Get the name of a field of an expression of composite type. The + * expression is usually a Var, but we handle other cases too. + * + * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * + * This is fairly straightforward when the expression has a named composite + * type; we need only look up the type in the catalogs. However, the type + * could also be RECORD. Since no actual table or view column is allowed to + * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE + * or to a subquery output. We drill down to find the ultimate defining + * expression and attempt to infer the field name from it. We ereport if we + * can't determine the name. + * + * Similarly, a PARAM of type RECORD has to refer to some expression of + * a determinable composite type. + */ +static const char * +get_name_for_var_field(Var *var, int fieldno, + int levelsup, deparse_context *context) +{ + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + int varno; + AttrNumber varattno; + TupleDesc tupleDesc; + Node *expr; + + /* + * If it's a RowExpr that was expanded from a whole-row Var, use the + * column names attached to it. (We could let get_expr_result_tupdesc() + * handle this, but it's much cheaper to just pull out the name we need.) + */ + if (IsA(var, RowExpr)) + { + RowExpr *r = (RowExpr *) var; + + if (fieldno > 0 && fieldno <= list_length(r->colnames)) + return strVal(list_nth(r->colnames, fieldno - 1)); + } + + /* + * If it's a Param of type RECORD, try to find what the Param refers to. + */ + if (IsA(var, Param)) + { + Param *param = (Param *) var; + ListCell *ancestor_cell; + + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so recurse to decipher the field name */ + deparse_namespace save_dpns; + const char *result; + + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + pop_ancestor_plan(dpns, &save_dpns); + return result; + } + } + + /* + * If it's a Var of type RECORD, we have to find what the Var refers to; + * if not, we can use get_expr_result_tupdesc(). + */ + if (!IsA(var, Var) || + var->vartype != RECORDOID) + { + tupleDesc = get_expr_result_tupdesc((Node *) var, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); + } + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + varno = var->varno; + varattno = var->varattno; + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. + */ + if (varno >= 1 && varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(varno, dpns->rtable); + attnum = varattno; + } + else if (varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->outer_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->outer_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->inner_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + const char *result; + + tle = get_tle_by_resno(dpns->index_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + return result; + } + else + { + elog(ERROR, "bogus varno: %d", varno); + return NULL; /* keep compiler quiet */ + } + + if (attnum == InvalidAttrNumber) + { + /* Var is whole-row reference to RTE, so select the right field */ + return get_rte_attribute_name(rte, fieldno); + } + + /* + * This part has essentially the same logic as the parser's + * expandRecordVariable() function, but we are dealing with a different + * representation of the input context, and we only need one field name + * not a TupleDesc. Also, we need special cases for finding subquery and + * CTE subplans when deparsing Plan trees. + */ + expr = (Node *) var; /* default if we can't drill down */ + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: + case RTE_RESULT: + + /* + * This case should not occur: a column of a table or values list + * shouldn't have type RECORD. Fall through and fail (most + * likely) at the bottom. + */ + break; + case RTE_SUBQUERY: + /* Subselect-in-FROM: examine sub-select's output expr */ + { + if (rte->subquery) + { + TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the sub-select to see what its Var + * refers to. We have to build an additional level of + * namespace to keep in step with varlevelsup in the + * subselect; furthermore, the subquery RTE might be + * from an outer query level, in which case the + * namespace for the subselect must have that outer + * level as parent namespace. + */ + List *save_nslist = context->namespaces; + List *parent_namespaces; + deparse_namespace mydpns; + const char *result; + + parent_namespaces = list_copy_tail(context->namespaces, + netlevelsup); + + set_deparse_for_query(&mydpns, rte->subquery, + parent_namespaces); + + context->namespaces = lcons(&mydpns, + parent_namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have complete + * RTE entries (in particular, rte->subquery is NULL). But + * the only place we'd normally see a Var directly + * referencing a SUBQUERY RTE is in a SubqueryScan plan + * node, and we can look into the child plan's tlist + * instead. An exception occurs if the subquery was + * proven empty and optimized away: then we'd find such a + * Var in a childless Result node, and there's nothing in + * the plan tree that would let us figure out what it had + * originally referenced. In that case, fall back on + * printing "fN", analogously to the default column names + * for RowExprs. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_plan) + { + char *dummy_name = palloc(32); + Assert(dpns->plan && IsA(dpns->plan, Result)); + snprintf(dummy_name, 32, "f%d", fieldno); + return dummy_name; + } + Assert(dpns->plan && IsA(dpns->plan, SubqueryScan)); + + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + case RTE_JOIN: + /* Join RTE --- recursively inspect the alias variable */ + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); + expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); + Assert(expr != NULL); + /* we intentionally don't strip implicit coercions here */ + if (IsA(expr, Var)) + return get_name_for_var_field((Var *) expr, fieldno, + var->varlevelsup + levelsup, + context); + /* else fall through to inspect the expression */ + break; + case RTE_FUNCTION: + case RTE_TABLEFUNC: + + /* + * We couldn't get here unless a function is declared with one of + * its result columns as RECORD, which is not allowed. + */ + break; + case RTE_CTE: + /* CTE reference: examine subquery's output expr */ + { + CommonTableExpr *cte = NULL; + Index ctelevelsup; + ListCell *lc; + + /* + * Try to find the referenced CTE using the namespace stack. + */ + ctelevelsup = rte->ctelevelsup + netlevelsup; + if (ctelevelsup >= list_length(context->namespaces)) + lc = NULL; + else + { + deparse_namespace *ctedpns; + + ctedpns = (deparse_namespace *) + list_nth(context->namespaces, ctelevelsup); + foreach(lc, ctedpns->ctes) + { + cte = (CommonTableExpr *) lfirst(lc); + if (strcmp(cte->ctename, rte->ctename) == 0) + break; + } + } + if (lc != NULL) + { + Query *ctequery = (Query *) cte->ctequery; + TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "CTE %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the CTE to see what its Var refers to. + * We have to build an additional level of namespace + * to keep in step with varlevelsup in the CTE; + * furthermore it could be an outer CTE (compare + * SUBQUERY case above). + */ + List *save_nslist = context->namespaces; + List *parent_namespaces; + deparse_namespace mydpns; + const char *result; + + parent_namespaces = list_copy_tail(context->namespaces, + ctelevelsup); + + set_deparse_for_query(&mydpns, ctequery, + parent_namespaces); + + context->namespaces = lcons(&mydpns, parent_namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have a CTE + * list. But the only places we'd normally see a Var + * directly referencing a CTE RTE are in CteScan or + * WorkTableScan plan nodes. For those cases, + * set_deparse_plan arranged for dpns->inner_plan to be + * the plan node that emits the CTE or RecursiveUnion + * result, and we can look at its tlist instead. As + * above, this can fail if the CTE has been proven empty, + * in which case fall back to "fN". + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_plan) + { + char *dummy_name = palloc(32); + Assert(dpns->plan && IsA(dpns->plan, Result)); + snprintf(dummy_name, 32, "f%d", fieldno); + return dummy_name; + } + Assert(dpns->plan && (IsA(dpns->plan, CteScan) || + IsA(dpns->plan, WorkTableScan))); + + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + } + + /* + * We now have an expression we can't expand any more, so see if + * get_expr_result_tupdesc() can do anything with it. + */ + tupleDesc = get_expr_result_tupdesc(expr, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); +} + +/* + * Try to find the referenced expression for a PARAM_EXEC Param that might + * reference a parameter supplied by an upper NestLoop or SubPlan plan node. + * + * If successful, return the expression and set *dpns_p and *ancestor_cell_p + * appropriately for calling push_ancestor_plan(). If no referent can be + * found, return NULL. + */ +static Node * +find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p) +{ + /* Initialize output parameters to prevent compiler warnings */ + *dpns_p = NULL; + *ancestor_cell_p = NULL; + + /* + * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or + * SubPlan argument. This will necessarily be in some ancestor of the + * current expression's Plan. + */ + if (param->paramkind == PARAM_EXEC) + { + deparse_namespace *dpns; + Plan *child_plan; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + child_plan = dpns->plan; + + foreach(lc, dpns->ancestors) + { + Node *ancestor = (Node *) lfirst(lc); + ListCell *lc2; + + /* + * NestLoops transmit params to their inner child only. + */ + if (IsA(ancestor, NestLoop) && + child_plan == innerPlan(ancestor)) + { + NestLoop *nl = (NestLoop *) ancestor; + + foreach(lc2, nl->nestParams) + { + NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); + + if (nlp->paramno == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return (Node *) nlp->paramval; + } + } + } + + /* + * Check to see if we're crawling up from a subplan. + */ + if(IsA(ancestor, SubPlan)) + { + SubPlan *subplan = (SubPlan *) ancestor; + ListCell *lc3; + ListCell *lc4; + + /* Matched subplan, so check its arguments */ + forboth(lc3, subplan->parParam, lc4, subplan->args) + { + int paramid = lfirst_int(lc3); + Node *arg = (Node *) lfirst(lc4); + + if (paramid == param->paramid) + { + /* + * Found a match, so return it. But, since Vars in + * the arg are to be evaluated in the surrounding + * context, we have to point to the next ancestor item + * that is *not* a SubPlan. + */ + ListCell *rest; + + for_each_cell(rest, dpns->ancestors, + lnext(dpns->ancestors, lc)) + { + Node *ancestor2 = (Node *) lfirst(rest); + + if (!IsA(ancestor2, SubPlan)) + { + *dpns_p = dpns; + *ancestor_cell_p = rest; + return arg; + } + } + elog(ERROR, "SubPlan cannot be outermost ancestor"); + } + } + + /* SubPlan isn't a kind of Plan, so skip the rest */ + continue; + } + + /* + * We need not consider the ancestor's initPlan list, since + * initplans never have any parParams. + */ + + /* No luck, crawl up to next ancestor */ + child_plan = (Plan *) ancestor; + } + } + + /* No referent found */ + return NULL; +} + +/* + * Try to find a subplan/initplan that emits the value for a PARAM_EXEC Param. + * + * If successful, return the generating subplan/initplan and set *column_p + * to the subplan's 0-based output column number. + * Otherwise, return NULL. + */ +static SubPlan * +find_param_generator(Param *param, deparse_context *context, int *column_p) +{ + /* Initialize output parameter to prevent compiler warnings */ + *column_p = 0; + + /* + * If it's a PARAM_EXEC parameter, search the current plan node as well as + * ancestor nodes looking for a subplan or initplan that emits the value + * for the Param. It could appear in the setParams of an initplan or + * MULTIEXPR_SUBLINK subplan, or in the paramIds of an ancestral SubPlan. + */ + if (param->paramkind == PARAM_EXEC) + { + SubPlan *result; + deparse_namespace *dpns; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + + /* First check the innermost plan node's initplans */ + result = find_param_generator_initplan(param, dpns->plan, column_p); + if (result) + return result; + + /* + * The plan's targetlist might contain MULTIEXPR_SUBLINK SubPlans, + * which can be referenced by Params elsewhere in the targetlist. + * (Such Params should always be in the same targetlist, so there's no + * need to do this work at upper plan nodes.) + */ + foreach_node(TargetEntry, tle, dpns->plan->targetlist) + { + if (tle->expr && IsA(tle->expr, SubPlan)) + { + SubPlan *subplan = (SubPlan *) tle->expr; + + if (subplan->subLinkType == MULTIEXPR_SUBLINK) + { + foreach_int(paramid, subplan->setParam) + { + if (paramid == param->paramid) + { + /* Found a match, so return it. */ + *column_p = foreach_current_index(paramid); + return subplan; + } + } + } + } + } + + /* No luck, so check the ancestor nodes */ + foreach(lc, dpns->ancestors) + { + Node *ancestor = (Node *) lfirst(lc); + + /* + * If ancestor is a SubPlan, check the paramIds it provides. + */ + if (IsA(ancestor, SubPlan)) + { + SubPlan *subplan = (SubPlan *) ancestor; + + foreach_int(paramid, subplan->paramIds) + { + if (paramid == param->paramid) + { + /* Found a match, so return it. */ + *column_p = foreach_current_index(paramid); + return subplan; + } + } + + /* SubPlan isn't a kind of Plan, so skip the rest */ + continue; + } + + /* + * Otherwise, it's some kind of Plan node, so check its initplans. + */ + result = find_param_generator_initplan(param, (Plan *) ancestor, + column_p); + if (result) + return result; + + /* No luck, crawl up to next ancestor */ + } + } + + /* No generator found */ + return NULL; +} + +/* + * Subroutine for find_param_generator: search one Plan node's initplans + */ +static SubPlan * +find_param_generator_initplan(Param *param, Plan *plan, int *column_p) +{ + foreach_node(SubPlan, subplan, plan->initPlan) + { + foreach_int(paramid, subplan->setParam) + { + if (paramid == param->paramid) + { + /* Found a match, so return it. */ + *column_p = foreach_current_index(paramid); + return subplan; + } + } + } + return NULL; +} + +/* + * Display a Param appropriately. + */ +static void +get_parameter(Param *param, deparse_context *context) +{ + Node *expr; + deparse_namespace *dpns; + ListCell *ancestor_cell; + SubPlan *subplan; + int column; + + /* + * If it's a PARAM_EXEC parameter, try to locate the expression from which + * the parameter was computed. This stanza handles only cases in which + * the Param represents an input to the subplan we are currently in. + */ + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so print it */ + deparse_namespace save_dpns; + bool save_varprefix; + bool need_paren; + + /* Switch attention to the ancestor plan node */ + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + + /* + * Force prefixing of Vars, since they won't belong to the relation + * being scanned in the original plan node. + */ + save_varprefix = context->varprefix; + context->varprefix = true; + + /* + * A Param's expansion is typically a Var, Aggref, GroupingFunc, or + * upper-level Param, which wouldn't need extra parentheses. + * Otherwise, insert parens to ensure the expression looks atomic. + */ + need_paren = !(IsA(expr, Var) || + IsA(expr, Aggref) || + IsA(expr, GroupingFunc) || + IsA(expr, Param)); + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(expr, context, false); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); + + context->varprefix = save_varprefix; + + pop_ancestor_plan(dpns, &save_dpns); + + return; + } + + /* + * Alternatively, maybe it's a subplan output, which we print as a + * reference to the subplan. (We could drill down into the subplan and + * print the relevant targetlist expression, but that has been deemed too + * confusing since it would violate normal SQL scope rules. Also, we're + * relying on this reference to show that the testexpr containing the + * Param has anything to do with that subplan at all.) + */ + subplan = find_param_generator(param, context, &column); + if (subplan) + { + appendStringInfo(context->buf, "(%s%s).col%d", + subplan->useHashTable ? "hashed " : "", + subplan->plan_name, column + 1); + + return; + } + + /* + * If it's an external parameter, see if the outermost namespace provides + * function argument names. + */ + if (param->paramkind == PARAM_EXTERN && context->namespaces != NIL) + { + dpns = llast(context->namespaces); + if (dpns->argnames && + param->paramid > 0 && + param->paramid <= dpns->numargs) + { + char *argname = dpns->argnames[param->paramid - 1]; + + if (argname) + { + bool should_qualify = false; + ListCell *lc; + + /* + * Qualify the parameter name if there are any other deparse + * namespaces with range tables. This avoids qualifying in + * trivial cases like "RETURN a + b", but makes it safe in all + * other cases. + */ + foreach(lc, context->namespaces) + { + deparse_namespace *depns = lfirst(lc); + + if (depns->rtable_names != NIL) + { + should_qualify = true; + break; + } + } + if (should_qualify) + { + appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); + appendStringInfoChar(context->buf, '.'); + } + + appendStringInfoString(context->buf, quote_identifier(argname)); + return; + } + } + } + + /* + * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. + * For composite types, add cast to the parameter to ease remote node detect + * the type. + * + * It's a bug if we get here for anything except PARAM_EXTERN Params, but + * in production builds printing $N seems more useful than failing. + */ + Assert(param->paramkind == PARAM_EXTERN); + + if (param->paramtype >= FirstNormalObjectId) + { + char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); + + appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); + } + else + { + appendStringInfo(context->buf, "$%d", param->paramid); + } +} + +/* + * get_simple_binary_op_name + * + * helper function for isSimpleNode + * will return single char binary operator name, or NULL if it's not + */ +static const char * +get_simple_binary_op_name(OpExpr *expr) +{ + List *args = expr->args; + + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + const char *op; + + op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); + if (strlen(op) == 1) + return op; + } + return NULL; +} + +/* + * isSimpleNode - check if given node is simple (doesn't need parenthesizing) + * + * true : simple in the context of parent node's type + * false : not simple + */ +static bool +isSimpleNode(Node *node, Node *parentNode, int prettyFlags) +{ + if (!node) + return false; + + switch (nodeTag(node)) + { + case T_Var: + case T_Const: + case T_Param: + case T_CoerceToDomainValue: + case T_SetToDefault: + case T_CurrentOfExpr: + /* single words: always simple */ + return true; + + case T_SubscriptingRef: + case T_ArrayExpr: + case T_RowExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_NextValueExpr: + case T_NullIfExpr: + case T_Aggref: + case T_GroupingFunc: + case T_WindowFunc: + case T_MergeSupportFunc: + case T_FuncExpr: + case T_JsonConstructorExpr: + case T_JsonExpr: + /* function-like: name(..) or name[..] */ + return true; + + /* CASE keywords act as parentheses */ + case T_CaseExpr: + return true; + + case T_FieldSelect: + + /* + * appears simple since . has top precedence, unless parent is + * T_FieldSelect itself! + */ + return !IsA(parentNode, FieldSelect); + + case T_FieldStore: + + /* + * treat like FieldSelect (probably doesn't matter) + */ + return !IsA(parentNode, FieldStore); + + case T_CoerceToDomain: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, + node, prettyFlags); + case T_RelabelType: + return isSimpleNode((Node *) ((RelabelType *) node)->arg, + node, prettyFlags); + case T_CoerceViaIO: + return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, + node, prettyFlags); + case T_ArrayCoerceExpr: + return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, + node, prettyFlags); + case T_ConvertRowtypeExpr: + return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, + node, prettyFlags); + + case T_OpExpr: + { + /* depends on parent node type; needs further checking */ + if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) + { + const char *op; + const char *parentOp; + bool is_lopriop; + bool is_hipriop; + bool is_lopriparent; + bool is_hipriparent; + + op = get_simple_binary_op_name((OpExpr *) node); + if (!op) + return false; + + /* We know only the basic operators + - and * / % */ + is_lopriop = (strchr("+-", *op) != NULL); + is_hipriop = (strchr("*/%", *op) != NULL); + if (!(is_lopriop || is_hipriop)) + return false; + + parentOp = get_simple_binary_op_name((OpExpr *) parentNode); + if (!parentOp) + return false; + + is_lopriparent = (strchr("+-", *parentOp) != NULL); + is_hipriparent = (strchr("*/%", *parentOp) != NULL); + if (!(is_lopriparent || is_hipriparent)) + return false; + + if (is_hipriop && is_lopriparent) + return true; /* op binds tighter than parent */ + + if (is_lopriop && is_hipriparent) + return false; + + /* + * Operators are same priority --- can skip parens only if + * we have (a - b) - c, not a - (b - c). + */ + if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) + return true; + + return false; + } + /* else do the same stuff as for T_SubLink et al. */ + } + /* FALLTHROUGH */ + + case T_SubLink: + case T_NullTest: + case T_BooleanTest: + case T_DistinctExpr: + case T_JsonIsPredicate: + switch (nodeTag(parentNode)) + { + case T_FuncExpr: + { + /* special handling for casts and COERCE_SQL_SYNTAX */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST || + type == COERCE_SQL_SYNTAX) + return false; + return true; /* own parentheses */ + } + case T_BoolExpr: /* lower precedence */ + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_GroupingFunc: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + case T_BoolExpr: + switch (nodeTag(parentNode)) + { + case T_BoolExpr: + if (prettyFlags & PRETTYFLAG_PAREN) + { + BoolExprType type; + BoolExprType parentType; + + type = ((BoolExpr *) node)->boolop; + parentType = ((BoolExpr *) parentNode)->boolop; + switch (type) + { + case NOT_EXPR: + case AND_EXPR: + if (parentType == AND_EXPR || parentType == OR_EXPR) + return true; + break; + case OR_EXPR: + if (parentType == OR_EXPR) + return true; + break; + } + } + return false; + case T_FuncExpr: + { + /* special handling for casts and COERCE_SQL_SYNTAX */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST || + type == COERCE_SQL_SYNTAX) + return false; + return true; /* own parentheses */ + } + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_GroupingFunc: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + case T_JsonExpr: /* own parentheses */ + return true; + default: + return false; + } + + case T_JsonValueExpr: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((JsonValueExpr *) node)->raw_expr, + node, prettyFlags); + + default: + break; + } + /* those we don't know: in dubio complexo */ + return false; +} + +/* + * appendContextKeyword - append a keyword to buffer + * + * If prettyPrint is enabled, perform a line break, and adjust indentation. + * Otherwise, just append the keyword. + */ +static void +appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus) +{ + StringInfo buf = context->buf; + + if (PRETTY_INDENT(context)) + { + int indentAmount; + + context->indentLevel += indentBefore; + + /* remove any trailing spaces currently in the buffer ... */ + removeStringInfoSpaces(buf); + /* ... then add a newline and some spaces */ + appendStringInfoChar(buf, '\n'); + + if (context->indentLevel < PRETTYINDENT_LIMIT) + indentAmount = Max(context->indentLevel, 0) + indentPlus; + else + { + /* + * If we're indented more than PRETTYINDENT_LIMIT characters, try + * to conserve horizontal space by reducing the per-level + * indentation. For best results the scale factor here should + * divide all the indent amounts that get added to indentLevel + * (PRETTYINDENT_STD, etc). It's important that the indentation + * not grow unboundedly, else deeply-nested trees use O(N^2) + * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. + */ + indentAmount = PRETTYINDENT_LIMIT + + (context->indentLevel - PRETTYINDENT_LIMIT) / + (PRETTYINDENT_STD / 2); + indentAmount %= PRETTYINDENT_LIMIT; + /* scale/wrap logic affects indentLevel, but not indentPlus */ + indentAmount += indentPlus; + } + appendStringInfoSpaces(buf, indentAmount); + + appendStringInfoString(buf, str); + + context->indentLevel += indentAfter; + if (context->indentLevel < 0) + context->indentLevel = 0; + } + else + appendStringInfoString(buf, str); +} + +/* + * removeStringInfoSpaces - delete trailing spaces from a buffer. + * + * Possibly this should move to stringinfo.c at some point. + */ +static void +removeStringInfoSpaces(StringInfo str) +{ + while (str->len > 0 && str->data[str->len - 1] == ' ') + str->data[--(str->len)] = '\0'; +} + +/* + * get_rule_expr_paren - deparse expr using get_rule_expr, + * embracing the string with parentheses if necessary for prettyPrint. + * + * Never embrace if prettyFlags=0, because it's done in the calling node. + * + * Any node that does *not* embrace its argument node by sql syntax (with + * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should + * use get_rule_expr_paren instead of get_rule_expr so parentheses can be + * added. + */ +static void +get_rule_expr_paren(Node *node, deparse_context *context, + bool showimplicit, Node *parentNode) +{ + bool need_paren; + + need_paren = PRETTY_PAREN(context) && + !isSimpleNode(node, parentNode, context->prettyFlags); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(node, context, showimplicit); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); +} + +static void +get_json_behavior(JsonBehavior *behavior, deparse_context *context, + const char *on) +{ + /* + * The order of array elements must correspond to the order of + * JsonBehaviorType members. + */ + const char *behavior_names[] = + { + " NULL", + " ERROR", + " EMPTY", + " TRUE", + " FALSE", + " UNKNOWN", + " EMPTY ARRAY", + " EMPTY OBJECT", + " DEFAULT " + }; + + if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names)) + elog(ERROR, "invalid json behavior type: %d", behavior->btype); + + appendStringInfoString(context->buf, behavior_names[behavior->btype]); + + if (behavior->btype == JSON_BEHAVIOR_DEFAULT) + get_rule_expr(behavior->expr, context, false); + + appendStringInfo(context->buf, " ON %s", on); +} + +/* + * get_json_expr_options + * + * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and + * JSON_TABLE columns. + */ +static void +get_json_expr_options(JsonExpr *jsexpr, deparse_context *context, + JsonBehaviorType default_behavior) +{ + if (jsexpr->op == JSON_QUERY_OP) + { + if (jsexpr->wrapper == JSW_CONDITIONAL) + appendStringInfoString(context->buf, " WITH CONDITIONAL WRAPPER"); + else if (jsexpr->wrapper == JSW_UNCONDITIONAL) + appendStringInfoString(context->buf, " WITH UNCONDITIONAL WRAPPER"); + /* The default */ + else if (jsexpr->wrapper == JSW_NONE || jsexpr->wrapper == JSW_UNSPEC) + appendStringInfoString(context->buf, " WITHOUT WRAPPER"); + + if (jsexpr->omit_quotes) + appendStringInfoString(context->buf, " OMIT QUOTES"); + /* The default */ + else + appendStringInfoString(context->buf, " KEEP QUOTES"); + } + + if (jsexpr->on_empty && jsexpr->on_empty->btype != default_behavior) + get_json_behavior(jsexpr->on_empty, context, "EMPTY"); + + if (jsexpr->on_error && jsexpr->on_error->btype != default_behavior) + get_json_behavior(jsexpr->on_error, context, "ERROR"); +} + +/* ---------- + * get_rule_expr - Parse back an expression + * + * Note: showimplicit determines whether we display any implicit cast that + * is present at the top of the expression tree. It is a passed argument, + * not a field of the context struct, because we change the value as we + * recurse down into the expression. In general we suppress implicit casts + * when the result type is known with certainty (eg, the arguments of an + * OR must be boolean). We display implicit casts for arguments of functions + * and operators, since this is needed to be certain that the same function + * or operator will be chosen when the expression is re-parsed. + * ---------- + */ +static void +get_rule_expr(Node *node, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + + if (node == NULL) + return; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Each level of get_rule_expr must emit an indivisible term + * (parenthesized if necessary) to ensure result is reparsed into the same + * expression tree. The only exception is that when the input is a List, + * we emit the component items comma-separated with no surrounding + * decoration; this is convenient for most callers. + */ + switch (nodeTag(node)) + { + case T_Var: + (void) get_variable((Var *) node, 0, false, context); + break; + + case T_Const: + get_const_expr((Const *) node, context, 0); + break; + + case T_Param: + get_parameter((Param *) node, context); + break; + + case T_Aggref: + get_agg_expr((Aggref *) node, context, (Aggref *) node); + break; + + case T_GroupingFunc: + { + GroupingFunc *gexpr = (GroupingFunc *) node; + + appendStringInfoString(buf, "GROUPING("); + get_rule_expr((Node *) gexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_WindowFunc: + get_windowfunc_expr((WindowFunc *) node, context); + break; + + case T_MergeSupportFunc: + appendStringInfoString(buf, "MERGE_ACTION()"); + break; + + case T_SubscriptingRef: + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + bool need_parens; + + /* + * If the argument is a CaseTestExpr, we must be inside a + * FieldStore, ie, we are assigning to an element of an array + * within a composite column. Since we already punted on + * displaying the FieldStore's target information, just punt + * here too, and display only the assignment source + * expression. + */ + if (IsA(sbsref->refexpr, CaseTestExpr)) + { + Assert(sbsref->refassgnexpr); + get_rule_expr((Node *) sbsref->refassgnexpr, + context, showimplicit); + break; + } + + /* + * Parenthesize the argument unless it's a simple Var or a + * FieldSelect. (In particular, if it's another + * SubscriptingRef, we *must* parenthesize to avoid + * confusion.) + */ + need_parens = !IsA(sbsref->refexpr, Var) && + !IsA(sbsref->refexpr, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * If there's a refassgnexpr, we want to print the node in the + * format "container[subscripts] := refassgnexpr". This is + * not legal SQL, so decompilation of INSERT or UPDATE + * statements should always use processIndirection as part of + * the statement-level syntax. We should only see this when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. + */ + if (sbsref->refassgnexpr) + { + Node *refassgnexpr; + + /* + * Use processIndirection to print this node's subscripts + * as well as any additional field selections or + * subscripting in immediate descendants. It returns the + * RHS expr that is actually being "assigned". + */ + refassgnexpr = processIndirection(node, context); + appendStringInfoString(buf, " := "); + get_rule_expr(refassgnexpr, context, showimplicit); + } + else + { + /* Just an ordinary container fetch, so print subscripts */ + printSubscripts(sbsref, context); + } + } + break; + + case T_FuncExpr: + get_func_expr((FuncExpr *) node, context, showimplicit); + break; + + case T_NamedArgExpr: + { + NamedArgExpr *na = (NamedArgExpr *) node; + + appendStringInfo(buf, "%s => ", quote_identifier(na->name)); + get_rule_expr((Node *) na->arg, context, showimplicit); + } + break; + + case T_OpExpr: + get_oper_expr((OpExpr *) node, context); + break; + + case T_DistinctExpr: + { + DistinctExpr *expr = (DistinctExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfoString(buf, " IS DISTINCT FROM "); + get_rule_expr_paren(arg2, context, true, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullIfExpr: + { + NullIfExpr *nullifexpr = (NullIfExpr *) node; + + appendStringInfoString(buf, "NULLIF("); + get_rule_expr((Node *) nullifexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfo(buf, " %s %s (", + generate_operator_name(expr->opno, + exprType(arg1), + get_base_element_type(exprType(arg2))), + expr->useOr ? "ANY" : "ALL"); + get_rule_expr_paren(arg2, context, true, node); + + /* + * There's inherent ambiguity in "x op ANY/ALL (y)" when y is + * a bare sub-SELECT. Since we're here, the sub-SELECT must + * be meant as a scalar sub-SELECT yielding an array value to + * be used in ScalarArrayOpExpr; but the grammar will + * preferentially interpret such a construct as an ANY/ALL + * SubLink. To prevent misparsing the output that way, insert + * a dummy coercion (which will be stripped by parse analysis, + * so no inefficiency is added in dump and reload). This is + * indeed most likely what the user wrote to get the construct + * accepted in the first place. + */ + if (IsA(arg2, SubLink) && + ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) + appendStringInfo(buf, "::%s", + format_type_with_typemod(exprType(arg2), + exprTypmod(arg2))); + appendStringInfoChar(buf, ')'); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *) node; + Node *first_arg = linitial(expr->args); + ListCell *arg; + + switch (expr->boolop) + { + case AND_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + for_each_from(arg, expr->args, 1) + { + appendStringInfoString(buf, " AND "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case OR_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + for_each_from(arg, expr->args, 1) + { + appendStringInfoString(buf, " OR "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case NOT_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + appendStringInfoString(buf, "NOT "); + get_rule_expr_paren(first_arg, context, + false, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + default: + elog(ERROR, "unrecognized boolop: %d", + (int) expr->boolop); + } + } + break; + + case T_SubLink: + get_sublink_expr((SubLink *) node, context); + break; + + case T_SubPlan: + { + SubPlan *subplan = (SubPlan *) node; + + /* + * We cannot see an already-planned subplan in rule deparsing, + * only while EXPLAINing a query plan. We don't try to + * reconstruct the original SQL, just reference the subplan + * that appears elsewhere in EXPLAIN's result. It does seem + * useful to show the subLinkType and testexpr (if any), and + * we also note whether the subplan will be hashed. + */ + switch (subplan->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS("); + Assert(subplan->testexpr == NULL); + break; + case ALL_SUBLINK: + appendStringInfoString(buf, "(ALL "); + Assert(subplan->testexpr != NULL); + break; + case ANY_SUBLINK: + appendStringInfoString(buf, "(ANY "); + Assert(subplan->testexpr != NULL); + break; + case ROWCOMPARE_SUBLINK: + /* Parenthesizing the testexpr seems sufficient */ + appendStringInfoChar(buf, '('); + Assert(subplan->testexpr != NULL); + break; + case EXPR_SUBLINK: + /* No need to decorate these subplan references */ + appendStringInfoChar(buf, '('); + Assert(subplan->testexpr == NULL); + break; + case MULTIEXPR_SUBLINK: + /* MULTIEXPR isn't executed in the normal way */ + appendStringInfoString(buf, "(rescan "); + Assert(subplan->testexpr == NULL); + break; + case ARRAY_SUBLINK: + appendStringInfoString(buf, "ARRAY("); + Assert(subplan->testexpr == NULL); + break; + case CTE_SUBLINK: + /* This case is unreachable within expressions */ + appendStringInfoString(buf, "CTE("); + Assert(subplan->testexpr == NULL); + break; + } + + if (subplan->testexpr != NULL) + { + deparse_namespace *dpns; + + /* + * Push SubPlan into ancestors list while deparsing + * testexpr, so that we can handle PARAM_EXEC references + * to the SubPlan's paramIds. (This makes it look like + * the SubPlan is an "ancestor" of the current plan node, + * which is a little weird, but it does no harm.) In this + * path, we don't need to mention the SubPlan explicitly, + * because the referencing Params will show its existence. + */ + dpns = (deparse_namespace *) linitial(context->namespaces); + dpns->ancestors = lcons(subplan, dpns->ancestors); + + get_rule_expr(subplan->testexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + dpns->ancestors = list_delete_first(dpns->ancestors); + } + else + { + /* No referencing Params, so show the SubPlan's name */ + if (subplan->useHashTable) + appendStringInfo(buf, "hashed %s)", subplan->plan_name); + else + appendStringInfo(buf, "%s)", subplan->plan_name); + } + } + break; + + case T_AlternativeSubPlan: + { + AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; + ListCell *lc; + + /* + * This case cannot be reached in normal usage, since no + * AlternativeSubPlan can appear either in parsetrees or + * finished plan trees. We keep it just in case somebody + * wants to use this code to print planner data structures. + */ + appendStringInfoString(buf, "(alternatives: "); + foreach(lc, asplan->subplans) + { + SubPlan *splan = lfirst_node(SubPlan, lc); + + if (splan->useHashTable) + appendStringInfo(buf, "hashed %s", splan->plan_name); + else + appendStringInfoString(buf, splan->plan_name); + if (lnext(asplan->subplans, lc)) + appendStringInfoString(buf, " or "); + } + appendStringInfoChar(buf, ')'); + } + break; + + case T_FieldSelect: + { + FieldSelect *fselect = (FieldSelect *) node; + Node *arg = (Node *) fselect->arg; + int fno = fselect->fieldnum; + const char *fieldname; + bool need_parens; + + /* + * Parenthesize the argument unless it's an SubscriptingRef or + * another FieldSelect. Note in particular that it would be + * WRONG to not parenthesize a Var argument; simplicity is not + * the issue here, having the right number of names is. + */ + need_parens = !IsA(arg, SubscriptingRef) && + !IsA(arg, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr(arg, context, true); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * Get and print the field name. + */ + fieldname = get_name_for_var_field((Var *) arg, fno, + 0, context); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + } + break; + + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *) node; + bool need_parens; + + /* + * There is no good way to represent a FieldStore as real SQL, + * so decompilation of INSERT or UPDATE statements should + * always use processIndirection as part of the + * statement-level syntax. We should only get here when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. The plan case is even harder than + * ordinary rules would be, because the planner tries to + * collapse multiple assignments to the same field or subfield + * into one FieldStore; so we can see a list of target fields + * not just one, and the arguments could be FieldStores + * themselves. We don't bother to try to print the target + * field names; we just print the source arguments, with a + * ROW() around them if there's more than one. This isn't + * terribly complete, but it's probably good enough for + * EXPLAIN's purposes; especially since anything more would be + * either hopelessly confusing or an even poorer + * representation of what the plan is actually doing. + */ + need_parens = (list_length(fstore->newvals) != 1); + if (need_parens) + appendStringInfoString(buf, "ROW("); + get_rule_expr((Node *) fstore->newvals, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + } + break; + + case T_RelabelType: + { + RelabelType *relabel = (RelabelType *) node; + Node *arg = (Node *) relabel->arg; + + if (relabel->relabelformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + relabel->resulttype, + relabel->resulttypmod, + node); + } + } + break; + + case T_CoerceViaIO: + { + CoerceViaIO *iocoerce = (CoerceViaIO *) node; + Node *arg = (Node *) iocoerce->arg; + + if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + iocoerce->resulttype, + -1, + node); + } + } + break; + + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; + Node *arg = (Node *) acoerce->arg; + + if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + acoerce->resulttype, + acoerce->resulttypmod, + node); + } + } + break; + + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; + Node *arg = (Node *) convert->arg; + + if (convert->convertformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + convert->resulttype, -1, + node); + } + } + break; + + case T_CollateExpr: + { + CollateExpr *collate = (CollateExpr *) node; + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *) node; + ListCell *temp; + + appendContextKeyword(context, "CASE", + 0, PRETTYINDENT_VAR, 0); + if (caseexpr->arg) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) caseexpr->arg, context, true); + } + foreach(temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *) lfirst(temp); + Node *w = (Node *) when->expr; + + if (caseexpr->arg) + { + /* + * The parser should have produced WHEN clauses of the + * form "CaseTestExpr = RHS", possibly with an + * implicit coercion inserted above the CaseTestExpr. + * For accurate decompilation of rules it's essential + * that we show just the RHS. However in an + * expression that's been through the optimizer, the + * WHEN clause could be almost anything (since the + * equality operator could have been expanded into an + * inline function). If we don't recognize the form + * of the WHEN clause, just punt and display it as-is. + */ + if (IsA(w, OpExpr)) + { + List *args = ((OpExpr *) w)->args; + + if (list_length(args) == 2 && + IsA(strip_implicit_coercions(linitial(args)), + CaseTestExpr)) + w = (Node *) lsecond(args); + } + } + + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "WHEN ", + 0, 0, 0); + get_rule_expr(w, context, false); + appendStringInfoString(buf, " THEN "); + get_rule_expr((Node *) when->result, context, true); + } + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "ELSE ", + 0, 0, 0); + get_rule_expr((Node *) caseexpr->defresult, context, true); + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "END", + -PRETTYINDENT_VAR, 0, 0); + } + break; + + case T_CaseTestExpr: + { + /* + * Normally we should never get here, since for expressions + * that can contain this node type we attempt to avoid + * recursing to it. But in an optimized expression we might + * be unable to avoid that (see comments for CaseExpr). If we + * do see one, print it as CASE_TEST_EXPR. + */ + appendStringInfoString(buf, "CASE_TEST_EXPR"); + } + break; + + case T_ArrayExpr: + { + ArrayExpr *arrayexpr = (ArrayExpr *) node; + + appendStringInfoString(buf, "ARRAY["); + get_rule_expr((Node *) arrayexpr->elements, context, true); + appendStringInfoChar(buf, ']'); + + /* + * If the array isn't empty, we assume its elements are + * coerced to the desired type. If it's empty, though, we + * need an explicit coercion to the array type. + */ + if (arrayexpr->elements == NIL) + appendStringInfo(buf, "::%s", + format_type_with_typemod(arrayexpr->array_typeid, -1)); + } + break; + + case T_RowExpr: + { + RowExpr *rowexpr = (RowExpr *) node; + TupleDesc tupdesc = NULL; + ListCell *arg; + int i; + char *sep; + + /* + * If it's a named type and not RECORD, we may have to skip + * dropped columns and/or claim there are NULLs for added + * columns. + */ + if (rowexpr->row_typeid != RECORDOID) + { + tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); + Assert(list_length(rowexpr->args) <= tupdesc->natts); + } + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "ROW("); + sep = ""; + i = 0; + foreach(arg, rowexpr->args) + { + Node *e = (Node *) lfirst(arg); + + if (tupdesc == NULL || + !TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + /* Whole-row Vars need special treatment here */ + get_rule_expr_toplevel(e, context, true); + sep = ", "; + } + i++; + } + if (tupdesc != NULL) + { + while (i < tupdesc->natts) + { + if (!TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + appendStringInfoString(buf, "NULL"); + sep = ", "; + } + i++; + } + + ReleaseTupleDesc(tupdesc); + } + appendStringInfoChar(buf, ')'); + if (rowexpr->row_format == COERCE_EXPLICIT_CAST) + appendStringInfo(buf, "::%s", + format_type_with_typemod(rowexpr->row_typeid, -1)); + } + break; + + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *) node; + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. Within + * a ROW expression, whole-row Vars need special treatment, so + * use get_rule_list_toplevel. + */ + appendStringInfoString(buf, "(ROW("); + get_rule_list_toplevel(rcexpr->largs, context, true); + + /* + * We assume that the name of the first-column operator will + * do for all the rest too. This is definitely open to + * failure, eg if some but not all operators were renamed + * since the construct was parsed, but there seems no way to + * be perfect. + */ + appendStringInfo(buf, ") %s ROW(", + generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs)))); + get_rule_list_toplevel(rcexpr->rargs, context, true); + appendStringInfoString(buf, "))"); + } + break; + + case T_CoalesceExpr: + { + CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; + + appendStringInfoString(buf, "COALESCE("); + get_rule_expr((Node *) coalesceexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_MinMaxExpr: + { + MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; + + switch (minmaxexpr->op) + { + case IS_GREATEST: + appendStringInfoString(buf, "GREATEST("); + break; + case IS_LEAST: + appendStringInfoString(buf, "LEAST("); + break; + } + get_rule_expr((Node *) minmaxexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *) node; + bool needcomma = false; + ListCell *arg; + ListCell *narg; + Const *con; + + switch (xexpr->op) + { + case IS_XMLCONCAT: + appendStringInfoString(buf, "XMLCONCAT("); + break; + case IS_XMLELEMENT: + appendStringInfoString(buf, "XMLELEMENT("); + break; + case IS_XMLFOREST: + appendStringInfoString(buf, "XMLFOREST("); + break; + case IS_XMLPARSE: + appendStringInfoString(buf, "XMLPARSE("); + break; + case IS_XMLPI: + appendStringInfoString(buf, "XMLPI("); + break; + case IS_XMLROOT: + appendStringInfoString(buf, "XMLROOT("); + break; + case IS_XMLSERIALIZE: + appendStringInfoString(buf, "XMLSERIALIZE("); + break; + case IS_DOCUMENT: + break; + } + if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) + { + if (xexpr->xmloption == XMLOPTION_DOCUMENT) + appendStringInfoString(buf, "DOCUMENT "); + else + appendStringInfoString(buf, "CONTENT "); + } + if (xexpr->name) + { + appendStringInfo(buf, "NAME %s", + quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); + needcomma = true; + } + if (xexpr->named_args) + { + if (xexpr->op != IS_XMLFOREST) + { + if (needcomma) + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, "XMLATTRIBUTES("); + needcomma = false; + } + forboth(arg, xexpr->named_args, narg, xexpr->arg_names) + { + Node *e = (Node *) lfirst(arg); + char *argname = strVal(lfirst(narg)); + + if (needcomma) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) e, context, true); + appendStringInfo(buf, " AS %s", + quote_identifier(map_xml_name_to_sql_identifier(argname))); + needcomma = true; + } + if (xexpr->op != IS_XMLFOREST) + appendStringInfoChar(buf, ')'); + } + if (xexpr->args) + { + if (needcomma) + appendStringInfoString(buf, ", "); + switch (xexpr->op) + { + case IS_XMLCONCAT: + case IS_XMLELEMENT: + case IS_XMLFOREST: + case IS_XMLPI: + case IS_XMLSERIALIZE: + /* no extra decoration needed */ + get_rule_expr((Node *) xexpr->args, context, true); + break; + case IS_XMLPARSE: + Assert(list_length(xexpr->args) == 2); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + con = lsecond_node(Const, xexpr->args); + Assert(!con->constisnull); + if (DatumGetBool(con->constvalue)) + appendStringInfoString(buf, + " PRESERVE WHITESPACE"); + else + appendStringInfoString(buf, + " STRIP WHITESPACE"); + break; + case IS_XMLROOT: + Assert(list_length(xexpr->args) == 3); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + appendStringInfoString(buf, ", VERSION "); + con = (Const *) lsecond(xexpr->args); + if (IsA(con, Const) && + con->constisnull) + appendStringInfoString(buf, "NO VALUE"); + else + get_rule_expr((Node *) con, context, false); + + con = lthird_node(Const, xexpr->args); + if (con->constisnull) + /* suppress STANDALONE NO VALUE */ ; + else + { + switch (DatumGetInt32(con->constvalue)) + { + case XML_STANDALONE_YES: + appendStringInfoString(buf, + ", STANDALONE YES"); + break; + case XML_STANDALONE_NO: + appendStringInfoString(buf, + ", STANDALONE NO"); + break; + case XML_STANDALONE_NO_VALUE: + appendStringInfoString(buf, + ", STANDALONE NO VALUE"); + break; + default: + break; + } + } + break; + case IS_DOCUMENT: + get_rule_expr_paren((Node *) xexpr->args, context, false, node); + break; + } + } + if (xexpr->op == IS_XMLSERIALIZE) + appendStringInfo(buf, " AS %s", + format_type_with_typemod(xexpr->type, + xexpr->typmod)); + if (xexpr->op == IS_DOCUMENT) + appendStringInfoString(buf, " IS DOCUMENT"); + else + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullTest: + { + NullTest *ntest = (NullTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) ntest->arg, context, true, node); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, + * which is shorter and traditional. If it's a rowtype input + * but we're applying a scalar test, must print IS [NOT] + * DISTINCT FROM NULL to be semantically correct. + */ + if (ntest->argisrow || + !type_is_rowtype(exprType((Node *) ntest->arg))) + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS NOT NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + else + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS DISTINCT FROM NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BooleanTest: + { + BooleanTest *btest = (BooleanTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) btest->arg, context, false, node); + switch (btest->booltesttype) + { + case IS_TRUE: + appendStringInfoString(buf, " IS TRUE"); + break; + case IS_NOT_TRUE: + appendStringInfoString(buf, " IS NOT TRUE"); + break; + case IS_FALSE: + appendStringInfoString(buf, " IS FALSE"); + break; + case IS_NOT_FALSE: + appendStringInfoString(buf, " IS NOT FALSE"); + break; + case IS_UNKNOWN: + appendStringInfoString(buf, " IS UNKNOWN"); + break; + case IS_NOT_UNKNOWN: + appendStringInfoString(buf, " IS NOT UNKNOWN"); + break; + default: + elog(ERROR, "unrecognized booltesttype: %d", + (int) btest->booltesttype); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CoerceToDomain: + { + CoerceToDomain *ctest = (CoerceToDomain *) node; + Node *arg = (Node *) ctest->arg; + + if (ctest->coercionformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr(arg, context, false); + } + else + { + get_coercion_expr(arg, context, + ctest->resulttype, + ctest->resulttypmod, + node); + } + } + break; + + case T_CoerceToDomainValue: + appendStringInfoString(buf, "VALUE"); + break; + + case T_SetToDefault: + appendStringInfoString(buf, "DEFAULT"); + break; + + case T_CurrentOfExpr: + { + CurrentOfExpr *cexpr = (CurrentOfExpr *) node; + + if (cexpr->cursor_name) + appendStringInfo(buf, "CURRENT OF %s", + quote_identifier(cexpr->cursor_name)); + else + appendStringInfo(buf, "CURRENT OF $%d", + cexpr->cursor_param); + } + break; + + case T_NextValueExpr: + { + NextValueExpr *nvexpr = (NextValueExpr *) node; + + /* + * This isn't exactly nextval(), but that seems close enough + * for EXPLAIN's purposes. + */ + appendStringInfoString(buf, "nextval("); + simple_quote_literal(buf, + generate_relation_name(nvexpr->seqid, + NIL)); + appendStringInfoChar(buf, ')'); + } + break; + + case T_InferenceElem: + { + InferenceElem *iexpr = (InferenceElem *) node; + bool save_varprefix; + bool need_parens; + + /* + * InferenceElem can only refer to target relation, so a + * prefix is not useful, and indeed would cause parse errors. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + /* + * Parenthesize the element unless it's a simple Var or a bare + * function call. Follows pg_get_indexdef_worker(). + */ + need_parens = !IsA(iexpr->expr, Var); + if (IsA(iexpr->expr, FuncExpr) && + ((FuncExpr *) iexpr->expr)->funcformat == + COERCE_EXPLICIT_CALL) + need_parens = false; + + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) iexpr->expr, + context, false); + if (need_parens) + appendStringInfoChar(buf, ')'); + + context->varprefix = save_varprefix; + + if (iexpr->infercollid) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(iexpr->infercollid)); + + /* Add the operator class name, if not default */ + if (iexpr->inferopclass) + { + Oid inferopclass = iexpr->inferopclass; + Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); + + get_opclass_name(inferopclass, inferopcinputtype, buf); + } + } + break; + + case T_PartitionBoundSpec: + { + PartitionBoundSpec *spec = (PartitionBoundSpec *) node; + ListCell *cell; + char *sep; + + if (spec->is_default) + { + appendStringInfoString(buf, "DEFAULT"); + break; + } + + switch (spec->strategy) + { + case PARTITION_STRATEGY_HASH: + Assert(spec->modulus > 0 && spec->remainder >= 0); + Assert(spec->modulus > spec->remainder); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfo(buf, " WITH (modulus %d, remainder %d)", + spec->modulus, spec->remainder); + break; + + case PARTITION_STRATEGY_LIST: + Assert(spec->listdatums != NIL); + + appendStringInfoString(buf, "FOR VALUES IN ("); + sep = ""; + foreach(cell, spec->listdatums) + { + Const *val = lfirst_node(Const, cell); + + appendStringInfoString(buf, sep); + get_const_expr(val, context, -1); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); + break; + + case PARTITION_STRATEGY_RANGE: + Assert(spec->lowerdatums != NIL && + spec->upperdatums != NIL && + list_length(spec->lowerdatums) == + list_length(spec->upperdatums)); + + appendStringInfo(buf, "FOR VALUES FROM %s TO %s", + get_range_partbound_string(spec->lowerdatums), + get_range_partbound_string(spec->upperdatums)); + break; + + default: + elog(ERROR, "unrecognized partition strategy: %d", + (int) spec->strategy); + break; + } + } + break; + + case T_JsonValueExpr: + { + JsonValueExpr *jve = (JsonValueExpr *) node; + + get_rule_expr((Node *) jve->raw_expr, context, false); + get_json_format(jve->format, context->buf); + } + break; + + case T_JsonConstructorExpr: + get_json_constructor((JsonConstructorExpr *) node, context, false); + break; + + case T_JsonIsPredicate: + { + JsonIsPredicate *pred = (JsonIsPredicate *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(context->buf, '('); + + get_rule_expr_paren(pred->expr, context, true, node); + + appendStringInfoString(context->buf, " IS JSON"); + + /* TODO: handle FORMAT clause */ + + switch (pred->item_type) + { + case JS_TYPE_SCALAR: + appendStringInfoString(context->buf, " SCALAR"); + break; + case JS_TYPE_ARRAY: + appendStringInfoString(context->buf, " ARRAY"); + break; + case JS_TYPE_OBJECT: + appendStringInfoString(context->buf, " OBJECT"); + break; + default: + break; + } + + if (pred->unique_keys) + appendStringInfoString(context->buf, " WITH UNIQUE KEYS"); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(context->buf, ')'); + } + break; + + case T_JsonExpr: + { + JsonExpr *jexpr = (JsonExpr *) node; + + switch (jexpr->op) + { + case JSON_EXISTS_OP: + appendStringInfoString(buf, "JSON_EXISTS("); + break; + case JSON_QUERY_OP: + appendStringInfoString(buf, "JSON_QUERY("); + break; + case JSON_VALUE_OP: + appendStringInfoString(buf, "JSON_VALUE("); + break; + default: + elog(ERROR, "unrecognized JsonExpr op: %d", + (int) jexpr->op); + } + + get_rule_expr(jexpr->formatted_expr, context, showimplicit); + + appendStringInfoString(buf, ", "); + + get_json_path_spec(jexpr->path_spec, context, showimplicit); + + if (jexpr->passing_values) + { + ListCell *lc1, + *lc2; + bool needcomma = false; + + appendStringInfoString(buf, " PASSING "); + + forboth(lc1, jexpr->passing_names, + lc2, jexpr->passing_values) + { + if (needcomma) + appendStringInfoString(buf, ", "); + needcomma = true; + + get_rule_expr((Node *) lfirst(lc2), context, showimplicit); + appendStringInfo(buf, " AS %s", + ((String *) lfirst_node(String, lc1))->sval); + } + } + + if (jexpr->op != JSON_EXISTS_OP || + jexpr->returning->typid != BOOLOID) + get_json_returning(jexpr->returning, context->buf, + jexpr->op == JSON_QUERY_OP); + + get_json_expr_options(jexpr, context, + jexpr->op != JSON_EXISTS_OP ? + JSON_BEHAVIOR_NULL : + JSON_BEHAVIOR_FALSE); + + appendStringInfoChar(buf, ')'); + } + break; + + case T_List: + { + char *sep; + ListCell *l; + + sep = ""; + foreach(l, (List *) node) + { + appendStringInfoString(buf, sep); + get_rule_expr((Node *) lfirst(l), context, showimplicit); + sep = ", "; + } + } + break; + + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + + case T_CallStmt: + get_proc_expr((CallStmt *) node, context, showimplicit); + break; + + default: + elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); + break; + } +} + +/* + * get_rule_expr_toplevel - Parse back a toplevel expression + * + * Same as get_rule_expr(), except that if the expr is just a Var, we pass + * istoplevel = true not false to get_variable(). This causes whole-row Vars + * to get printed with decoration that will prevent expansion of "*". + * We need to use this in contexts such as ROW() and VALUES(), where the + * parser would expand "foo.*" appearing at top level. (In principle we'd + * use this in get_target_list() too, but that has additional worries about + * whether to print AS, so it needs to invoke get_variable() directly anyway.) + */ +static void +get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit) +{ + if (node && IsA(node, Var)) + (void) get_variable((Var *) node, 0, true, context); + else + get_rule_expr(node, context, showimplicit); +} + +/* + * get_rule_list_toplevel - Parse back a list of toplevel expressions + * + * Apply get_rule_expr_toplevel() to each element of a List. + * + * This adds commas between the expressions, but caller is responsible + * for printing surrounding decoration. + */ +static void +get_rule_list_toplevel(List *lst, deparse_context *context, + bool showimplicit) +{ + const char *sep; + ListCell *lc; + + sep = ""; + foreach(lc, lst) + { + Node *e = (Node *) lfirst(lc); + + appendStringInfoString(context->buf, sep); + get_rule_expr_toplevel(e, context, showimplicit); + sep = ", "; + } +} + +/* + * get_rule_expr_funccall - Parse back a function-call expression + * + * Same as get_rule_expr(), except that we guarantee that the output will + * look like a function call, or like one of the things the grammar treats as + * equivalent to a function call (see the func_expr_windowless production). + * This is needed in places where the grammar uses func_expr_windowless and + * you can't substitute a parenthesized a_expr. If what we have isn't going + * to look like a function call, wrap it in a dummy CAST() expression, which + * will satisfy the grammar --- and, indeed, is likely what the user wrote to + * produce such a thing. + */ +static void +get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit) +{ + if (looks_like_function(node)) + get_rule_expr(node, context, showimplicit); + else + { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "CAST("); + /* no point in showing any top-level implicit cast */ + get_rule_expr(node, context, false); + appendStringInfo(buf, " AS %s)", + format_type_with_typemod(exprType(node), + exprTypmod(node))); + } +} + +/* + * Helper function to identify node types that satisfy func_expr_windowless. + * If in doubt, "false" is always a safe answer. + */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + return false; /* probably shouldn't happen */ + switch (nodeTag(node)) + { + case T_FuncExpr: + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || + ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_JsonExpr: + /* these are all accepted by func_expr_common_subexpr */ + return true; + default: + break; + } + return false; +} + +/* + * get_oper_expr - Parse back an OpExpr node + */ +static void +get_oper_expr(OpExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid opno = expr->opno; + List *args = expr->args; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + get_rule_expr_paren(arg1, context, true, (Node *) expr); + appendStringInfo(buf, " %s ", + generate_operator_name(opno, + exprType(arg1), + exprType(arg2))); + get_rule_expr_paren(arg2, context, true, (Node *) expr); + } + else + { + /* prefix operator */ + Node *arg = (Node *) linitial(args); + + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); +} + +/* + * get_func_expr - Parse back a FuncExpr node + */ +static void +get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + bool use_variadic; + ListCell *l; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument --- unless caller wants to see implicit coercions. + */ + if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) + { + get_rule_expr_paren((Node *) linitial(expr->args), context, + false, (Node *) expr); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (expr->funcformat == COERCE_EXPLICIT_CAST || + expr->funcformat == COERCE_IMPLICIT_CAST) + { + Node *arg = linitial(expr->args); + Oid rettype = expr->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); + + get_coercion_expr(arg, context, + rettype, coercedTypmod, + (Node *) expr); + + return; + } + + /* + * If the function was called using one of the SQL spec's random special + * syntaxes, try to reproduce that. If we don't recognize the function, + * fall through. + */ + if (expr->funcformat == COERCE_SQL_SYNTAX) + { + if (get_func_sql_syntax(expr, context)) + return; + } + + + /* + * Normal function: display as proname(args). First we need to extract + * the argument datatypes. + */ + if (list_length(expr->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, expr->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(funcoid, nargs, + argnames, argtypes, + expr->funcvariadic, + &use_variadic, + context->inGroupBy)); + nargs = 0; + foreach(l, expr->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(expr->args, l) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(l), context, true); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_proc_expr - Parse back a CallStmt node + */ +static void +get_proc_expr(CallStmt *stmt, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid functionOid = stmt->funcexpr->funcid; + bool use_variadic; + Oid *argumentTypes; + List *finalArgumentList = NIL; + ListCell *argumentCell; + List *namedArgList = NIL; + int numberOfArgs = -1; + + if (!get_merged_argument_list(stmt, &namedArgList, &argumentTypes, + &finalArgumentList, &numberOfArgs)) + { + /* Nothing merged i.e. no OUT arguments */ + get_func_expr((FuncExpr *) stmt->funcexpr, context, showimplicit); + return; + } + + appendStringInfo(buf, "%s(", + generate_function_name(functionOid, numberOfArgs, + namedArgList, argumentTypes, + stmt->funcexpr->funcvariadic, + &use_variadic, + context->inGroupBy)); + int argNumber = 0; + foreach(argumentCell, finalArgumentList) + { + if (argNumber++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(finalArgumentList, argumentCell) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(argumentCell), context, true); + argNumber++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_agg_expr - Parse back an Aggref node + */ +static void +get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref) +{ + get_agg_expr_helper(aggref, context, original_aggref, NULL, NULL, + false); +} + +/* + * get_agg_expr_helper - subroutine for get_agg_expr and + * get_json_agg_constructor + */ +static void +get_agg_expr_helper(Aggref *aggref, deparse_context *context, + Aggref *original_aggref, const char *funcname, + const char *options, bool is_json_objectagg) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + bool use_variadic = false; + + /* + * For a combining aggregate, we look up and deparse the corresponding + * partial aggregate instead. This is necessary because our input + * argument list has been replaced; the new argument list always has just + * one element, which will point to a partial Aggref that supplies us with + * transition states to combine. + */ + if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) + { + TargetEntry *tle; + + + Assert(list_length(aggref->args) == 1); + tle = linitial_node(TargetEntry, aggref->args); + resolve_special_varno((Node *) tle->expr, context, + get_agg_combine_expr, original_aggref); + return; + } + + /* + * Mark as PARTIAL, if appropriate. We look to the original aggref so as + * to avoid printing this when recursing from the code just above. + */ + if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) + appendStringInfoString(buf, "PARTIAL "); + + /* Extract the argument types as seen by the parser */ + nargs = get_aggregate_argtypes(aggref, argtypes); + + if (!funcname) + funcname = generate_function_name(aggref->aggfnoid, nargs, NIL, + argtypes, aggref->aggvariadic, + &use_variadic, + context->inGroupBy); + + /* Print the aggregate name, schema-qualified if needed */ + appendStringInfo(buf, "%s(%s", funcname, + (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); + + if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) + { + /* + * Ordered-set aggregates do not use "*" syntax. Also, we needn't + * worry about inserting VARIADIC. So we can just dump the direct + * args as-is. + */ + Assert(!aggref->aggvariadic); + get_rule_expr((Node *) aggref->aggdirectargs, context, true); + Assert(aggref->aggorder != NIL); + appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + else + { + /* aggstar can be set only in zero-argument aggregates */ + if (aggref->aggstar) + appendStringInfoChar(buf, '*'); + else + { + ListCell *l; + int i; + + i = 0; + foreach(l, aggref->args) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *arg = (Node *) tle->expr; + + Assert(!IsA(arg, NamedArgExpr)); + if (tle->resjunk) + continue; + if (i++ > 0) + { + if (is_json_objectagg) + { + /* + * the ABSENT ON NULL and WITH UNIQUE args are printed + * separately, so ignore them here + */ + if (i > 2) + break; + + appendStringInfoString(buf, " : "); + } + else + appendStringInfoString(buf, ", "); + } + if (use_variadic && i == nargs) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr(arg, context, true); + } + } + + if (aggref->aggorder != NIL) + { + appendStringInfoString(buf, " ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + } + + if (options) + appendStringInfoString(buf, options); + + if (aggref->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) aggref->aggfilter, context, false); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * This is a helper function for get_agg_expr(). It's used when we deparse + * a combining Aggref; resolve_special_varno locates the corresponding partial + * Aggref and then calls this. + */ +static void +get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) +{ + Aggref *aggref; + Aggref *original_aggref = callback_arg; + + if (!IsA(node, Aggref)) + elog(ERROR, "combining Aggref does not point to an Aggref"); + + aggref = (Aggref *) node; + get_agg_expr(aggref, context, original_aggref); +} + +/* + * get_windowfunc_expr - Parse back a WindowFunc node + */ +static void +get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) +{ + get_windowfunc_expr_helper(wfunc, context, NULL, NULL, false); +} + + +/* + * get_windowfunc_expr_helper - subroutine for get_windowfunc_expr and + * get_json_agg_constructor + */ +static void +get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, + const char *funcname, const char *options, + bool is_json_objectagg) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + ListCell *l; + + if (list_length(wfunc->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, wfunc->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + if (!funcname) + funcname = generate_function_name(wfunc->winfnoid, nargs, argnames, + argtypes, false, NULL, + context->inGroupBy); + + appendStringInfo(buf, "%s(", funcname); + + /* winstar can be set only in zero-argument aggregates */ + if (wfunc->winstar) + appendStringInfoChar(buf, '*'); + else + { + if (is_json_objectagg) + { + get_rule_expr((Node *) linitial(wfunc->args), context, false); + appendStringInfoString(buf, " : "); + get_rule_expr((Node *) lsecond(wfunc->args), context, false); + } + else + get_rule_expr((Node *) wfunc->args, context, true); + } + + if (options) + appendStringInfoString(buf, options); + + if (wfunc->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) wfunc->aggfilter, context, false); + } + + appendStringInfoString(buf, ") OVER "); + + foreach(l, context->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->winref == wfunc->winref) + { + if (wc->name) + appendStringInfoString(buf, quote_identifier(wc->name)); + else + get_rule_windowspec(wc, context->targetList, context); + break; + } + } + if (l == NULL) + { + if (context->windowClause) + elog(ERROR, "could not find window clause for winref %u", + wfunc->winref); + + /* + * In EXPLAIN, we don't have window context information available, so + * we have to settle for this: + */ + appendStringInfoString(buf, "(?)"); + } +} + +/* + * get_func_sql_syntax - Parse back a SQL-syntax function call + * + * Returns true if we successfully deparsed, false if we did not + * recognize the function. + */ +static bool +get_func_sql_syntax(FuncExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + + switch (funcoid) + { + case F_TIMEZONE_INTERVAL_TIMESTAMP: + case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: + case F_TIMEZONE_INTERVAL_TIMETZ: + case F_TIMEZONE_TEXT_TIMESTAMP: + case F_TIMEZONE_TEXT_TIMESTAMPTZ: + case F_TIMEZONE_TEXT_TIMETZ: + /* AT TIME ZONE ... note reversed argument order */ + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) lsecond(expr->args), context, false, + (Node *) expr); + appendStringInfoString(buf, " AT TIME ZONE "); + get_rule_expr_paren((Node *) linitial(expr->args), context, false, + (Node *) expr); + appendStringInfoChar(buf, ')'); + return true; + + case F_TIMEZONE_TIMESTAMP: + case F_TIMEZONE_TIMESTAMPTZ: + case F_TIMEZONE_TIMETZ: + /* AT LOCAL */ + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) linitial(expr->args), context, false, + (Node *) expr); + appendStringInfoString(buf, " AT LOCAL)"); + return true; + + case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: + case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: + case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: + case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: + case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: + case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: + case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: + case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: + case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: + case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: + case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: + case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: + case F_OVERLAPS_TIME_TIME_TIME_TIME: + /* (x1, x2) OVERLAPS (y1, y2) */ + appendStringInfoString(buf, "(("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, ") OVERLAPS ("); + get_rule_expr((Node *) lthird(expr->args), context, false); + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfourth(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + + case F_EXTRACT_TEXT_DATE: + case F_EXTRACT_TEXT_TIME: + case F_EXTRACT_TEXT_TIMETZ: + case F_EXTRACT_TEXT_TIMESTAMP: + case F_EXTRACT_TEXT_TIMESTAMPTZ: + case F_EXTRACT_TEXT_INTERVAL: + /* EXTRACT (x FROM y) */ + appendStringInfoString(buf, "EXTRACT("); + { + Const *con = (Const *) linitial(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_IS_NORMALIZED: + /* IS xxx NORMALIZED */ + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) linitial(expr->args), context, false, + (Node *) expr); + appendStringInfoString(buf, " IS"); + if (list_length(expr->args) == 2) + { + Const *con = (Const *) lsecond(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfo(buf, " %s", + TextDatumGetCString(con->constvalue)); + } + appendStringInfoString(buf, " NORMALIZED)"); + return true; + + case F_PG_COLLATION_FOR: + /* COLLATION FOR */ + appendStringInfoString(buf, "COLLATION FOR ("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_NORMALIZE: + /* NORMALIZE() */ + appendStringInfoString(buf, "NORMALIZE("); + get_rule_expr((Node *) linitial(expr->args), context, false); + if (list_length(expr->args) == 2) + { + Const *con = (Const *) lsecond(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfo(buf, ", %s", + TextDatumGetCString(con->constvalue)); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_OVERLAY_BIT_BIT_INT4: + case F_OVERLAY_BIT_BIT_INT4_INT4: + case F_OVERLAY_BYTEA_BYTEA_INT4: + case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: + case F_OVERLAY_TEXT_TEXT_INT4: + case F_OVERLAY_TEXT_TEXT_INT4_INT4: + /* OVERLAY() */ + appendStringInfoString(buf, "OVERLAY("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " PLACING "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lthird(expr->args), context, false); + if (list_length(expr->args) == 4) + { + appendStringInfoString(buf, " FOR "); + get_rule_expr((Node *) lfourth(expr->args), context, false); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_POSITION_BIT_BIT: + case F_POSITION_BYTEA_BYTEA: + case F_POSITION_TEXT_TEXT: + /* POSITION() ... extra parens since args are b_expr not a_expr */ + appendStringInfoString(buf, "POSITION(("); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, ") IN ("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + + case F_SUBSTRING_BIT_INT4: + case F_SUBSTRING_BIT_INT4_INT4: + case F_SUBSTRING_BYTEA_INT4: + case F_SUBSTRING_BYTEA_INT4_INT4: + case F_SUBSTRING_TEXT_INT4: + case F_SUBSTRING_TEXT_INT4_INT4: + /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ + appendStringInfoString(buf, "SUBSTRING("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + if (list_length(expr->args) == 3) + { + appendStringInfoString(buf, " FOR "); + get_rule_expr((Node *) lthird(expr->args), context, false); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_SUBSTRING_TEXT_TEXT_TEXT: + /* SUBSTRING SIMILAR/ESCAPE */ + appendStringInfoString(buf, "SUBSTRING("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " SIMILAR "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, " ESCAPE "); + get_rule_expr((Node *) lthird(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_BTRIM_BYTEA_BYTEA: + case F_BTRIM_TEXT: + case F_BTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(BOTH"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_LTRIM_BYTEA_BYTEA: + case F_LTRIM_TEXT: + case F_LTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(LEADING"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_RTRIM_BYTEA_BYTEA: + case F_RTRIM_TEXT: + case F_RTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(TRAILING"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_SYSTEM_USER: + appendStringInfoString(buf, "SYSTEM_USER"); + return true; + + case F_XMLEXISTS: + /* XMLEXISTS ... extra parens because args are c_expr */ + appendStringInfoString(buf, "XMLEXISTS(("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + } + return false; +} + +/* ---------- + * get_coercion_expr + * + * Make a string representation of a value coerced to a specific type + * ---------- + */ +static void +get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode) +{ + StringInfo buf = context->buf; + + /* + * Since parse_coerce.c doesn't immediately collapse application of + * length-coercion functions to constants, what we'll typically see in + * such cases is a Const with typmod -1 and a length-coercion function + * right above it. Avoid generating redundant output. However, beware of + * suppressing casts when the user actually wrote something like + * 'foo'::text::char(3). + * + * Note: it might seem that we are missing the possibility of needing to + * print a COLLATE clause for such a Const. However, a Const could only + * have nondefault collation in a post-constant-folding tree, in which the + * length coercion would have been folded too. See also the special + * handling of CollateExpr in coerce_to_target_type(): any collation + * marking will be above the coercion node, not below it. + */ + if (arg && IsA(arg, Const) && + ((Const *) arg)->consttype == resulttype && + ((Const *) arg)->consttypmod == -1) + { + /* Show the constant without normal ::typename decoration */ + get_const_expr((Const *) arg, context, -1); + } + else + { + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, false, parentNode); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + appendStringInfo(buf, "::%s", + format_type_with_typemod(resulttype, resulttypmod)); +} + +/* ---------- + * get_const_expr + * + * Make a string representation of a Const + * + * showtype can be -1 to never show "::typename" decoration, or +1 to always + * show it, or 0 to show it only if the constant wouldn't be assumed to be + * the right type by default. + * + * If the Const's collation isn't default for its type, show that too. + * We mustn't do this when showtype is -1 (since that means the caller will + * print "::typename", and we can't put a COLLATE clause in between). It's + * caller's responsibility that collation isn't missed in such cases. + * ---------- + */ +static void +get_const_expr(Const *constval, deparse_context *context, int showtype) +{ + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char *extval; + bool needlabel = false; + + if (constval->constisnull) + { + /* + * Always label the type of a NULL constant to prevent misdecisions + * about type when reparsing. + */ + appendStringInfoString(buf, "NULL"); + if (showtype >= 0) + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + get_const_collation(constval, context); + } + return; + } + + getTypeOutputInfo(constval->consttype, + &typoutput, &typIsVarlena); + + extval = OidOutputFunctionCall(typoutput, constval->constvalue); + + switch (constval->consttype) + { + case INT4OID: + + /* + * INT4 can be printed without any decoration, unless it is + * negative; in that case print it as '-nnn'::integer to ensure + * that the output will re-parse as a constant, not as a constant + * plus operator. In most cases we could get away with printing + * (-nnn) instead, because of the way that gram.y handles negative + * literals; but that doesn't work for INT_MIN, and it doesn't + * seem that much prettier anyway. + */ + if (extval[0] != '-') + appendStringInfoString(buf, extval); + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case NUMERICOID: + + /* + * NUMERIC can be printed without quotes if it looks like a float + * constant (not an integer, and not Infinity or NaN) and doesn't + * have a leading sign (for the same reason as for INT4). + */ + if (isdigit((unsigned char) extval[0]) && + strcspn(extval, "eE.") != strlen(extval)) + { + appendStringInfoString(buf, extval); + } + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + + case BOOLOID: + if (strcmp(extval, "t") == 0) + appendStringInfoString(buf, "true"); + else + appendStringInfoString(buf, "false"); + break; + + default: + simple_quote_literal(buf, extval); + break; + } + + pfree(extval); + + if (showtype < 0) + return; + + /* + * For showtype == 0, append ::typename unless the constant will be + * implicitly typed as the right type when it is read in. + * + * XXX this code has to be kept in sync with the behavior of the parser, + * especially make_const. + */ + switch (constval->consttype) + { + case BOOLOID: + case UNKNOWNOID: + /* These types can be left unlabeled */ + needlabel = false; + break; + case INT4OID: + /* We determined above whether a label is needed */ + break; + case NUMERICOID: + + /* + * Float-looking constants will be typed as numeric, which we + * checked above; but if there's a nondefault typmod we need to + * show it. + */ + needlabel |= (constval->consttypmod >= 0); + break; + default: + needlabel = true; + break; + } + if (needlabel || showtype > 0) + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + + get_const_collation(constval, context); +} + +/* + * helper for get_const_expr: append COLLATE if needed + */ +static void +get_const_collation(Const *constval, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (OidIsValid(constval->constcollid)) + { + Oid typcollation = get_typcollation(constval->consttype); + + if (constval->constcollid != typcollation) + { + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(constval->constcollid)); + } + } +} + +/* + * get_json_path_spec - Parse back a JSON path specification + */ +static void +get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit) +{ + if (IsA(path_spec, Const)) + get_const_expr((Const *) path_spec, context, -1); + else + get_rule_expr(path_spec, context, showimplicit); +} + +/* + * get_json_format - Parse back a JsonFormat node + */ +static void +get_json_format(JsonFormat *format, StringInfo buf) +{ + if (format->format_type == JS_FORMAT_DEFAULT) + return; + + appendStringInfoString(buf, + format->format_type == JS_FORMAT_JSONB ? + " FORMAT JSONB" : " FORMAT JSON"); + + if (format->encoding != JS_ENC_DEFAULT) + { + const char *encoding; + + encoding = + format->encoding == JS_ENC_UTF16 ? "UTF16" : + format->encoding == JS_ENC_UTF32 ? "UTF32" : "UTF8"; + + appendStringInfo(buf, " ENCODING %s", encoding); + } +} + +/* + * get_json_returning - Parse back a JsonReturning structure + */ +static void +get_json_returning(JsonReturning *returning, StringInfo buf, + bool json_format_by_default) +{ + if (!OidIsValid(returning->typid)) + return; + + appendStringInfo(buf, " RETURNING %s", + format_type_with_typemod(returning->typid, + returning->typmod)); + + if (!json_format_by_default || + returning->format->format_type != + (returning->typid == JSONBOID ? JS_FORMAT_JSONB : JS_FORMAT_JSON)) + get_json_format(returning->format, buf); +} + +/* + * get_json_constructor - Parse back a JsonConstructorExpr node + */ +static void +get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + const char *funcname; + bool is_json_object; + int curridx; + ListCell *lc; + + if (ctor->type == JSCTOR_JSON_OBJECTAGG) + { + get_json_agg_constructor(ctor, context, "JSON_OBJECTAGG", true); + return; + } + else if (ctor->type == JSCTOR_JSON_ARRAYAGG) + { + get_json_agg_constructor(ctor, context, "JSON_ARRAYAGG", false); + return; + } + + switch (ctor->type) + { + case JSCTOR_JSON_OBJECT: + funcname = "JSON_OBJECT"; + break; + case JSCTOR_JSON_ARRAY: + funcname = "JSON_ARRAY"; + break; + case JSCTOR_JSON_PARSE: + funcname = "JSON"; + break; + case JSCTOR_JSON_SCALAR: + funcname = "JSON_SCALAR"; + break; + case JSCTOR_JSON_SERIALIZE: + funcname = "JSON_SERIALIZE"; + break; + default: + elog(ERROR, "invalid JsonConstructorType %d", ctor->type); + } + + appendStringInfo(buf, "%s(", funcname); + + is_json_object = ctor->type == JSCTOR_JSON_OBJECT; + foreach(lc, ctor->args) + { + curridx = foreach_current_index(lc); + if (curridx > 0) + { + const char *sep; + + sep = (is_json_object && (curridx % 2) != 0) ? " : " : ", "; + appendStringInfoString(buf, sep); + } + + get_rule_expr((Node *) lfirst(lc), context, true); + } + + get_json_constructor_options(ctor, buf); + appendStringInfoChar(buf, ')'); +} + +/* + * Append options, if any, to the JSON constructor being deparsed + */ +static void +get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf) +{ + if (ctor->absent_on_null) + { + if (ctor->type == JSCTOR_JSON_OBJECT || + ctor->type == JSCTOR_JSON_OBJECTAGG) + appendStringInfoString(buf, " ABSENT ON NULL"); + } + else + { + if (ctor->type == JSCTOR_JSON_ARRAY || + ctor->type == JSCTOR_JSON_ARRAYAGG) + appendStringInfoString(buf, " NULL ON NULL"); + } + + if (ctor->unique) + appendStringInfoString(buf, " WITH UNIQUE KEYS"); + + /* + * Append RETURNING clause if needed; JSON() and JSON_SCALAR() don't + * support one. + */ + if (ctor->type != JSCTOR_JSON_PARSE && ctor->type != JSCTOR_JSON_SCALAR) + get_json_returning(ctor->returning, buf, true); +} + +/* + * get_json_agg_constructor - Parse back an aggregate JsonConstructorExpr node + */ +static void +get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, + const char *funcname, bool is_json_objectagg) +{ + StringInfoData options; + + initStringInfo(&options); + get_json_constructor_options(ctor, &options); + + if (IsA(ctor->func, Aggref)) + get_agg_expr_helper((Aggref *) ctor->func, context, + (Aggref *) ctor->func, + funcname, options.data, is_json_objectagg); + else if (IsA(ctor->func, WindowFunc)) + get_windowfunc_expr_helper((WindowFunc *) ctor->func, context, + funcname, options.data, + is_json_objectagg); + else + elog(ERROR, "invalid JsonConstructorExpr underlying node type: %d", + nodeTag(ctor->func)); +} + +/* + * simple_quote_literal - Format a string as a SQL literal, append to buf + */ +static void +simple_quote_literal(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * We form the string literal according to the prevailing setting of + * standard_conforming_strings; we never use E''. User is responsible for + * making sure result is used correctly. + */ + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} + +/* ---------- + * get_sublink_expr - Parse back a sublink + * ---------- + */ +static void +get_sublink_expr(SubLink *sublink, deparse_context *context) +{ + StringInfo buf = context->buf; + Query *query = (Query *) (sublink->subselect); + char *opname = NULL; + bool need_paren; + + if (sublink->subLinkType == ARRAY_SUBLINK) + appendStringInfoString(buf, "ARRAY("); + else + appendStringInfoChar(buf, '('); + + /* + * Note that we print the name of only the first operator, when there are + * multiple combining operators. This is an approximation that could go + * wrong in various scenarios (operators in different schemas, renamed + * operators, etc) but there is not a whole lot we can do about it, since + * the syntax allows only one operator to be shown. + */ + if (sublink->testexpr) + { + if (IsA(sublink->testexpr, OpExpr)) + { + /* single combining operator */ + OpExpr *opexpr = (OpExpr *) sublink->testexpr; + + get_rule_expr(linitial(opexpr->args), context, true); + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + } + else if (IsA(sublink->testexpr, BoolExpr)) + { + /* multiple combining operators, = or <> cases */ + char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(l, ((BoolExpr *) sublink->testexpr)->args) + { + OpExpr *opexpr = lfirst_node(OpExpr, l); + + appendStringInfoString(buf, sep); + get_rule_expr(linitial(opexpr->args), context, true); + if (!opname) + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else if (IsA(sublink->testexpr, RowCompareExpr)) + { + /* multiple combining operators, < <= > >= cases */ + RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) rcexpr->largs, context, true); + opname = generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs))); + appendStringInfoChar(buf, ')'); + } + else + elog(ERROR, "unrecognized testexpr type: %d", + (int) nodeTag(sublink->testexpr)); + } + + need_paren = true; + + switch (sublink->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS "); + break; + + case ANY_SUBLINK: + if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ + appendStringInfoString(buf, " IN "); + else + appendStringInfo(buf, " %s ANY ", opname); + break; + + case ALL_SUBLINK: + appendStringInfo(buf, " %s ALL ", opname); + break; + + case ROWCOMPARE_SUBLINK: + appendStringInfo(buf, " %s ", opname); + break; + + case EXPR_SUBLINK: + case MULTIEXPR_SUBLINK: + case ARRAY_SUBLINK: + need_paren = false; + break; + + case CTE_SUBLINK: /* shouldn't occur in a SubLink */ + default: + elog(ERROR, "unrecognized sublink type: %d", + (int) sublink->subLinkType); + break; + } + + if (need_paren) + appendStringInfoChar(buf, '('); + + get_query_def(query, buf, context->namespaces, NULL, false, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + + if (need_paren) + appendStringInfoString(buf, "))"); + else + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_xmltable - Parse back a XMLTABLE function + * ---------- + */ +static void +get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + appendStringInfoString(buf, " COLUMNS "); + forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, + l4, tf->colexprs, l5, tf->coldefexprs) + { + char *colname = strVal(lfirst(l1)); + Oid typid = lfirst_oid(l2); + int32 typmod = lfirst_int(l3); + Node *colexpr = (Node *) lfirst(l4); + Node *coldefexpr = (Node *) lfirst(l5); + bool ordinality = (tf->ordinalitycol == colnum); + bool notnull = bms_is_member(colnum, tf->notnulls); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_json_table_nested_columns - Parse back nested JSON_TABLE columns + */ +static void +get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, + deparse_context *context, bool showimplicit, + bool needcomma) +{ + if (IsA(plan, JsonTablePathScan)) + { + JsonTablePathScan *scan = castNode(JsonTablePathScan, plan); + + if (needcomma) + appendStringInfoChar(context->buf, ','); + + appendStringInfoChar(context->buf, ' '); + appendContextKeyword(context, "NESTED PATH ", 0, 0, 0); + get_const_expr(scan->path->value, context, -1); + appendStringInfo(context->buf, " AS %s", quote_identifier(scan->path->name)); + get_json_table_columns(tf, scan, context, showimplicit); + } + else if (IsA(plan, JsonTableSiblingJoin)) + { + JsonTableSiblingJoin *join = (JsonTableSiblingJoin *) plan; + + get_json_table_nested_columns(tf, join->lplan, context, showimplicit, + needcomma); + get_json_table_nested_columns(tf, join->rplan, context, showimplicit, + true); + } +} + +/* + * get_json_table_columns - Parse back JSON_TABLE columns + */ +static void +get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, + deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + ListCell *lc_colname; + ListCell *lc_coltype; + ListCell *lc_coltypmod; + ListCell *lc_colvalexpr; + int colnum = 0; + + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "COLUMNS (", 0, 0, 0); + + if (PRETTY_INDENT(context)) + context->indentLevel += PRETTYINDENT_VAR; + + forfour(lc_colname, tf->colnames, + lc_coltype, tf->coltypes, + lc_coltypmod, tf->coltypmods, + lc_colvalexpr, tf->colvalexprs) + { + char *colname = strVal(lfirst(lc_colname)); + JsonExpr *colexpr; + Oid typid; + int32 typmod; + bool ordinality; + JsonBehaviorType default_behavior; + + typid = lfirst_oid(lc_coltype); + typmod = lfirst_int(lc_coltypmod); + colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr)); + + /* Skip columns that don't belong to this scan. */ + if (scan->colMin < 0 || colnum < scan->colMin) + { + colnum++; + continue; + } + if (colnum > scan->colMax) + break; + + if (colnum > scan->colMin) + appendStringInfoString(buf, ", "); + + colnum++; + + ordinality = !colexpr; + + appendContextKeyword(context, "", 0, 0, 0); + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + /* + * Set default_behavior to guide get_json_expr_options() on whether to + * to emit the ON ERROR / EMPTY clauses. + */ + if (colexpr->op == JSON_EXISTS_OP) + { + appendStringInfoString(buf, " EXISTS"); + default_behavior = JSON_BEHAVIOR_FALSE; + } + else + { + if (colexpr->op == JSON_QUERY_OP) + { + char typcategory; + bool typispreferred; + + get_type_category_preferred(typid, &typcategory, &typispreferred); + + if (typcategory == TYPCATEGORY_STRING) + appendStringInfoString(buf, + colexpr->format->format_type == JS_FORMAT_JSONB ? + " FORMAT JSONB" : " FORMAT JSON"); + } + + default_behavior = JSON_BEHAVIOR_NULL; + } + + appendStringInfoString(buf, " PATH "); + + get_json_path_spec(colexpr->path_spec, context, showimplicit); + + get_json_expr_options(colexpr, context, default_behavior); + } + + if (scan->child) + get_json_table_nested_columns(tf, scan->child, context, showimplicit, + scan->colMin >= 0); + + if (PRETTY_INDENT(context)) + context->indentLevel -= PRETTYINDENT_VAR; + + appendContextKeyword(context, ")", 0, 0, 0); +} + +/* ---------- + * get_json_table - Parse back a JSON_TABLE function + * ---------- + */ +static void +get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr); + JsonTablePathScan *root = castNode(JsonTablePathScan, tf->plan); + + appendStringInfoString(buf, "JSON_TABLE("); + + if (PRETTY_INDENT(context)) + context->indentLevel += PRETTYINDENT_VAR; + + appendContextKeyword(context, "", 0, 0, 0); + + get_rule_expr(jexpr->formatted_expr, context, showimplicit); + + appendStringInfoString(buf, ", "); + + get_const_expr(root->path->value, context, -1); + + appendStringInfo(buf, " AS %s", quote_identifier(root->path->name)); + + if (jexpr->passing_values) + { + ListCell *lc1, + *lc2; + bool needcomma = false; + + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "PASSING ", 0, 0, 0); + + if (PRETTY_INDENT(context)) + context->indentLevel += PRETTYINDENT_VAR; + + forboth(lc1, jexpr->passing_names, + lc2, jexpr->passing_values) + { + if (needcomma) + appendStringInfoString(buf, ", "); + needcomma = true; + + appendContextKeyword(context, "", 0, 0, 0); + + get_rule_expr((Node *) lfirst(lc2), context, false); + appendStringInfo(buf, " AS %s", + quote_identifier((lfirst_node(String, lc1))->sval) + ); + } + + if (PRETTY_INDENT(context)) + context->indentLevel -= PRETTYINDENT_VAR; + } + + get_json_table_columns(tf, castNode(JsonTablePathScan, tf->plan), context, + showimplicit); + + if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY_ARRAY) + get_json_behavior(jexpr->on_error, context, "ERROR"); + + if (PRETTY_INDENT(context)) + context->indentLevel -= PRETTYINDENT_VAR; + + appendContextKeyword(context, ")", 0, 0, 0); +} + +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + /* XMLTABLE and JSON_TABLE are the only existing implementations. */ + + if (tf->functype == TFT_XMLTABLE) + get_xmltable(tf, context, showimplicit); + else if (tf->functype == TFT_JSON_TABLE) + get_json_table(tf, context, showimplicit); +} + +/* ---------- + * get_from_clause - Parse back a FROM clause + * + * "prefix" is the keyword that denotes the start of the list of FROM + * elements. It is FROM when used to parse back SELECT and UPDATE, but + * is USING when parsing back DELETE. + * ---------- + */ +static void +get_from_clause(Query *query, const char *prefix, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first = true; + ListCell *l; + + /* + * We use the query's jointree as a guide to what to print. However, we + * must ignore auto-added RTEs that are marked not inFromCl. (These can + * only appear at the top level of the jointree, so it's sufficient to + * check here.) This check also ensures we ignore the rule pseudo-RTEs + * for NEW and OLD. + */ + foreach(l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + + if (!rte->inFromCl) + continue; + } + + if (first) + { + appendContextKeyword(context, prefix, + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + first = false; + + get_from_clause_item(jtnode, query, context); + } + else + { + StringInfoData itembuf; + + appendStringInfoString(buf, ", "); + + /* + * Put the new FROM item's text into itembuf so we can decide + * after we've got it whether or not it needs to go on a new line. + */ + initStringInfo(&itembuf); + context->buf = &itembuf; + + get_from_clause_item(jtnode, query, context); + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + /* Does the new item start with a new line? */ + if (itembuf.len > 0 && itembuf.data[0] == '\n') + { + /* If so, we shouldn't add anything */ + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new item + * would cause an overflow. + */ + if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_VAR); + } + } + + /* Add the new item */ + appendStringInfoString(buf, itembuf.data); + + /* clean up */ + pfree(itembuf.data); + } + } +} + +static void +get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + RangeTblFunction *rtfunc1 = NULL; + CitusRTEKind rteKind = GetRangeTblKind(rte); + + if (rte->lateral) + appendStringInfoString(buf, "LATERAL "); + + /* Print the FROM item proper */ + switch (rte->rtekind) + { + case RTE_RELATION: + /* Normal relation RTE */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, + context->namespaces)); + break; + case RTE_SUBQUERY: + /* Subquery RTE */ + appendStringInfoChar(buf, '('); + get_query_def(rte->subquery, buf, context->namespaces, NULL, + true, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + appendStringInfoChar(buf, ')'); + break; + case RTE_FUNCTION: + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, + fragmentTableName)); + break; + } + + /* Function RTE */ + rtfunc1 = (RangeTblFunction *) linitial(rte->functions); + + /* + * Omit ROWS FROM() syntax for just one function, unless it + * has both a coldeflist and WITH ORDINALITY. If it has both, + * we must use ROWS FROM() syntax to avoid ambiguity about + * whether the coldeflist includes the ordinality column. + */ + if (list_length(rte->functions) == 1 && + (rtfunc1->funccolnames == NIL || !rte->funcordinality)) + { + get_rule_expr_funccall(rtfunc1->funcexpr, context, true); + /* we'll print the coldeflist below, if it has one */ + } + else + { + bool all_unnest; + ListCell *lc; + + /* + * If all the function calls in the list are to unnest, + * and none need a coldeflist, then collapse the list back + * down to UNNEST(args). (If we had more than one + * built-in unnest function, this would get more + * difficult.) + * + * XXX This is pretty ugly, since it makes not-terribly- + * future-proof assumptions about what the parser would do + * with the output; but the alternative is to emit our + * nonstandard ROWS FROM() notation for what might have + * been a perfectly spec-compliant multi-argument + * UNNEST(). + */ + all_unnest = true; + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (!IsA(rtfunc->funcexpr, FuncExpr) || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || + rtfunc->funccolnames != NIL) + { + all_unnest = false; + break; + } + } + + if (all_unnest) + { + List *allargs = NIL; + + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + List *args = ((FuncExpr *) rtfunc->funcexpr)->args; + + allargs = list_concat(allargs, args); + } + + appendStringInfoString(buf, "UNNEST("); + get_rule_expr((Node *) allargs, context, true); + appendStringInfoChar(buf, ')'); + } + else + { + int funcno = 0; + + appendStringInfoString(buf, "ROWS FROM("); + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (funcno > 0) + appendStringInfoString(buf, ", "); + get_rule_expr_funccall(rtfunc->funcexpr, context, true); + if (rtfunc->funccolnames != NIL) + { + /* Reconstruct the column definition list */ + appendStringInfoString(buf, " AS "); + get_from_clause_coldeflist(rtfunc, + NULL, + context); + } + funcno++; + } + appendStringInfoChar(buf, ')'); + } + /* prevent printing duplicate coldeflist below */ + rtfunc1 = NULL; + } + if (rte->funcordinality) + appendStringInfoString(buf, " WITH ORDINALITY"); + break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; + case RTE_VALUES: + /* Values list RTE */ + appendStringInfoChar(buf, '('); + get_values_def(rte->values_lists, context); + appendStringInfoChar(buf, ')'); + break; + case RTE_CTE: + appendStringInfoString(buf, quote_identifier(rte->ctename)); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + break; + } + + /* Print the relation alias, if needed */ + get_rte_alias(rte, varno, false, context); + + /* Print the column definitions or aliases, if needed */ + if (rtfunc1 && rtfunc1->funccolnames != NIL) + { + /* Reconstruct the columndef list, which is also the aliases */ + get_from_clause_coldeflist(rtfunc1, colinfo, context); + } + else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD || + (rte->alias != NULL && rte->alias->colnames != NIL)) + { + /* Else print column aliases as needed */ + get_column_alias_list(colinfo, context); + } + /* check if column's are given aliases in distributed tables */ + else if (colinfo->parentUsing != NIL) + { + Assert(colinfo->printaliases); + get_column_alias_list(colinfo, context); + } + + /* Tablesample clause must go after any alias */ + if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && + rte->tablesample) + { + get_tablesample_def(rte->tablesample, context); + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + bool need_paren_on_right; + + need_paren_on_right = PRETTY_PAREN(context) && + !IsA(j->rarg, RangeTblRef) && + !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, '('); + + get_from_clause_item(j->larg, query, context); + + switch (j->jointype) + { + case JOIN_INNER: + if (j->quals) + appendContextKeyword(context, " JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + else + appendContextKeyword(context, " CROSS JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_LEFT: + appendContextKeyword(context, " LEFT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_FULL: + appendContextKeyword(context, " FULL JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_RIGHT: + appendContextKeyword(context, " RIGHT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + default: + elog(ERROR, "unrecognized join type: %d", + (int) j->jointype); + } + + if (need_paren_on_right) + appendStringInfoChar(buf, '('); + get_from_clause_item(j->rarg, query, context); + if (need_paren_on_right) + appendStringInfoChar(buf, ')'); + + if (j->usingClause) + { + ListCell *lc; + bool first = true; + + appendStringInfoString(buf, " USING ("); + /* Use the assigned names, not what's in usingClause */ + foreach(lc, colinfo->usingNames) + { + char *colname = (char *) lfirst(lc); + + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + appendStringInfoChar(buf, ')'); + + if (j->join_using_alias) + appendStringInfo(buf, " AS %s", + quote_identifier(j->join_using_alias->aliasname)); + } + else if (j->quals) + { + appendStringInfoString(buf, " ON "); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr(j->quals, context, false); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + else if (j->jointype != JOIN_INNER) + { + /* If we didn't say CROSS JOIN above, we must provide an ON */ + appendStringInfoString(buf, " ON TRUE"); + } + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, ')'); + + /* Yes, it's correct to put alias after the right paren ... */ + if (j->alias != NULL) + { + /* + * Note that it's correct to emit an alias clause if and only if + * there was one originally. Otherwise we'd be converting a named + * join to unnamed or vice versa, which creates semantic + * subtleties we don't want. However, we might print a different + * alias name than was there originally. + */ + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(j->rtindex, + context))); + get_column_alias_list(colinfo, context); + } + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * get_rte_alias - print the relation's alias, if needed + * + * If printed, the alias is preceded by a space, or by " AS " if use_as is true. + */ +static void +get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, + deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + char *refname = get_rtable_name(varno, context); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + bool printalias = false; + + if (rte->alias != NULL) + { + /* Always print alias if user provided one */ + printalias = true; + } + else if (colinfo->printaliases) + { + /* Always print alias if we need to print column aliases */ + printalias = true; + } + else if (rte->rtekind == RTE_RELATION) + { + /* + * No need to print alias if it's same as relation name (this would + * normally be the case, but not if set_rtable_names had to resolve a + * conflict). + */ + if (strcmp(refname, get_relation_name(rte->relid)) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_FUNCTION) + { + /* + * For a function RTE, always print alias. This covers possible + * renaming of the function and/or instability of the FigureColname + * rules for things that aren't simple functions. Note we'd need to + * force it anyway for the columndef list case. + */ + printalias = true; + } + else if (rte->rtekind == RTE_SUBQUERY || + rte->rtekind == RTE_VALUES) + { + /* + * For a subquery, always print alias. This makes the output + * SQL-spec-compliant, even though we allow such aliases to be omitted + * on input. + */ + printalias = true; + } + else if (rte->rtekind == RTE_CTE) + { + /* + * No need to print alias if it's same as CTE name (this would + * normally be the case, but not if set_rtable_names had to resolve a + * conflict). + */ + if (strcmp(refname, rte->ctename) != 0) + printalias = true; + } + + if (printalias) + appendStringInfo(context->buf, "%s%s", + use_as ? " AS " : " ", + quote_identifier(refname)); +} + +/* + * get_column_alias_list - print column alias list for an RTE + * + * Caller must already have printed the relation's alias name. + */ +static void +get_column_alias_list(deparse_columns *colinfo, deparse_context *context) +{ + StringInfo buf = context->buf; + int i; + bool first = true; + + /* Don't print aliases if not needed */ + if (!colinfo->printaliases) + return; + + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *colname = colinfo->new_colnames[i]; + + if (first) + { + appendStringInfoChar(buf, '('); + first = false; + } + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + if (!first) + appendStringInfoChar(buf, ')'); +} + +/* + * get_from_clause_coldeflist - reproduce FROM clause coldeflist + * + * When printing a top-level coldeflist (which is syntactically also the + * relation's column alias list), use column names from colinfo. But when + * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the + * original coldeflist's names, which are available in rtfunc->funccolnames. + * Pass NULL for colinfo to select the latter behavior. + * + * The coldeflist is appended immediately (no space) to buf. Caller is + * responsible for ensuring that an alias or AS is present before it. + */ +static void +get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + int i; + + appendStringInfoChar(buf, '('); + + i = 0; + forfour(l1, rtfunc->funccoltypes, + l2, rtfunc->funccoltypmods, + l3, rtfunc->funccolcollations, + l4, rtfunc->funccolnames) + { + Oid atttypid = lfirst_oid(l1); + int32 atttypmod = lfirst_int(l2); + Oid attcollation = lfirst_oid(l3); + char *attname; + + if (colinfo) + attname = colinfo->colnames[i]; + else + attname = strVal(lfirst(l4)); + + Assert(attname); /* shouldn't be any dropped columns here */ + + if (i > 0) + appendStringInfoString(buf, ", "); + appendStringInfo(buf, "%s %s", + quote_identifier(attname), + format_type_with_typemod(atttypid, atttypmod)); + if (OidIsValid(attcollation) && + attcollation != get_typcollation(atttypid)) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(attcollation)); + + i++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_tablesample_def - print a TableSampleClause + */ +static void +get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[1]; + int nargs; + ListCell *l; + + /* + * We should qualify the handler's function name if it wouldn't be + * resolved by lookup in the current search path. + */ + argtypes[0] = INTERNALOID; + appendStringInfo(buf, " TABLESAMPLE %s (", + generate_function_name(tablesample->tsmhandler, 1, + NIL, argtypes, + false, NULL, false)); + + nargs = 0; + foreach(l, tablesample->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfirst(l), context, false); + } + appendStringInfoChar(buf, ')'); + + if (tablesample->repeatable != NULL) + { + appendStringInfoString(buf, " REPEATABLE ("); + get_rule_expr((Node *) tablesample->repeatable, context, false); + appendStringInfoChar(buf, ')'); + } +} + +/* + * get_opclass_name - fetch name of an index operator class + * + * The opclass name is appended (after a space) to buf. + * + * Output is suppressed if the opclass is the default for the given + * actual_datatype. (If you don't want this behavior, just pass + * InvalidOid for actual_datatype.) + */ +static void +get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf) +{ + HeapTuple ht_opc; + Form_pg_opclass opcrec; + char *opcname; + char *nspname; + + ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); + if (!HeapTupleIsValid(ht_opc)) + elog(ERROR, "cache lookup failed for opclass %u", opclass); + opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); + + if (!OidIsValid(actual_datatype) || + GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) + { + /* Okay, we need the opclass name. Do we need to qualify it? */ + opcname = NameStr(opcrec->opcname); + if (OpclassIsVisible(opclass)) + appendStringInfo(buf, " %s", quote_identifier(opcname)); + else + { + nspname = get_namespace_name_or_temp(opcrec->opcnamespace); + appendStringInfo(buf, " %s.%s", + quote_identifier(nspname), + quote_identifier(opcname)); + } + } + ReleaseSysCache(ht_opc); +} + +/* + * processIndirection - take care of array and subfield assignment + * + * We strip any top-level FieldStore or assignment SubscriptingRef nodes that + * appear in the input, printing them as decoration for the base column + * name (which we assume the caller just printed). We might also need to + * strip CoerceToDomain nodes, but only ones that appear above assignment + * nodes. + * + * Returns the subexpression that's to be assigned. + */ +static Node * +processIndirection(Node *node, deparse_context *context) +{ + StringInfo buf = context->buf; + CoerceToDomain *cdomain = NULL; + + for (;;) + { + if (node == NULL) + break; + if (IsA(node, FieldStore)) + { + FieldStore *fstore = (FieldStore *) node; + Oid typrelid; + char *fieldname; + + /* lookup tuple type */ + typrelid = get_typ_typrelid(fstore->resulttype); + if (!OidIsValid(typrelid)) + elog(ERROR, "argument type %s of FieldStore is not a tuple type", + format_type_be(fstore->resulttype)); + + /* + * Print the field name. There should only be one target field in + * stored rules. There could be more than that in executable + * target lists, but this function cannot be used for that case. + */ + Assert(list_length(fstore->fieldnums) == 1); + fieldname = get_attname(typrelid, + linitial_int(fstore->fieldnums), false); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + + /* + * We ignore arg since it should be an uninteresting reference to + * the target column or subcolumn. + */ + node = (Node *) linitial(fstore->newvals); + } + else if (IsA(node, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + + if (sbsref->refassgnexpr == NULL) + break; + printSubscripts(sbsref, context); + + /* + * We ignore refexpr since it should be an uninteresting reference + * to the target column or subcolumn. + */ + node = (Node *) sbsref->refassgnexpr; + } + else if (IsA(node, CoerceToDomain)) + { + cdomain = (CoerceToDomain *) node; + /* If it's an explicit domain coercion, we're done */ + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + /* Tentatively descend past the CoerceToDomain */ + node = (Node *) cdomain->arg; + } + else + break; + } + + /* + * If we descended past a CoerceToDomain whose argument turned out not to + * be a FieldStore or array assignment, back up to the CoerceToDomain. + * (This is not enough to be fully correct if there are nested implicit + * CoerceToDomains, but such cases shouldn't ever occur.) + */ + if (cdomain && node == (Node *) cdomain->arg) + node = (Node *) cdomain; + + return node; +} + +static void +printSubscripts(SubscriptingRef *sbsref, deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *lowlist_item; + ListCell *uplist_item; + + lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ + foreach(uplist_item, sbsref->refupperindexpr) + { + appendStringInfoChar(buf, '['); + if (lowlist_item) + { + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(lowlist_item), context, false); + appendStringInfoChar(buf, ':'); + lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); + } + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(uplist_item), context, false); + appendStringInfoChar(buf, ']'); + } +} + +/* + * get_relation_name + * Get the unqualified name of a relation specified by OID + * + * This differs from the underlying get_rel_name() function in that it will + * throw error instead of silently returning NULL if the OID is bad. + */ +static char * +get_relation_name(Oid relid) +{ + char *relname = get_rel_name(relid); + + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", relid); + return relname; +} + +/* + * generate_relation_or_shard_name + * Compute the name to display for a relation or shard + * + * If the provided relid is equal to the provided distrelid, this function + * returns a shard-extended relation name; otherwise, it falls through to a + * simple generate_relation_name call. + */ +static char * +generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, + List *namespaces) +{ + char *relname = NULL; + + if (relid == distrelid) + { + relname = get_relation_name(relid); + + if (shardid > 0) + { + Oid schemaOid = get_rel_namespace(relid); + char *schemaName = get_namespace_name_or_temp(schemaOid); + + AppendShardIdToName(&relname, shardid); + + relname = quote_qualified_identifier(schemaName, relname); + } + } + else + { + relname = generate_relation_name(relid, namespaces); + } + + return relname; +} + +/* + * generate_relation_name + * Compute the name to display for a relation specified by OID + * + * The result includes all necessary quoting and schema-prefixing. + * + * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. + * We will forcibly qualify the relation name if it equals any CTE name + * visible in the namespace list. + */ +char * +generate_relation_name(Oid relid, List *namespaces) +{ + HeapTuple tp; + Form_pg_class reltup; + bool need_qual; + ListCell *nslist; + char *relname; + char *nspname; + char *result; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for relation %u", relid); + reltup = (Form_pg_class) GETSTRUCT(tp); + relname = NameStr(reltup->relname); + + /* Check for conflicting CTE name */ + need_qual = false; + foreach(nslist, namespaces) + { + deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); + ListCell *ctlist; + + foreach(ctlist, dpns->ctes) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); + + if (strcmp(cte->ctename, relname) == 0) + { + need_qual = true; + break; + } + } + if (need_qual) + break; + } + + /* Otherwise, qualify the name if not visible in search path */ + if (!need_qual) + need_qual = !RelationIsVisible(relid); + + if (need_qual) + nspname = get_namespace_name_or_temp(reltup->relnamespace); + else + nspname = NULL; + + result = quote_qualified_identifier(nspname, relname); + + ReleaseSysCache(tp); + + return result; +} + +/* + * generate_rte_shard_name returns the qualified name of the shard given a + * CITUS_RTE_SHARD range table entry. + */ +static char * +generate_rte_shard_name(RangeTblEntry *rangeTableEntry) +{ + char *shardSchemaName = NULL; + char *shardTableName = NULL; + + Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); + + ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, + NULL); + + return generate_fragment_name(shardSchemaName, shardTableName); +} + +/* + * generate_fragment_name + * Compute the name to display for a shard or merged table + * + * The result includes all necessary quoting and schema-prefixing. The schema + * name can be NULL for regular shards. For merged tables, they are always + * declared within a job-specific schema, and therefore can't have null schema + * names. + */ +static char * +generate_fragment_name(char *schemaName, char *tableName) +{ + StringInfo fragmentNameString = makeStringInfo(); + + if (schemaName != NULL) + { + appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), + quote_identifier(tableName)); + } + else + { + appendStringInfoString(fragmentNameString, quote_identifier(tableName)); + } + + return fragmentNameString->data; +} + +/* + * generate_function_name + * Compute the name to display for a function specified by OID, + * given that it is being called with the specified actual arg names and + * types. (Those matter because of ambiguous-function resolution rules.) + * + * If we're dealing with a potentially variadic function (in practice, this + * means a FuncExpr or Aggref, not some other way of calling a function), then + * has_variadic must specify whether variadic arguments have been merged, + * and *use_variadic_p will be set to indicate whether to print VARIADIC in + * the output. For non-FuncExpr cases, has_variadic should be false and + * use_variadic_p can be NULL. + * + * inGroupBy must be true if we're deparsing a GROUP BY clause. + * + * The result includes all necessary quoting and schema-prefixing. + */ +static char * +generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + bool inGroupBy) +{ + char *result; + HeapTuple proctup; + Form_pg_proc procform; + char *proname; + bool use_variadic; + char *nspname; + FuncDetailCode p_result; + Oid p_funcid; + Oid p_rettype; + bool p_retset; + int p_nvargs; + Oid p_vatype; + Oid *p_true_typeids; + bool force_qualify = false; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + procform = (Form_pg_proc) GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* + * Due to parser hacks to avoid needing to reserve CUBE, we need to force + * qualification of some function names within GROUP BY. + */ + if (inGroupBy) + { + if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) + force_qualify = true; + } + + /* + * Determine whether VARIADIC should be printed. We must do this first + * since it affects the lookup rules in func_get_detail(). + * + * Currently, we always print VARIADIC if the function has a merged + * variadic-array argument. Note that this is always the case for + * functions taking a VARIADIC argument type other than VARIADIC ANY. + * + * In principle, if VARIADIC wasn't originally specified and the array + * actual argument is deconstructable, we could print the array elements + * separately and not print VARIADIC, thus more nearly reproducing the + * original input. For the moment that seems like too much complication + * for the benefit, and anyway we do not know whether VARIADIC was + * originally specified if it's a non-ANY type. + */ + if (use_variadic_p) + { + /* Parser should not have set funcvariadic unless fn is variadic */ + Assert(!has_variadic || OidIsValid(procform->provariadic)); + use_variadic = has_variadic; + *use_variadic_p = use_variadic; + } + else + { + Assert(!has_variadic); + use_variadic = false; + } + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct function given the unqualified func name with the + * specified argtypes and VARIADIC flag. But if we already decided to + * force qualification, then we can skip the lookup and pretend we didn't + * find it. + */ + if (!force_qualify) + p_result = func_get_detail(list_make1(makeString(proname)), + NIL, argnames, nargs, argtypes, + !use_variadic, true, false, + &p_funcid, &p_rettype, + &p_retset, &p_nvargs, &p_vatype, + &p_true_typeids, NULL); + else + { + p_result = FUNCDETAIL_NOTFOUND; + p_funcid = InvalidOid; + } + + if ((p_result == FUNCDETAIL_NORMAL || + p_result == FUNCDETAIL_AGGREGATE || + p_result == FUNCDETAIL_WINDOWFUNC) && + p_funcid == funcid) + nspname = NULL; + else + nspname = get_namespace_name_or_temp(procform->pronamespace); + + result = quote_qualified_identifier(nspname, proname); + + ReleaseSysCache(proctup); + + return result; +} + +/* + * generate_operator_name + * Compute the name to display for an operator specified by OID, + * given that it is being called with the specified actual arg types. + * (Arg types matter because of ambiguous-operator resolution rules. + * Pass InvalidOid for unused arg of a unary operator.) + * + * The result includes all necessary quoting and schema-prefixing, + * plus the OPERATOR() decoration needed to use a qualified operator name + * in an expression. + */ +char * +generate_operator_name(Oid operid, Oid arg1, Oid arg2) +{ + StringInfoData buf; + HeapTuple opertup; + Form_pg_operator operform; + char *oprname; + char *nspname; + + initStringInfo(&buf); + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); + if (!HeapTupleIsValid(opertup)) + elog(ERROR, "cache lookup failed for operator %u", operid); + operform = (Form_pg_operator) GETSTRUCT(opertup); + oprname = NameStr(operform->oprname); + + /* + * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, + * we don't check if the operator is in current namespace or not. This is + * because this check is costly when the operator is not in current namespace. + */ + nspname = get_namespace_name_or_temp(operform->oprnamespace); + Assert(nspname != NULL); + appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); + appendStringInfoString(&buf, oprname); + appendStringInfoChar(&buf, ')'); + + ReleaseSysCache(opertup); + + return buf.data; +} + +/* + * get_one_range_partition_bound_string + * A C string representation of one range partition bound + */ +char * +get_range_partbound_string(List *bound_datums) +{ + deparse_context context; + StringInfo buf = makeStringInfo(); + ListCell *cell; + char *sep; + + memset(&context, 0, sizeof(deparse_context)); + context.buf = buf; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(cell, bound_datums) + { + PartitionRangeDatum *datum = + lfirst_node(PartitionRangeDatum, cell); + + appendStringInfoString(buf, sep); + if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) + appendStringInfoString(buf, "MINVALUE"); + else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) + appendStringInfoString(buf, "MAXVALUE"); + else + { + Const *val = castNode(Const, datum->value); + + get_const_expr(val, &context, -1); + } + sep = ", "; + } + appendStringInfoChar(buf, ')'); + + return buf->data; +} + +/* + * Collect a list of OIDs of all sequences owned by the specified relation, + * and column if specified. If deptype is not zero, then only find sequences + * with the specified dependency type. + */ +List * +getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) +{ + List *result = NIL; + Relation depRel; + ScanKeyData key[3]; + SysScanDesc scan; + HeapTuple tup; + + depRel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + if (attnum) + ScanKeyInit(&key[2], + Anum_pg_depend_refobjsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(attnum)); + + scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, attnum ? 3 : 2, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); + + /* + * We assume any auto or internal dependency of a sequence on a column + * must be what we are looking for. (We need the relkind test because + * indexes can also have auto dependencies on columns.) + */ + if (deprec->classid == RelationRelationId && + deprec->objsubid == 0 && + deprec->refobjsubid != 0 && + (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && + get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) + { + if (!deptype || deprec->deptype == deptype) + result = lappend_oid(result, deprec->objid); + } + } + + systable_endscan(scan); + + table_close(depRel, AccessShareLock); + + return result; +} + +/* + * get_insert_column_names_list Prepares the insert-column-names list. Any indirection + * decoration needed on the column names can be inferred from the top targetlist. + */ +static List * +get_insert_column_names_list(List *targetList, StringInfo buf, + deparse_context *context, RangeTblEntry *rte) +{ + char *sep; + ListCell *l; + List *strippedexprs; + + strippedexprs = NIL; + sep = ""; + appendStringInfoChar(buf, '('); + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + * Add the stripped expressions to strippedexprs. (If it's a + * single-VALUES statement, the stripped expressions are the VALUES to + * print below. Otherwise they're just Vars and not really + * interesting.) + */ + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + appendStringInfoString(buf, ") "); + + return strippedexprs; +} +#endif /* (PG_VERSION_NUM >= PG_VERSION_17) && (PG_VERSION_NUM < PG_VERSION_18) */ diff --git a/src/include/pg_version_constants.h b/src/include/pg_version_constants.h index 9761dff83..ba2a9a03e 100644 --- a/src/include/pg_version_constants.h +++ b/src/include/pg_version_constants.h @@ -15,5 +15,6 @@ #define PG_VERSION_15 150000 #define PG_VERSION_16 160000 #define PG_VERSION_17 170000 +#define PG_VERSION_18 180000 #endif /* PG_VERSION_CONSTANTS */ From 16ab11622b2445eee715debf1830810941e82495 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Thu, 14 Nov 2024 15:30:09 +0300 Subject: [PATCH 056/155] Add devcontainer support to release 13.0 (#7739) Add devcontainer support to release 13.0 --- .devcontainer/.gdbinit | 33 + .devcontainer/.gitignore | 1 + .devcontainer/.psqlrc | 7 + .devcontainer/.vscode/Pipfile | 12 + .devcontainer/.vscode/Pipfile.lock | 28 + .../.vscode/generate_c_cpp_properties-json.py | 84 ++ .devcontainer/.vscode/launch.json | 40 + .devcontainer/Dockerfile | 217 ++++ .devcontainer/Makefile | 11 + .devcontainer/devcontainer.json | 37 + .devcontainer/pgenv/config/default.conf | 15 + .devcontainer/requirements.txt | 9 + .devcontainer/src/test/regress/Pipfile | 28 + .devcontainer/src/test/regress/Pipfile.lock | 1022 +++++++++++++++++ 14 files changed, 1544 insertions(+) create mode 100644 .devcontainer/.gdbinit create mode 100644 .devcontainer/.gitignore create mode 100644 .devcontainer/.psqlrc create mode 100644 .devcontainer/.vscode/Pipfile create mode 100644 .devcontainer/.vscode/Pipfile.lock create mode 100755 .devcontainer/.vscode/generate_c_cpp_properties-json.py create mode 100644 .devcontainer/.vscode/launch.json create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/Makefile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/pgenv/config/default.conf create mode 100644 .devcontainer/requirements.txt create mode 100644 .devcontainer/src/test/regress/Pipfile create mode 100644 .devcontainer/src/test/regress/Pipfile.lock diff --git a/.devcontainer/.gdbinit b/.devcontainer/.gdbinit new file mode 100644 index 000000000..9d544512b --- /dev/null +++ b/.devcontainer/.gdbinit @@ -0,0 +1,33 @@ +# gdbpg.py contains scripts to nicely print the postgres datastructures +# while in a gdb session. Since the vscode debugger is based on gdb this +# actually also works when debugging with vscode. Providing nice tools +# to understand the internal datastructures we are working with. +source /root/gdbpg.py + +# when debugging postgres it is convenient to _always_ have a breakpoint +# trigger when an error is logged. Because .gdbinit is sourced before gdb +# is fully attached and has the sources loaded. To make sure the breakpoint +# is added when the library is loaded we temporary set the breakpoint pending +# to on. After we have added out breakpoint we revert back to the default +# configuration for breakpoint pending. +# The breakpoint is hard to read, but at entry of the function we don't have +# the level loaded in elevel. Instead we hardcode the location where the +# level of the current error is stored. Also gdb doesn't understand the +# ERROR symbol so we hardcode this to the value of ERROR. It is very unlikely +# this value will ever change in postgres, but if it does we might need to +# find a way to conditionally load the correct breakpoint. +set breakpoint pending on +break elog.c:errfinish if errordata[errordata_stack_depth].elevel == 21 +set breakpoint pending auto + +echo \n +echo ----------------------------------------------------------------------------------\n +echo when attaching to a postgres backend a breakpoint will be set on elog.c:errfinish \n +echo it will only break on errors being raised in postgres \n +echo \n +echo to disable this breakpoint from vscode run `-exec disable 1` in the debug console \n +echo this assumes it's the first breakpoint loaded as it is loaded from .gdbinit \n +echo this can be verified with `-exec info break`, enabling can be done with \n +echo `-exec enable 1` \n +echo ----------------------------------------------------------------------------------\n +echo \n diff --git a/.devcontainer/.gitignore b/.devcontainer/.gitignore new file mode 100644 index 000000000..3a7f553fc --- /dev/null +++ b/.devcontainer/.gitignore @@ -0,0 +1 @@ +postgresql-*.tar.bz2 diff --git a/.devcontainer/.psqlrc b/.devcontainer/.psqlrc new file mode 100644 index 000000000..07ea06cdd --- /dev/null +++ b/.devcontainer/.psqlrc @@ -0,0 +1,7 @@ +\timing on +\pset linestyle unicode +\pset border 2 +\setenv PAGER 'pspg --no-mouse -bX --no-commandbar --no-topbar' +\set HISTSIZE 100000 +\set PROMPT1 '\n%[%033[1m%]%M %n@%/:%> (PID: %p)%R%[%033[0m%]%# ' +\set PROMPT2 ' ' diff --git a/.devcontainer/.vscode/Pipfile b/.devcontainer/.vscode/Pipfile new file mode 100644 index 000000000..57909c897 --- /dev/null +++ b/.devcontainer/.vscode/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +docopt = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/.devcontainer/.vscode/Pipfile.lock b/.devcontainer/.vscode/Pipfile.lock new file mode 100644 index 000000000..52ee8663c --- /dev/null +++ b/.devcontainer/.vscode/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "6956a6700ead5804aa56bd597c93bb4a13f208d2d49d3b5399365fd240ca0797" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "index": "pypi", + "version": "==0.6.2" + } + }, + "develop": {} +} diff --git a/.devcontainer/.vscode/generate_c_cpp_properties-json.py b/.devcontainer/.vscode/generate_c_cpp_properties-json.py new file mode 100755 index 000000000..6f49a1818 --- /dev/null +++ b/.devcontainer/.vscode/generate_c_cpp_properties-json.py @@ -0,0 +1,84 @@ +#! /usr/bin/env pipenv-shebang +"""Generate C/C++ properties file for VSCode. + +Uses pgenv to iterate postgres versions and generate +a C/C++ properties file for VSCode containing the +include paths for the postgres headers. + +Usage: + generate_c_cpp_properties-json.py + generate_c_cpp_properties-json.py (-h | --help) + generate_c_cpp_properties-json.py --version + +Options: + -h --help Show this screen. + --version Show version. + +""" +import json +import subprocess + +from docopt import docopt + + +def main(args): + target_path = args[''] + + output = subprocess.check_output(['pgenv', 'versions']) + # typical output is: + # 14.8 pgsql-14.8 + # * 15.3 pgsql-15.3 + # 16beta2 pgsql-16beta2 + # where the line marked with a * is the currently active version + # + # we are only interested in the first word of each line, which is the version number + # thus we strip the whitespace and the * from the line and split it into words + # and take the first word + versions = [line.strip('* ').split()[0] for line in output.decode('utf-8').splitlines()] + + # create the list of configurations per version + configurations = [] + for version in versions: + configurations.append(generate_configuration(version)) + + # create the json file + c_cpp_properties = { + "configurations": configurations, + "version": 4 + } + + # write the c_cpp_properties.json file + with open(target_path, 'w') as f: + json.dump(c_cpp_properties, f, indent=4) + + +def generate_configuration(version): + """Returns a configuration for the given postgres version. + + >>> generate_configuration('14.8') + { + "name": "Citus Development Configuration - Postgres 14.8", + "includePath": [ + "/usr/local/include", + "/home/citus/.pgenv/src/postgresql-14.8/src/**", + "${workspaceFolder}/**", + "${workspaceFolder}/src/include/", + ], + "configurationProvider": "ms-vscode.makefile-tools" + } + """ + return { + "name": f"Citus Development Configuration - Postgres {version}", + "includePath": [ + "/usr/local/include", + f"/home/citus/.pgenv/src/postgresql-{version}/src/**", + "${workspaceFolder}/**", + "${workspaceFolder}/src/include/", + ], + "configurationProvider": "ms-vscode.makefile-tools" + } + + +if __name__ == '__main__': + arguments = docopt(__doc__, version='0.1.0') + main(arguments) diff --git a/.devcontainer/.vscode/launch.json b/.devcontainer/.vscode/launch.json new file mode 100644 index 000000000..6de90ce09 --- /dev/null +++ b/.devcontainer/.vscode/launch.json @@ -0,0 +1,40 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach Citus (devcontainer)", + "type": "cppdbg", + "request": "attach", + "processId": "${command:pickProcess}", + "program": "/home/citus/.pgenv/pgsql/bin/postgres", + "additionalSOLibSearchPath": "/home/citus/.pgenv/pgsql/lib", + "setupCommands": [ + { + "text": "handle SIGUSR1 noprint nostop pass", + "description": "let gdb not stop when SIGUSR1 is sent to process", + "ignoreFailures": true + } + ], + }, + { + "name": "Open core file", + "type": "cppdbg", + "request": "launch", + "program": "/home/citus/.pgenv/pgsql/bin/postgres", + "coreDumpPath": "${input:corefile}", + "cwd": "${workspaceFolder}", + "MIMode": "gdb", + } + ], + "inputs": [ + { + "id": "corefile", + "type": "command", + "command": "extension.commandvariable.file.pickFile", + "args": { + "dialogTitle": "Select core file", + "include": "**/core*", + }, + }, + ], +} diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..33bba98d5 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,217 @@ +FROM ubuntu:22.04 AS base + +# environment is to make python pass an interactive shell, probably not the best timezone given a wide variety of colleagues +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# install build tools +RUN apt update && apt install -y \ + bzip2 \ + cpanminus \ + curl \ + flex \ + gcc \ + git \ + libcurl4-gnutls-dev \ + libicu-dev \ + libkrb5-dev \ + liblz4-dev \ + libpam0g-dev \ + libreadline-dev \ + libselinux1-dev \ + libssl-dev \ + libxslt-dev \ + libzstd-dev \ + locales \ + make \ + perl \ + pkg-config \ + python3 \ + python3-pip \ + software-properties-common \ + sudo \ + uuid-dev \ + valgrind \ + zlib1g-dev \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt install -y \ + python3.9-full \ + # software properties pulls in pkexec, which makes the debugger unusable in vscode + && apt purge -y \ + software-properties-common \ + && apt autoremove -y \ + && apt clean + +RUN sudo pip3 install pipenv pipenv-shebang + +RUN cpanm install IPC::Run + +RUN locale-gen en_US.UTF-8 + +# add the citus user to sudoers and allow all sudoers to login without a password prompt +RUN useradd -ms /bin/bash citus \ + && usermod -aG sudo citus \ + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +WORKDIR /home/citus +USER citus + +# run all make commands with the number of cores available +RUN echo "export MAKEFLAGS=\"-j \$(nproc)\"" >> "/home/citus/.bashrc" + +RUN git clone --branch v1.3.2 --depth 1 https://github.com/theory/pgenv.git .pgenv +COPY --chown=citus:citus pgenv/config/ .pgenv/config/ +ENV PATH="/home/citus/.pgenv/bin:${PATH}" +ENV PATH="/home/citus/.pgenv/pgsql/bin:${PATH}" + +USER citus + +# build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions +FROM base AS pg14 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.12 +RUN rm .pgenv/src/*.tar* +RUN make -C .pgenv/src/postgresql-*/ clean +RUN make -C .pgenv/src/postgresql-*/src/include install + +# create a staging directory with all files we want to copy from our pgenv build +# we will copy the contents of the staged folder into the final image at once +RUN mkdir .pgenv-staging/ +RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ +RUN rm .pgenv-staging/config/default.conf + +FROM base AS pg15 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.7 +RUN rm .pgenv/src/*.tar* +RUN make -C .pgenv/src/postgresql-*/ clean +RUN make -C .pgenv/src/postgresql-*/src/include install + +# create a staging directory with all files we want to copy from our pgenv build +# we will copy the contents of the staged folder into the final image at once +RUN mkdir .pgenv-staging/ +RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ +RUN rm .pgenv-staging/config/default.conf + +FROM base AS pg16 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.3 +RUN rm .pgenv/src/*.tar* +RUN make -C .pgenv/src/postgresql-*/ clean +RUN make -C .pgenv/src/postgresql-*/src/include install + +# create a staging directory with all files we want to copy from our pgenv build +# we will copy the contents of the staged folder into the final image at once +RUN mkdir .pgenv-staging/ +RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ +RUN rm .pgenv-staging/config/default.conf + +FROM base AS uncrustify-builder + +RUN sudo apt update && sudo apt install -y cmake tree + +WORKDIR /uncrustify +RUN curl -L https://github.com/uncrustify/uncrustify/archive/uncrustify-0.68.1.tar.gz | tar xz +WORKDIR /uncrustify/uncrustify-uncrustify-0.68.1/ +RUN mkdir build +WORKDIR /uncrustify/uncrustify-uncrustify-0.68.1/build/ +RUN cmake .. +RUN MAKEFLAGS="-j $(nproc)" make -s + +RUN make install DESTDIR=/uncrustify + +# builder for all pipenv's to get them contained in a single layer +FROM base AS pipenv + +WORKDIR /workspaces/citus/ + +# tools to sync pgenv with vscode +COPY --chown=citus:citus .vscode/Pipfile .vscode/Pipfile.lock .devcontainer/.vscode/ +RUN ( cd .devcontainer/.vscode && pipenv install ) + +# environment to run our failure tests +COPY --chown=citus:citus src/ src/ +RUN ( cd src/test/regress && pipenv install ) + +# assemble the final container by copying over the artifacts from separately build containers +FROM base AS devcontainer + +LABEL org.opencontainers.image.source=https://github.com/citusdata/citus +LABEL org.opencontainers.image.description="Development container for the Citus project" +LABEL org.opencontainers.image.licenses=AGPL-3.0-only + +RUN yes | sudo unminimize + +# install developer productivity tools +RUN sudo apt update \ + && sudo apt install -y \ + autoconf2.69 \ + bash-completion \ + fswatch \ + gdb \ + htop \ + libdbd-pg-perl \ + libdbi-perl \ + lsof \ + man \ + net-tools \ + psmisc \ + pspg \ + tree \ + vim \ + && sudo apt clean + +# Since gdb will run in the context of the root user when debugging citus we will need to both +# download the gdbpg.py script as the root user, into their home directory, as well as add .gdbinit +# as a file owned by root +# This will make that as soon as the debugger attaches to a postgres backend (or frankly any other process) +# the gdbpg.py script will be sourced and the developer can direcly use it. +RUN sudo curl -o /root/gdbpg.py https://raw.githubusercontent.com/tvesely/gdbpg/6065eee7872457785f830925eac665aa535caf62/gdbpg.py +COPY --chown=root:root .gdbinit /root/ + +# install developer dependencies in the global environment +RUN --mount=type=bind,source=requirements.txt,target=requirements.txt pip install -r requirements.txt + +# for persistent bash history across devcontainers we need to have +# a) a directory to store the history in +# b) a prompt command to append the history to the file +# c) specify the history file to store the history in +# b and c are done in the .bashrc to make it persistent across shells only +RUN sudo install -d -o citus -g citus /commandhistory \ + && echo "export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" >> "/home/citus/.bashrc" + +# install citus-dev +RUN git clone --branch develop https://github.com/citusdata/tools.git citus-tools \ + && ( cd citus-tools/citus_dev && pipenv install ) \ + && mkdir -p ~/.local/bin \ + && ln -s /home/citus/citus-tools/citus_dev/citus_dev-pipenv .local/bin/citus_dev \ + && sudo make -C citus-tools/uncrustify install bindir=/usr/local/bin pkgsysconfdir=/usr/local/etc/ \ + && mkdir -p ~/.local/share/bash-completion/completions/ \ + && ln -s ~/citus-tools/citus_dev/bash_completion ~/.local/share/bash-completion/completions/citus_dev + +# TODO some LC_ALL errors, possibly solved by locale-gen +RUN git clone https://github.com/so-fancy/diff-so-fancy.git \ + && mkdir -p ~/.local/bin \ + && ln -s /home/citus/diff-so-fancy/diff-so-fancy .local/bin/ + +COPY --link --from=uncrustify-builder /uncrustify/usr/ /usr/ + +COPY --link --from=pg14 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ +COPY --link --from=pg15 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ +COPY --link --from=pg16 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ + +COPY --link --from=pipenv /home/citus/.local/share/virtualenvs/ /home/citus/.local/share/virtualenvs/ + +# place to run your cluster with citus_dev +VOLUME /data +RUN sudo mkdir /data \ + && sudo chown citus:citus /data + +COPY --chown=citus:citus .psqlrc . + +# with the copy linking of layers github actions seem to misbehave with the ownership of the +# directories leading upto the link, hence a small patch layer to have to right ownerships set +RUN sudo chown --from=root:root citus:citus -R ~ + +# sets default pg version +RUN pgenv switch 16.3 + +# make connecting to the coordinator easy +ENV PGPORT=9700 diff --git a/.devcontainer/Makefile b/.devcontainer/Makefile new file mode 100644 index 000000000..8f4174104 --- /dev/null +++ b/.devcontainer/Makefile @@ -0,0 +1,11 @@ + +init: ../.vscode/c_cpp_properties.json ../.vscode/launch.json + +../.vscode: + mkdir -p ../.vscode + +../.vscode/launch.json: ../.vscode .vscode/launch.json + cp .vscode/launch.json ../.vscode/launch.json + +../.vscode/c_cpp_properties.json: ../.vscode + ./.vscode/generate_c_cpp_properties-json.py ../.vscode/c_cpp_properties.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..cddfcebf4 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,37 @@ +{ + "image": "ghcr.io/citusdata/citus-devcontainer:main", + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--ulimit=core=-1", + ], + "forwardPorts": [ + 9700 + ], + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "GitHub.copilot-chat", + "GitHub.copilot", + "github.vscode-github-actions", + "github.vscode-pull-request-github", + "ms-vscode.cpptools-extension-pack", + "ms-vsliveshare.vsliveshare", + "rioj7.command-variable", + ], + "settings": { + "files.exclude": { + "**/*.o": true, + "**/.deps/": true, + } + }, + } + }, + "mounts": [ + "type=volume,target=/data", + "source=citus-bashhistory,target=/commandhistory,type=volume", + ], + "updateContentCommand": "./configure", + "postCreateCommand": "make -C .devcontainer/", +} + diff --git a/.devcontainer/pgenv/config/default.conf b/.devcontainer/pgenv/config/default.conf new file mode 100644 index 000000000..ab55493f9 --- /dev/null +++ b/.devcontainer/pgenv/config/default.conf @@ -0,0 +1,15 @@ +PGENV_MAKE_OPTIONS=(-s) + +PGENV_CONFIGURE_OPTIONS=( + --enable-debug + --enable-depend + --enable-cassert + --enable-tap-tests + 'CFLAGS=-ggdb -Og -g3 -fno-omit-frame-pointer -DUSE_VALGRIND' + --with-openssl + --with-libxml + --with-libxslt + --with-uuid=e2fs + --with-icu + --with-lz4 +) diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt new file mode 100644 index 000000000..7300b3b89 --- /dev/null +++ b/.devcontainer/requirements.txt @@ -0,0 +1,9 @@ +black==23.11.0 +click==8.1.7 +isort==5.12.0 +mypy-extensions==1.0.0 +packaging==23.2 +pathspec==0.11.2 +platformdirs==4.0.0 +tomli==2.0.1 +typing_extensions==4.8.0 diff --git a/.devcontainer/src/test/regress/Pipfile b/.devcontainer/src/test/regress/Pipfile new file mode 100644 index 000000000..8811bbd8c --- /dev/null +++ b/.devcontainer/src/test/regress/Pipfile @@ -0,0 +1,28 @@ +[[source]] +name = "pypi" +url = "https://pypi.python.org/simple" +verify_ssl = true + +[packages] +mitmproxy = {editable = true, ref = "main", git = "https://github.com/citusdata/mitmproxy.git"} +construct = "*" +docopt = "==0.6.2" +cryptography = ">=41.0.4" +pytest = "*" +psycopg = "*" +filelock = "*" +pytest-asyncio = "*" +pytest-timeout = "*" +pytest-xdist = "*" +pytest-repeat = "*" +pyyaml = "*" +werkzeug = "==2.3.7" + +[dev-packages] +black = "*" +isort = "*" +flake8 = "*" +flake8-bugbear = "*" + +[requires] +python_version = "3.9" diff --git a/.devcontainer/src/test/regress/Pipfile.lock b/.devcontainer/src/test/regress/Pipfile.lock new file mode 100644 index 000000000..fb82a6573 --- /dev/null +++ b/.devcontainer/src/test/regress/Pipfile.lock @@ -0,0 +1,1022 @@ +{ + "_meta": { + "hash": { + "sha256": "f8db86383082539f626f1402e720f5f2e3f9718b44a8f26110cf9f52e7ca46bc" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "asgiref": { + "hashes": [ + "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9", + "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214" + ], + "markers": "python_version >= '3.6'", + "version": "==3.4.1" + }, + "blinker": { + "hashes": [ + "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" + ], + "version": "==1.4" + }, + "brotli": { + "hashes": [ + "sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019", + "sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df", + "sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d", + "sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8", + "sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b", + "sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c", + "sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c", + "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70", + "sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f", + "sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181", + "sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130", + "sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19", + "sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be", + "sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be", + "sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a", + "sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa", + "sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429", + "sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126", + "sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7", + "sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad", + "sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679", + "sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4", + "sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0", + "sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b", + "sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6", + "sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438", + "sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f", + "sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389", + "sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6", + "sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26", + "sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337", + "sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7", + "sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14", + "sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2", + "sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430", + "sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296", + "sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12", + "sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f", + "sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7", + "sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d", + "sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a", + "sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452", + "sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c", + "sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761", + "sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649", + "sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b", + "sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea", + "sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c", + "sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f", + "sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a", + "sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031", + "sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267", + "sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5", + "sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7", + "sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d", + "sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c", + "sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43", + "sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa", + "sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde", + "sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17", + "sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f", + "sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8", + "sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb", + "sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb", + "sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d", + "sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b", + "sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4", + "sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755", + "sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a", + "sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d", + "sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a", + "sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3", + "sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7", + "sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1", + "sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb", + "sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a", + "sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91", + "sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b", + "sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1", + "sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806", + "sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3", + "sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1" + ], + "version": "==1.0.9" + }, + "certifi": { + "hashes": [ + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" + ], + "markers": "python_version >= '3.6'", + "version": "==2024.2.2" + }, + "cffi": { + "hashes": [ + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "platform_python_implementation != 'PyPy'", + "version": "==1.16.0" + }, + "click": { + "hashes": [ + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" + ], + "markers": "python_version >= '3.6'", + "version": "==8.0.4" + }, + "construct": { + "hashes": [ + "sha256:4d2472f9684731e58cc9c56c463be63baa1447d674e0d66aeb5627b22f512c29", + "sha256:c80be81ef595a1a821ec69dc16099550ed22197615f4320b57cc9ce2a672cb30" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==2.10.70" + }, + "cryptography": { + "hashes": [ + "sha256:04859aa7f12c2b5f7e22d25198ddd537391f1695df7057c8700f71f26f47a129", + "sha256:069d2ce9be5526a44093a0991c450fe9906cdf069e0e7cd67d9dee49a62b9ebe", + "sha256:0d3ec384058b642f7fb7e7bff9664030011ed1af8f852540c76a1317a9dd0d20", + "sha256:0fab2a5c479b360e5e0ea9f654bcebb535e3aa1e493a715b13244f4e07ea8eec", + "sha256:0fea01527d4fb22ffe38cd98951c9044400f6eff4788cf52ae116e27d30a1ba3", + "sha256:1b797099d221df7cce5ff2a1d272761d1554ddf9a987d3e11f6459b38cd300fd", + "sha256:1e935c2900fb53d31f491c0de04f41110351377be19d83d908c1fd502ae8daa5", + "sha256:20100c22b298c9eaebe4f0b9032ea97186ac2555f426c3e70670f2517989543b", + "sha256:20180da1b508f4aefc101cebc14c57043a02b355d1a652b6e8e537967f1e1b46", + "sha256:25b09b73db78facdfd7dd0fa77a3f19e94896197c86e9f6dc16bce7b37a96504", + "sha256:2619487f37da18d6826e27854a7f9d4d013c51eafb066c80d09c63cf24505306", + "sha256:2eb6368d5327d6455f20327fb6159b97538820355ec00f8cc9464d617caecead", + "sha256:35772a6cffd1f59b85cb670f12faba05513446f80352fe811689b4e439b5d89e", + "sha256:39d5c93e95bcbc4c06313fc6a500cee414ee39b616b55320c1904760ad686938", + "sha256:3d96ea47ce6d0055d5b97e761d37b4e84195485cb5a38401be341fabf23bc32a", + "sha256:4dcab7c25e48fc09a73c3e463d09ac902a932a0f8d0c568238b3696d06bf377b", + "sha256:5fbf0f3f0fac7c089308bd771d2c6c7b7d53ae909dce1db52d8e921f6c19bb3a", + "sha256:6c25e1e9c2ce682d01fc5e2dde6598f7313027343bd14f4049b82ad0402e52cd", + "sha256:762f3771ae40e111d78d77cbe9c1035e886ac04a234d3ee0856bf4ecb3749d54", + "sha256:90147dad8c22d64b2ff7331f8d4cddfdc3ee93e4879796f837bdbb2a0b141e0c", + "sha256:935cca25d35dda9e7bd46a24831dfd255307c55a07ff38fd1a92119cffc34857", + "sha256:93fbee08c48e63d5d1b39ab56fd3fdd02e6c2431c3da0f4edaf54954744c718f", + "sha256:9541c69c62d7446539f2c1c06d7046aef822940d248fa4b8962ff0302862cc1f", + "sha256:c23f03cfd7d9826cdcbad7850de67e18b4654179e01fe9bc623d37c2638eb4ef", + "sha256:c3d1f5a1d403a8e640fa0887e9f7087331abb3f33b0f2207d2cc7f213e4a864c", + "sha256:d1998e545081da0ab276bcb4b33cce85f775adb86a516e8f55b3dac87f469548", + "sha256:d5cf11bc7f0b71fb71af26af396c83dfd3f6eed56d4b6ef95d57867bf1e4ba65", + "sha256:db0480ffbfb1193ac4e1e88239f31314fe4c6cdcf9c0b8712b55414afbf80db4", + "sha256:de4ae486041878dc46e571a4c70ba337ed5233a1344c14a0790c4c4be4bbb8b4", + "sha256:de5086cd475d67113ccb6f9fae6d8fe3ac54a4f9238fd08bfdb07b03d791ff0a", + "sha256:df34312149b495d9d03492ce97471234fd9037aa5ba217c2a6ea890e9166f151", + "sha256:ead69ba488f806fe1b1b4050febafdbf206b81fa476126f3e16110c818bac396" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==42.0.3" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "index": "pypi", + "version": "==0.6.2" + }, + "exceptiongroup": { + "hashes": [ + "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", + "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68" + ], + "markers": "python_version < '3.11'", + "version": "==1.2.0" + }, + "execnet": { + "hashes": [ + "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41", + "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.2" + }, + "filelock": { + "hashes": [ + "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e", + "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==3.13.1" + }, + "flask": { + "hashes": [ + "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f", + "sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.3" + }, + "h11": { + "hashes": [ + "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6", + "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042" + ], + "markers": "python_version >= '3.6'", + "version": "==0.12.0" + }, + "h2": { + "hashes": [ + "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d", + "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==4.1.0" + }, + "hpack": { + "hashes": [ + "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c", + "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==4.0.0" + }, + "hyperframe": { + "hashes": [ + "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15", + "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==6.0.1" + }, + "iniconfig": { + "hashes": [ + "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "itsdangerous": { + "hashes": [ + "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", + "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.2" + }, + "jinja2": { + "hashes": [ + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.3" + }, + "kaitaistruct": { + "hashes": [ + "sha256:3d5845817ec8a4d5504379cc11bd570b038850ee49c4580bc0998c8fb1d327ad" + ], + "version": "==0.9" + }, + "ldap3": { + "hashes": [ + "sha256:2bc966556fc4d4fa9f445a1c31dc484ee81d44a51ab0e2d0fd05b62cac75daa6", + "sha256:5630d1383e09ba94839e253e013f1aa1a2cf7a547628ba1265cb7b9a844b5687", + "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70", + "sha256:5ab7febc00689181375de40c396dcad4f2659cd260fc5e94c508b6d77c17e9d5", + "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f" + ], + "version": "==2.9.1" + }, + "markupsafe": { + "hashes": [ + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.5" + }, + "mitmproxy": { + "editable": true, + "git": "https://github.com/citusdata/mitmproxy.git", + "markers": "python_version >= '3.9'", + "ref": "2fd18ef051b987925a36337ab1d61aa674353b44" + }, + "msgpack": { + "hashes": [ + "sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862", + "sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d", + "sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3", + "sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672", + "sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0", + "sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9", + "sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee", + "sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46", + "sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524", + "sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819", + "sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc", + "sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc", + "sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1", + "sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82", + "sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81", + "sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6", + "sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d", + "sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2", + "sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c", + "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87", + "sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84", + "sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e", + "sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95", + "sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f", + "sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b", + "sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93", + "sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf", + "sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61", + "sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c", + "sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8", + "sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d", + "sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c", + "sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4", + "sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba", + "sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415", + "sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee", + "sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d", + "sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9", + "sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075", + "sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f", + "sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7", + "sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681", + "sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329", + "sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1", + "sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf", + "sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c", + "sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5", + "sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b", + "sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5", + "sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e", + "sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b", + "sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad", + "sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd", + "sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7", + "sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002", + "sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc" + ], + "markers": "python_version >= '3.8'", + "version": "==1.0.7" + }, + "packaging": { + "hashes": [ + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + ], + "markers": "python_version >= '3.7'", + "version": "==23.2" + }, + "passlib": { + "hashes": [ + "sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1", + "sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04" + ], + "version": "==1.7.4" + }, + "pluggy": { + "hashes": [ + "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981", + "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be" + ], + "markers": "python_version >= '3.8'", + "version": "==1.4.0" + }, + "protobuf": { + "hashes": [ + "sha256:0c44e01f74109decea196b5b313b08edb5316df77313995594a6981e95674259", + "sha256:15cdecb0d192ab5f17cdc21a9c0ae7b5c6c4451e42c8a888a4f3344c190e369c", + "sha256:196a153e487c0e20d62259872bbf2e1c4fa18e2ce97e20984fcbf9d8b151058d", + "sha256:3149c373e9b7ce296bb24d42a3eb677d620185b5dff2c390b2cf57baf79afdc1", + "sha256:370a6b885e94adda021d4cbe43accdfbf6a02af651a0be337a28906a3fa77f3d", + "sha256:474247630834f93214fafce49d2ee6ff4c036c8c5382b88432b7eae6f08f131b", + "sha256:6380aae2683d0d1b41199e591c8ba06f867e8a778d44309af87073c1b34a9f3a", + "sha256:6741d7d1cfcbdd6cf610f38b7976cf8c0b41022203555298925e4061b6616608", + "sha256:700787cb56b4cb7b8ed5f7d197b9d8f30080f257f3c7431eec1fdd8060660929", + "sha256:8117b52c2531e4033f7d02b9be5a78564da41a8b02c255e1b731ad4bd75e7dc0", + "sha256:850da2072d98c6e576b7eb29734cdde6fd9f5d157e43d7818d79f4b373ef5d51", + "sha256:85d1fb5ff1d638a0045bbe4f01a8f287023aa4f2b29011445b1be0edc74a2103", + "sha256:93bca9aaeee8008e15696c2a6b5e56b992da03f9d237ff54310e397d635f8305", + "sha256:98d414513ec44bb3ba77ebdeffcbbe6ebbf3630c767d37a285890c2414fdd4e2", + "sha256:a7f91a4e5bf3cc58b2830c9cb01b04ac5e211c288048e9296cd407ec0455fb89", + "sha256:abbcb8ecd19cfb729b9b71f9a453e37c0c1c017be4bff47804ff25150685386d", + "sha256:b03966ca4d1aa7850f5bf0d841c22a8eeb6ce091f77e585ffeb8b95a6b0a96c4", + "sha256:cde2a73b03049b904dbc5d0f500b97e11abb4109dbe2940e6a1595e2eef4e8a9", + "sha256:d52a687e2c74c40f45abd6906f833d4e40f0f8cfa4226a80e4695fedafe6c57e", + "sha256:e68ad00695547d9397dd14abd3efba23cb31cef67228f4512d41396971889812", + "sha256:e9bffd52d6ee039a1cafb72475b2900c6fd0f0dca667fb7a09af0a3e119e78cb" + ], + "markers": "python_version >= '3.5'", + "version": "==3.18.3" + }, + "psycopg": { + "hashes": [ + "sha256:31144d3fb4c17d78094d9e579826f047d4af1da6a10427d91dfcfb6ecdf6f12b", + "sha256:4d5a0a5a8590906daa58ebd5f3cfc34091377354a1acced269dd10faf55da60e" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==3.1.18" + }, + "publicsuffix2": { + "hashes": [ + "sha256:00f8cc31aa8d0d5592a5ced19cccba7de428ebca985db26ac852d920ddd6fe7b", + "sha256:786b5e36205b88758bd3518725ec8cfe7a8173f5269354641f581c6b80a99893" + ], + "version": "==2.20191221" + }, + "pyasn1": { + "hashes": [ + "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58", + "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==0.5.1" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pyopenssl": { + "hashes": [ + "sha256:6aa33039a93fffa4563e655b61d11364d01264be8ccb49906101e02a334530bf", + "sha256:ba07553fb6fd6a7a2259adb9b84e12302a9a8a75c44046e8bb5d3e5ee887e3c3" + ], + "markers": "python_version >= '3.7'", + "version": "==24.0.0" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pyperclip": { + "hashes": [ + "sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57" + ], + "version": "==1.8.2" + }, + "pytest": { + "hashes": [ + "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c", + "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==8.0.0" + }, + "pytest-asyncio": { + "hashes": [ + "sha256:3a048872a9c4ba14c3e90cc1aa20cbc2def7d01c7c8db3777ec281ba9c057675", + "sha256:4e7093259ba018d58ede7d5315131d21923a60f8a6e9ee266ce1589685c89eac" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==0.23.5" + }, + "pytest-repeat": { + "hashes": [ + "sha256:26ab2df18226af9d5ce441c858f273121e92ff55f5bb311d25755b8d7abdd8ed", + "sha256:ffd3836dfcd67bb270bec648b330e20be37d2966448c4148c4092d1e8aba8185" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==0.9.3" + }, + "pytest-timeout": { + "hashes": [ + "sha256:3b0b95dabf3cb50bac9ef5ca912fa0cfc286526af17afc806824df20c2f72c90", + "sha256:bde531e096466f49398a59f2dde76fa78429a09a12411466f88a07213e220de2" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==2.2.0" + }, + "pytest-xdist": { + "hashes": [ + "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a", + "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==3.5.0" + }, + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==6.0.1" + }, + "ruamel.yaml": { + "hashes": [ + "sha256:1a771fc92d3823682b7f0893ad56cb5a5c87c48e62b5399d6f42c8759a583b33", + "sha256:ea21da1198c4b41b8e7a259301cc9710d3b972bf8ba52f06218478e6802dd1f1" + ], + "markers": "python_version >= '3'", + "version": "==0.17.16" + }, + "ruamel.yaml.clib": { + "hashes": [ + "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d", + "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001", + "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462", + "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9", + "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe", + "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b", + "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b", + "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615", + "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62", + "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15", + "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b", + "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1", + "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9", + "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675", + "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899", + "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7", + "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7", + "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312", + "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa", + "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91", + "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b", + "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6", + "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3", + "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334", + "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5", + "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3", + "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe", + "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c", + "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed", + "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337", + "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880", + "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f", + "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d", + "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248", + "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d", + "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf", + "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512", + "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069", + "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb", + "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942", + "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d", + "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31", + "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92", + "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5", + "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28", + "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d", + "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1", + "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2", + "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875", + "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412" + ], + "markers": "python_version < '3.10' and platform_python_implementation == 'CPython'", + "version": "==0.2.8" + }, + "sortedcontainers": { + "hashes": [ + "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", + "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" + ], + "version": "==2.4.0" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + }, + "tornado": { + "hashes": [ + "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0", + "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63", + "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263", + "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052", + "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f", + "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee", + "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78", + "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579", + "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212", + "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e", + "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2" + ], + "markers": "python_version >= '3.8'", + "version": "==6.4" + }, + "typing-extensions": { + "hashes": [ + "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", + "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + ], + "markers": "python_version >= '3.8'", + "version": "==4.9.0" + }, + "urwid": { + "hashes": [ + "sha256:588bee9c1cb208d0906a9f73c613d2bd32c3ed3702012f51efe318a3f2127eae" + ], + "version": "==2.1.2" + }, + "werkzeug": { + "hashes": [ + "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8", + "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.3.7" + }, + "wsproto": { + "hashes": [ + "sha256:868776f8456997ad0d9720f7322b746bbe9193751b5b290b7f924659377c8c38", + "sha256:d8345d1808dd599b5ffb352c25a367adb6157e664e140dbecba3f9bc007edb9f" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==1.0.0" + }, + "zstandard": { + "hashes": [ + "sha256:1c5ef399f81204fbd9f0df3debf80389fd8aa9660fe1746d37c80b0d45f809e9", + "sha256:1faefe33e3d6870a4dce637bcb41f7abb46a1872a595ecc7b034016081c37543", + "sha256:1fb23b1754ce834a3a1a1e148cc2faad76eeadf9d889efe5e8199d3fb839d3c6", + "sha256:22f127ff5da052ffba73af146d7d61db874f5edb468b36c9cb0b857316a21b3d", + "sha256:2353b61f249a5fc243aae3caa1207c80c7e6919a58b1f9992758fa496f61f839", + "sha256:24cdcc6f297f7c978a40fb7706877ad33d8e28acc1786992a52199502d6da2a4", + "sha256:31e35790434da54c106f05fa93ab4d0fab2798a6350e8a73928ec602e8505836", + "sha256:3547ff4eee7175d944a865bbdf5529b0969c253e8a148c287f0668fe4eb9c935", + "sha256:378ac053c0cfc74d115cbb6ee181540f3e793c7cca8ed8cd3893e338af9e942c", + "sha256:3e1cd2db25117c5b7c7e86a17cde6104a93719a9df7cb099d7498e4c1d13ee5c", + "sha256:3fe469a887f6142cc108e44c7f42c036e43620ebaf500747be2317c9f4615d4f", + "sha256:4800ab8ec94cbf1ed09c2b4686288750cab0642cb4d6fba2a56db66b923aeb92", + "sha256:52de08355fd5cfb3ef4533891092bb96229d43c2069703d4aff04fdbedf9c92f", + "sha256:5752f44795b943c99be367fee5edf3122a1690b0d1ecd1bd5ec94c7fd2c39c94", + "sha256:5d53f02aeb8fdd48b88bc80bece82542d084fb1a7ba03bf241fd53b63aee4f22", + "sha256:69b7a5720b8dfab9005a43c7ddb2e3ccacbb9a2442908ae4ed49dd51ab19698a", + "sha256:6cc162b5b6e3c40b223163a9ea86cd332bd352ddadb5fd142fc0706e5e4eaaff", + "sha256:6f5d0330bc992b1e267a1b69fbdbb5ebe8c3a6af107d67e14c7a5b1ede2c5945", + "sha256:6ffadd48e6fe85f27ca3ca10cfd3ef3d0f933bef7316870285ffeb58d791ca9c", + "sha256:72a011678c654df8323aa7b687e3147749034fdbe994d346f139ab9702b59cea", + "sha256:77d26452676f471223571efd73131fd4a626622c7960458aab2763e025836fc5", + "sha256:7a88cc773ffe55992ff7259a8df5fb3570168d7138c69aadba40142d0e5ce39a", + "sha256:7b16bd74ae7bfbaca407a127e11058b287a4267caad13bd41305a5e630472549", + "sha256:855d95ec78b6f0ff66e076d5461bf12d09d8e8f7e2b3fc9de7236d1464fd730e", + "sha256:8baf7991547441458325ca8fafeae79ef1501cb4354022724f3edd62279c5b2b", + "sha256:8fb77dd152054c6685639d855693579a92f276b38b8003be5942de31d241ebfb", + "sha256:92d49cc3b49372cfea2d42f43a2c16a98a32a6bc2f42abcde121132dbfc2f023", + "sha256:94d0de65e37f5677165725f1fc7fb1616b9542d42a9832a9a0bdcba0ed68b63b", + "sha256:9867206093d7283d7de01bd2bf60389eb4d19b67306a0a763d1a8a4dbe2fb7c3", + "sha256:9ee3c992b93e26c2ae827404a626138588e30bdabaaf7aa3aa25082a4e718790", + "sha256:a4f8af277bb527fa3d56b216bda4da931b36b2d3fe416b6fc1744072b2c1dbd9", + "sha256:ab9f19460dfa4c5dd25431b75bee28b5f018bf43476858d64b1aa1046196a2a0", + "sha256:ac43c1821ba81e9344d818c5feed574a17f51fca27976ff7d022645c378fbbf5", + "sha256:af5a011609206e390b44847da32463437505bf55fd8985e7a91c52d9da338d4b", + "sha256:b0975748bb6ec55b6d0f6665313c2cf7af6f536221dccd5879b967d76f6e7899", + "sha256:b4963dad6cf28bfe0b61c3265d1c74a26a7605df3445bfcd3ba25de012330b2d", + "sha256:b7d3a484ace91ed827aa2ef3b44895e2ec106031012f14d28bd11a55f24fa734", + "sha256:bd3c478a4a574f412efc58ba7e09ab4cd83484c545746a01601636e87e3dbf23", + "sha256:c9e2dcb7f851f020232b991c226c5678dc07090256e929e45a89538d82f71d2e", + "sha256:d25c8eeb4720da41e7afbc404891e3a945b8bb6d5230e4c53d23ac4f4f9fc52c", + "sha256:dc8c03d0c5c10c200441ffb4cce46d869d9e5c4ef007f55856751dc288a2dffd", + "sha256:ec58e84d625553d191a23d5988a19c3ebfed519fff2a8b844223e3f074152163", + "sha256:eda0719b29792f0fea04a853377cfff934660cb6cd72a0a0eeba7a1f0df4a16e", + "sha256:edde82ce3007a64e8434ccaf1b53271da4f255224d77b880b59e7d6d73df90c8", + "sha256:f36722144bc0a5068934e51dca5a38a5b4daac1be84f4423244277e4baf24e7a", + "sha256:f8bb00ced04a8feff05989996db47906673ed45b11d86ad5ce892b5741e5f9dd", + "sha256:f98fc5750aac2d63d482909184aac72a979bfd123b112ec53fd365104ea15b1c", + "sha256:ff5b75f94101beaa373f1511319580a010f6e03458ee51b1a386d7de5331440a" + ], + "markers": "python_version >= '3.5'", + "version": "==0.15.2" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", + "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" + ], + "markers": "python_version >= '3.7'", + "version": "==23.2.0" + }, + "black": { + "hashes": [ + "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8", + "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8", + "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd", + "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9", + "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31", + "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92", + "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f", + "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29", + "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4", + "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693", + "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218", + "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a", + "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23", + "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0", + "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982", + "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894", + "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540", + "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430", + "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b", + "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2", + "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", + "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==24.2.0" + }, + "click": { + "hashes": [ + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" + ], + "markers": "python_version >= '3.6'", + "version": "==8.0.4" + }, + "flake8": { + "hashes": [ + "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", + "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3" + ], + "index": "pypi", + "markers": "python_full_version >= '3.8.1'", + "version": "==7.0.0" + }, + "flake8-bugbear": { + "hashes": [ + "sha256:663ef5de80cd32aacd39d362212983bc4636435a6f83700b4ed35acbd0b7d1b8", + "sha256:f9cb5f2a9e792dd80ff68e89a14c12eed8620af8b41a49d823b7a33064ac9658" + ], + "index": "pypi", + "markers": "python_full_version >= '3.8.1'", + "version": "==24.2.6" + }, + "isort": { + "hashes": [ + "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", + "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" + ], + "index": "pypi", + "markers": "python_full_version >= '3.8.0'", + "version": "==5.13.2" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "mypy-extensions": { + "hashes": [ + "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", + "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.0" + }, + "packaging": { + "hashes": [ + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + ], + "markers": "python_version >= '3.7'", + "version": "==23.2" + }, + "pathspec": { + "hashes": [ + "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", + "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712" + ], + "markers": "python_version >= '3.8'", + "version": "==0.12.1" + }, + "platformdirs": { + "hashes": [ + "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", + "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" + ], + "markers": "python_version >= '3.8'", + "version": "==4.2.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", + "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" + ], + "markers": "python_version >= '3.8'", + "version": "==2.11.1" + }, + "pyflakes": { + "hashes": [ + "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f", + "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a" + ], + "markers": "python_version >= '3.8'", + "version": "==3.2.0" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", + "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + ], + "markers": "python_version >= '3.8'", + "version": "==4.9.0" + } + } +} From 8c0feee74d17899708b0340572673cf08e30e926 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Sun, 17 Nov 2024 22:41:53 +0300 Subject: [PATCH 057/155] PG17 compatibility: Fix -1/Null diff in stxstattarget test output (#7748) Changed stxstattarget in pg_statistic_ext to use nullable representation, removing explicit -1 for default statistics target in PostgreSQL 17. Relevant PG commit: 012460ee93c304fbc7220e5b55d9d0577fc766ab https://github.com/postgres/postgres/commit/012460ee93c304fbc7220e5b55d9d0577fc766ab ```diff SELECT stxstattarget, stxrelid::regclass FROM pg_statistic_ext WHERE stxnamespace IN ( SELECT oid FROM pg_namespace WHERE nspname IN ('statistics''TestTarget') ) AND stxname SIMILAR TO '%\_\d+' ORDER BY stxstattarget, stxrelid::regclass ASC; stxstattarget | stxrelid ---------------+----------------------------------- - -1 | "statistics'TestTarget".t1_980000 - -1 | "statistics'TestTarget".t1_980002 ... + | "statistics'TestTarget".t1_980000 + | "statistics'TestTarget".t1_980002 ... ``` --- .../expected/pg13_propagate_statistics.out | 50 +++++++++++-------- .../regress/sql/pg13_propagate_statistics.sql | 18 +++++-- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/src/test/regress/expected/pg13_propagate_statistics.out b/src/test/regress/expected/pg13_propagate_statistics.out index 9f64aad21..6013564ec 100644 --- a/src/test/regress/expected/pg13_propagate_statistics.out +++ b/src/test/regress/expected/pg13_propagate_statistics.out @@ -24,33 +24,41 @@ SELECT create_distributed_table('t1', 'b'); -- test alter target before distribution ALTER STATISTICS s3 SET STATISTICS 46; \c - - - :worker_1_port -SELECT stxstattarget, stxrelid::regclass +-- for stxstattarget, re-interpret -1 as null to avoid adding another test output for pg < 17 +-- Changed stxstattarget in pg_statistic_ext to use nullable representation, removing explicit -1 for default statistics target in PostgreSQL 17. +-- https://github.com/postgres/postgres/commit/012460ee93c304fbc7220e5b55d9d0577fc766ab +SELECT + nullif(stxstattarget, -1) AS stxstattarget, + stxrelid::regclass FROM pg_statistic_ext WHERE stxnamespace IN ( - SELECT oid - FROM pg_namespace - WHERE nspname IN ('statistics''TestTarget') + SELECT oid + FROM pg_namespace + WHERE nspname IN ('statistics''TestTarget') ) AND stxname SIMILAR TO '%\_\d+' -ORDER BY stxstattarget, stxrelid::regclass ASC; +ORDER BY + nullif(stxstattarget, -1) IS NULL DESC, -- Make sure null values are handled consistently + nullif(stxstattarget, -1) NULLS FIRST, -- Use NULLS FIRST to ensure consistent placement of nulls + stxrelid::regclass ASC; stxstattarget | stxrelid --------------------------------------------------------------------- - -1 | "statistics'TestTarget".t1_980000 - -1 | "statistics'TestTarget".t1_980002 - -1 | "statistics'TestTarget".t1_980004 - -1 | "statistics'TestTarget".t1_980006 - -1 | "statistics'TestTarget".t1_980008 - -1 | "statistics'TestTarget".t1_980010 - -1 | "statistics'TestTarget".t1_980012 - -1 | "statistics'TestTarget".t1_980014 - -1 | "statistics'TestTarget".t1_980016 - -1 | "statistics'TestTarget".t1_980018 - -1 | "statistics'TestTarget".t1_980020 - -1 | "statistics'TestTarget".t1_980022 - -1 | "statistics'TestTarget".t1_980024 - -1 | "statistics'TestTarget".t1_980026 - -1 | "statistics'TestTarget".t1_980028 - -1 | "statistics'TestTarget".t1_980030 + | "statistics'TestTarget".t1_980000 + | "statistics'TestTarget".t1_980002 + | "statistics'TestTarget".t1_980004 + | "statistics'TestTarget".t1_980006 + | "statistics'TestTarget".t1_980008 + | "statistics'TestTarget".t1_980010 + | "statistics'TestTarget".t1_980012 + | "statistics'TestTarget".t1_980014 + | "statistics'TestTarget".t1_980016 + | "statistics'TestTarget".t1_980018 + | "statistics'TestTarget".t1_980020 + | "statistics'TestTarget".t1_980022 + | "statistics'TestTarget".t1_980024 + | "statistics'TestTarget".t1_980026 + | "statistics'TestTarget".t1_980028 + | "statistics'TestTarget".t1_980030 3 | "statistics'TestTarget".t1_980000 3 | "statistics'TestTarget".t1_980002 3 | "statistics'TestTarget".t1_980004 diff --git a/src/test/regress/sql/pg13_propagate_statistics.sql b/src/test/regress/sql/pg13_propagate_statistics.sql index 5b19f793a..7b5619db7 100644 --- a/src/test/regress/sql/pg13_propagate_statistics.sql +++ b/src/test/regress/sql/pg13_propagate_statistics.sql @@ -23,15 +23,23 @@ SELECT create_distributed_table('t1', 'b'); ALTER STATISTICS s3 SET STATISTICS 46; \c - - - :worker_1_port -SELECT stxstattarget, stxrelid::regclass +-- for stxstattarget, re-interpret -1 as null to avoid adding another test output for pg < 17 +-- Changed stxstattarget in pg_statistic_ext to use nullable representation, removing explicit -1 for default statistics target in PostgreSQL 17. +-- https://github.com/postgres/postgres/commit/012460ee93c304fbc7220e5b55d9d0577fc766ab +SELECT + nullif(stxstattarget, -1) AS stxstattarget, + stxrelid::regclass FROM pg_statistic_ext WHERE stxnamespace IN ( - SELECT oid - FROM pg_namespace - WHERE nspname IN ('statistics''TestTarget') + SELECT oid + FROM pg_namespace + WHERE nspname IN ('statistics''TestTarget') ) AND stxname SIMILAR TO '%\_\d+' -ORDER BY stxstattarget, stxrelid::regclass ASC; +ORDER BY + nullif(stxstattarget, -1) IS NULL DESC, -- Make sure null values are handled consistently + nullif(stxstattarget, -1) NULLS FIRST, -- Use NULLS FIRST to ensure consistent placement of nulls + stxrelid::regclass ASC; \c - - - :master_port -- the first one should log a notice that says statistics object does not exist From 32a2a31b137923f90774553142ddc2310b8b6e81 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Sun, 17 Nov 2024 23:43:39 +0300 Subject: [PATCH 058/155] PG17 compatibility: Fix -1/Null diff in attstattarget test output (#7749) Changed `attstattarget` in `pg_attribute` to use `NullableDatum`, allowing null representation for default statistics target in PostgreSQL 17. Relevant PG commit: 6a004f1be87d34cfe51acf2fe2552d2b08a79273 https://github.com/postgres/postgres/commit/6a004f1be87d34cfe51acf2fe2552d2b08a79273 ```diff -- verify statistics is set SELECT c.relname, a.attstattarget FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid AND c.relname LIKE 'test\_idx%' ORDER BY c.relname, a.attnum; relname | attstattarget -----------+--------------- test_idx | 4646 - test_idx2 | -1 + test_idx2 | test_idx2 | 10000 test_idx2 | 3737 (4 rows) ``` --- src/test/regress/expected/alter_index.out | 14 +++++++++----- src/test/regress/sql/alter_index.sql | 8 ++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/test/regress/expected/alter_index.out b/src/test/regress/expected/alter_index.out index 4d4a725b3..a863b83b0 100644 --- a/src/test/regress/expected/alter_index.out +++ b/src/test/regress/expected/alter_index.out @@ -32,29 +32,33 @@ SELECT create_distributed_table('t2','a'); (1 row) -- verify statistics is set -SELECT c.relname, a.attstattarget +-- pg17 Changed `attstattarget` in `pg_attribute` to use `NullableDatum`, allowing null representation for default statistics target in PostgreSQL 17. +-- https://github.com/postgres/postgres/commit/6a004f1be87d34cfe51acf2fe2552d2b08a79273 +SELECT c.relname, + CASE WHEN a.attstattarget = -1 THEN NULL ELSE a.attstattarget END AS attstattarget FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid AND c.relname LIKE 'test\_idx%' ORDER BY c.relname, a.attnum; relname | attstattarget --------------------------------------------------------------------- test_idx | 4646 - test_idx2 | -1 + test_idx2 | test_idx2 | 10000 test_idx2 | 3737 (4 rows) \c - - - :worker_1_port -SELECT c.relname, a.attstattarget +SELECT c.relname, + CASE WHEN a.attstattarget = -1 THEN NULL ELSE a.attstattarget END AS attstattarget FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid AND c.relname SIMILAR TO 'test\_idx%\_\d%' ORDER BY c.relname, a.attnum; relname | attstattarget --------------------------------------------------------------------- - test_idx2_980004 | -1 + test_idx2_980004 | test_idx2_980004 | 10000 test_idx2_980004 | 3737 - test_idx2_980006 | -1 + test_idx2_980006 | test_idx2_980006 | 10000 test_idx2_980006 | 3737 test_idx_980000 | 4646 diff --git a/src/test/regress/sql/alter_index.sql b/src/test/regress/sql/alter_index.sql index 3531bad18..2f3a39c57 100644 --- a/src/test/regress/sql/alter_index.sql +++ b/src/test/regress/sql/alter_index.sql @@ -23,13 +23,17 @@ ALTER INDEX test_idx2 ALTER COLUMN 2 SET STATISTICS 99999; SELECT create_distributed_table('t2','a'); -- verify statistics is set -SELECT c.relname, a.attstattarget +-- pg17 Changed `attstattarget` in `pg_attribute` to use `NullableDatum`, allowing null representation for default statistics target in PostgreSQL 17. +-- https://github.com/postgres/postgres/commit/6a004f1be87d34cfe51acf2fe2552d2b08a79273 +SELECT c.relname, + CASE WHEN a.attstattarget = -1 THEN NULL ELSE a.attstattarget END AS attstattarget FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid AND c.relname LIKE 'test\_idx%' ORDER BY c.relname, a.attnum; \c - - - :worker_1_port -SELECT c.relname, a.attstattarget +SELECT c.relname, + CASE WHEN a.attstattarget = -1 THEN NULL ELSE a.attstattarget END AS attstattarget FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid AND c.relname SIMILAR TO 'test\_idx%\_\d%' ORDER BY c.relname, a.attnum; From 84b52fc908be62bc6da516288683a6cc8713632f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:27:49 +0300 Subject: [PATCH 059/155] PG17 compatibility - Check if there are blocks left in columnar_scan_analyze_next_block (#7738) In PG17, the outer loop in `acquire_sample_rows()` changed from `while (BlockSampler_HasMore(&bs))` to `while (table_scan_analyze_next_block(scan, stream))` Relevant PG commit: 041b96802efa33d2bc9456f2ad946976b92b5ae1 https://github.com/postgres/postgres/commit/041b96802efa33d2bc9456f2ad946976b92b5ae1 It is expected that the `scan_analyze_next_block` function will check if there are any blocks left. So we add that check in `columnar_scan_analyze_next_block` Without this fix, we will have an indefinite loop causing timeout. Specifically, in our test schedules, `multi schedule` stuck at `drop_column_partitioned_table` test `multi-mx` schedule stuck at `start_stop_metadata_sync` test `columnar schedule` stuck at `columnar_create` test --- src/backend/columnar/columnar_tableam.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 0e6c423c2..1292f7b99 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -1437,7 +1437,19 @@ columnar_scan_analyze_next_block(TableScanDesc scan, * to pages boundaries. So not much to do here. We return true anyway * so acquire_sample_rows() in analyze.c would call our * columnar_scan_analyze_next_tuple() callback. + * In PG17, we return false in case there is no buffer left, since + * the outer loop changed in acquire_sample_rows(), and it is + * expected for the scan_analyze_next_block function to check whether + * there are any blocks left in the block sampler. */ +#if PG_VERSION_NUM >= PG_VERSION_17 + Buffer buf = read_stream_next_buffer(stream, NULL); + if (!BufferIsValid(buf)) + { + return false; + } + ReleaseBuffer(buf); +#endif return true; } From a93887b3de09ddea53d020984fdc9c877524ec72 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 18 Nov 2024 18:09:43 +0300 Subject: [PATCH 060/155] PG17 compatibility: Check whether table AM is default (#7747) PG 17 added support for DEFAULT in ALTER TABLE .. SET ACCESS METHOD Relevant PG commit: d61a6cad6418f643a5773352038d0dfe5d3535b8 https://github.com/postgres/postgres/commit/d61a6cad6418f643a5773352038d0dfe5d3535b8 In that case, name in `AlterTableCmd->name` would be null. Add a null check here to avoid crash. --- src/backend/columnar/columnar_tableam.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 1292f7b99..148ccb507 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -2256,7 +2256,9 @@ ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions "Specify SET ACCESS METHOD before storage parameters, or use separate ALTER TABLE commands."))); } - destIsColumnar = (strcmp(alterTableCmd->name, COLUMNAR_AM_NAME) == 0); + destIsColumnar = (strcmp(alterTableCmd->name ? alterTableCmd->name : + default_table_access_method, + COLUMNAR_AM_NAME) == 0); if (srcIsColumnar && !destIsColumnar) { From ed137001a5c5f68fbc63ced9c41bf34abb748b49 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:26:45 +0300 Subject: [PATCH 061/155] PG17 compatibility: add COLLPROVIDER_BUILTIN option and fix tests (#7752) In PG17 adds builtin C.UTF-8 locale option, we add it in the code to avoid "unknown collation provider" in vanilla tests. Relevant PG commit: https://github.com/postgres/postgres/commit/f69319f2f1fb16eda4b535bcccec90dff3a6795e f69319f2f1fb16eda4b535bcccec90dff3a6795e Also in PG17, colliculocale, daticulocale renamed to colllocale, datlocale Here we fix the following tests to avoid alternative output pg15 pg16 multi_mx_create_table multi_schema_support Relevant PG commit: https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d f696c0cd5f299f1b51e214efc55a22a782cc175d --- src/backend/distributed/commands/collation.c | 1 + src/include/pg_version_compat.h | 2 + .../expected/multi_mx_create_table.out | 9 ++++- .../regress/expected/multi_schema_support.out | 9 ++++- src/test/regress/expected/pg15.out | 33 ++++++++++++++--- src/test/regress/expected/pg16.out | 16 ++++---- .../regress/sql/multi_mx_create_table.sql | 9 ++++- src/test/regress/sql/multi_schema_support.sql | 9 ++++- src/test/regress/sql/pg15.sql | 37 ++++++++++++++++--- src/test/regress/sql/pg16.sql | 4 +- 10 files changed, 103 insertions(+), 26 deletions(-) diff --git a/src/backend/distributed/commands/collation.c b/src/backend/distributed/commands/collation.c index 1a8c211f9..4a47b5c18 100644 --- a/src/backend/distributed/commands/collation.c +++ b/src/backend/distributed/commands/collation.c @@ -132,6 +132,7 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati char *schemaName = get_namespace_name(collnamespace); *quotedCollationName = quote_qualified_identifier(schemaName, collname); const char *providerString = + collprovider == COLLPROVIDER_BUILTIN ? "builtin" : collprovider == COLLPROVIDER_DEFAULT ? "default" : collprovider == COLLPROVIDER_ICU ? "icu" : collprovider == COLLPROVIDER_LIBC ? "libc" : NULL; diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 80c4e9d3d..0db4f9a26 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -144,6 +144,8 @@ getStxstattarget_compat(HeapTuple tup) #define getProcNo_compat(a) (a->pgprocno) #define getLxid_compat(a) (a->lxid) +#define COLLPROVIDER_BUILTIN 'b' + #endif #if PG_VERSION_NUM >= PG_VERSION_16 diff --git a/src/test/regress/expected/multi_mx_create_table.out b/src/test/regress/expected/multi_mx_create_table.out index ac7f90826..72f16aea5 100644 --- a/src/test/regress/expected/multi_mx_create_table.out +++ b/src/test/regress/expected/multi_mx_create_table.out @@ -56,7 +56,14 @@ SET search_path TO public; SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset -\if :server_version_ge_16 +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d +SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN datlocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset +\elif :server_version_ge_16 -- In PG16, read-only server settings lc_collate and lc_ctype are removed -- Relevant PG commit: b0f6c437160db640d4ea3e49398ebc3ba39d1982 SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN daticulocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset diff --git a/src/test/regress/expected/multi_schema_support.out b/src/test/regress/expected/multi_schema_support.out index 2de95266b..e6b5ac9a9 100644 --- a/src/test/regress/expected/multi_schema_support.out +++ b/src/test/regress/expected/multi_schema_support.out @@ -350,7 +350,14 @@ SET search_path TO public; SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset -\if :server_version_ge_16 +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d +SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN datlocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset +\elif :server_version_ge_16 -- In PG16, read-only server settings lc_collate and lc_ctype are removed -- Relevant PG commit: b0f6c437160db640d4ea3e49398ebc3ba39d1982 SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN daticulocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index fcbb0cd12..c06142671 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -51,9 +51,32 @@ SELECT result FROM run_command_on_all_nodes(' (3 rows) -SELECT result FROM run_command_on_all_nodes(' - SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test''; -'); +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +SELECT '$$' || + 'SELECT colllocale FROM pg_collation WHERE collname = ''german_phonebook_test'';' + || '$$' + AS worker_query_1 \gset +SELECT '$$' || + 'SELECT colllocale FROM pg_collation WHERE collname = ''default_provider'';' + || '$$' + AS worker_query_2 \gset +\else +SELECT '$$' || + 'SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test'';' + || '$$' + AS worker_query_1 \gset +SELECT '$$' || + 'SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider'';' + || '$$' + AS worker_query_2 \gset +\endif +SELECT result FROM run_command_on_all_nodes(:worker_query_1); result --------------------------------------------------------------------- de-u-co-phonebk @@ -83,9 +106,7 @@ SELECT result FROM run_command_on_all_nodes(' POSIX (3 rows) -SELECT result FROM run_command_on_all_nodes(' - SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider''; -'); +SELECT result FROM run_command_on_all_nodes(:worker_query_2); result --------------------------------------------------------------------- diff --git a/src/test/regress/expected/pg16.out b/src/test/regress/expected/pg16.out index 8d47b6f1b..0af010348 100644 --- a/src/test/regress/expected/pg16.out +++ b/src/test/regress/expected/pg16.out @@ -330,14 +330,14 @@ SELECT create_distributed_table('test_collation_rules', 'a'); (1 row) INSERT INTO test_collation_rules VALUES ('Abernathy'), ('apple'), ('bird'), ('Boston'), ('Graham'), ('green'); -SELECT collname, collprovider, colliculocale, collicurules +SELECT collname, collprovider, collicurules FROM pg_collation WHERE collname like '%_rule%' ORDER BY 1; - collname | collprovider | colliculocale | collicurules + collname | collprovider | collicurules --------------------------------------------------------------------- - default_rule | i | und | - special_rule | i | und | &a < g + default_rule | i | + special_rule | i | &a < g (2 rows) SELECT * FROM test_collation_rules ORDER BY a COLLATE default_rule; @@ -364,14 +364,14 @@ SELECT * FROM test_collation_rules ORDER BY a COLLATE special_rule; \c - - - :worker_1_port SET search_path TO pg16; -SELECT collname, collprovider, colliculocale, collicurules +SELECT collname, collprovider, collicurules FROM pg_collation WHERE collname like '%_rule%' ORDER BY 1; - collname | collprovider | colliculocale | collicurules + collname | collprovider | collicurules --------------------------------------------------------------------- - default_rule | i | und | - special_rule | i | und | &a < g + default_rule | i | + special_rule | i | &a < g (2 rows) SELECT * FROM test_collation_rules ORDER BY a COLLATE default_rule; diff --git a/src/test/regress/sql/multi_mx_create_table.sql b/src/test/regress/sql/multi_mx_create_table.sql index de3468415..2b24da8cd 100644 --- a/src/test/regress/sql/multi_mx_create_table.sql +++ b/src/test/regress/sql/multi_mx_create_table.sql @@ -55,8 +55,15 @@ SET search_path TO public; SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset -\if :server_version_ge_16 +\if :server_version_ge_17 +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d +SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN datlocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset +\elif :server_version_ge_16 -- In PG16, read-only server settings lc_collate and lc_ctype are removed -- Relevant PG commit: b0f6c437160db640d4ea3e49398ebc3ba39d1982 SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN daticulocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset diff --git a/src/test/regress/sql/multi_schema_support.sql b/src/test/regress/sql/multi_schema_support.sql index 146cf78d4..13be94857 100644 --- a/src/test/regress/sql/multi_schema_support.sql +++ b/src/test/regress/sql/multi_schema_support.sql @@ -297,8 +297,15 @@ SET search_path TO public; SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset -\if :server_version_ge_16 +\if :server_version_ge_17 +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d +SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN datlocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset +\elif :server_version_ge_16 -- In PG16, read-only server settings lc_collate and lc_ctype are removed -- Relevant PG commit: b0f6c437160db640d4ea3e49398ebc3ba39d1982 SELECT quote_ident((SELECT CASE WHEN datlocprovider='i' THEN daticulocale ELSE datcollate END FROM pg_database WHERE datname = current_database())) as current_locale \gset diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index fe60222dd..0d53bc9dd 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -41,9 +41,36 @@ SELECT result FROM run_command_on_all_nodes(' SELECT result FROM run_command_on_all_nodes(' SELECT collctype FROM pg_collation WHERE collname = ''german_phonebook_test''; '); -SELECT result FROM run_command_on_all_nodes(' - SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test''; -'); + +-- PG17 renamed colliculocale to colllocale +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/f696c0cd5f299f1b51e214efc55a22a782cc175d + +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset + +\if :server_version_ge_17 +SELECT '$$' || + 'SELECT colllocale FROM pg_collation WHERE collname = ''german_phonebook_test'';' + || '$$' + AS worker_query_1 \gset +SELECT '$$' || + 'SELECT colllocale FROM pg_collation WHERE collname = ''default_provider'';' + || '$$' + AS worker_query_2 \gset +\else +SELECT '$$' || + 'SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test'';' + || '$$' + AS worker_query_1 \gset +SELECT '$$' || + 'SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider'';' + || '$$' + AS worker_query_2 \gset +\endif + +SELECT result FROM run_command_on_all_nodes(:worker_query_1); -- with non-icu provider, colliculocale will be null, collcollate and collctype will be set CREATE COLLATION default_provider (provider = libc, lc_collate = "POSIX", lc_ctype = "POSIX"); @@ -54,9 +81,7 @@ SELECT result FROM run_command_on_all_nodes(' SELECT result FROM run_command_on_all_nodes(' SELECT collctype FROM pg_collation WHERE collname = ''default_provider''; '); -SELECT result FROM run_command_on_all_nodes(' - SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider''; -'); +SELECT result FROM run_command_on_all_nodes(:worker_query_2); -- -- In PG15, Renaming triggers on partitioned tables had two problems diff --git a/src/test/regress/sql/pg16.sql b/src/test/regress/sql/pg16.sql index 82e9edf1e..736119914 100644 --- a/src/test/regress/sql/pg16.sql +++ b/src/test/regress/sql/pg16.sql @@ -159,7 +159,7 @@ CREATE TABLE test_collation_rules (a text); SELECT create_distributed_table('test_collation_rules', 'a'); INSERT INTO test_collation_rules VALUES ('Abernathy'), ('apple'), ('bird'), ('Boston'), ('Graham'), ('green'); -SELECT collname, collprovider, colliculocale, collicurules +SELECT collname, collprovider, collicurules FROM pg_collation WHERE collname like '%_rule%' ORDER BY 1; @@ -170,7 +170,7 @@ SELECT * FROM test_collation_rules ORDER BY a COLLATE special_rule; \c - - - :worker_1_port SET search_path TO pg16; -SELECT collname, collprovider, colliculocale, collicurules +SELECT collname, collprovider, collicurules FROM pg_collation WHERE collname like '%_rule%' ORDER BY 1; From b29ecd1b12ee28cd787535e75f17df4e3b70d07f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:02:04 +0300 Subject: [PATCH 062/155] citus_indent fix (#7746) --- src/include/distributed/commands.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index ec2cb8a57..32bb38100 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -80,11 +80,11 @@ typedef enum DistOpsOperationType */ typedef struct DistributeObjectOps { - char * (*deparse)(Node *); + char *(*deparse)(Node *); void (*qualify)(Node *); - List * (*preprocess)(Node *, const char *, ProcessUtilityContext); - List * (*postprocess)(Node *, const char *); - List * (*address)(Node *, bool, bool); + List *(*preprocess)(Node *, const char *, ProcessUtilityContext); + List *(*postprocess)(Node *, const char *); + List *(*address)(Node *, bool, bool); bool markDistributed; /* fields used by common implementations, omitted for specialized implementations */ From 0fed87ada9bd25cce6b5c097bd0c720df9661f96 Mon Sep 17 00:00:00 2001 From: Colm Date: Tue, 19 Nov 2024 21:14:57 +0000 Subject: [PATCH 063/155] PG17 compatibility: Preserve DEBUG output in cte_inline (#7755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression test cte_inline has the following diff; ``` DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 ``` DEBUG message `query has a single distribution column value` does not appear with PG17. This is because PG17 can recognize when a Result node does not need to have an input node, so the predicate on the distribution column is not present in the query plan. Comparing the query plan obtained before PG17: ``` │ Result │ │ One-Time Filter: false │ │ -> GroupAggregate │ │ -> Seq Scan on public.test_table │ │ Filter: (test_table.key = 1) │ ``` with the PG17 query plan: ``` ┌──────────────────────────────────┐ │ QUERY PLAN │ ├──────────────────────────────────┤ │ Result │ │ One-Time Filter: false │ └──────────────────────────────────┘ ``` we see that the Result node in the PG16 plan has an Aggregate node, but the Result node in the PG17 plan does not have any input node; PG17 recognizes it is not needed given a Filter that evaluates to False at compile-time. The Result node is present in both plans because PG in both versions can recognize when a combination of predicates equate to false at compile time; this is the because the successive predicates in the test query (key=6, key=5, key=4, etc) become contradictory when the CTEs are inlined. Here is an example query showing the effect of the CTE inlining: ``` select count(*), key FROM test_table WHERE key = 1 AND key = 2 GROUP BY key; ``` In this case, the WHERE clause obviously evaluates to False. The PG16 query plan for this query is: ``` ┌────────────────────────────────────┐ │ QUERY PLAN │ ├────────────────────────────────────┤ │ GroupAggregate │ │ -> Result │ │ One-Time Filter: false │ │ -> Seq Scan on test_table │ │ Filter: (key = 1) │ └────────────────────────────────────┘ ``` The PG17 query plan is: ``` ┌────────────────────────────────┐ │ QUERY PLAN │ ├────────────────────────────────┤ │ GroupAggregate │ │ -> Result │ │ One-Time Filter: false │ └────────────────────────────────┘ ``` In both plans the PG optimizer is able to derive the predicate 1=2 from the equivalence class { key, 1, 2 } and then constant fold this to False. But, in the PG16 plan the Result node has an input node (a sequential scan on test_table), while in the PG17 plan the Result node does not have any input. This is because PG17 recognizes that when the Result filter resolves to False at compile time it is not necessary to set an input on the Result. I think this is a consequence of this PG17 commit: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=b262ad440 which handles redundant IS [NOT] NULL predicates, but also refactored evaluating of predicates to true/false at compile-time, enabling optimizations such as those seen here. Given the reason for the diff, the fix preserves the test output by modifying the query so the predicates are not contradictory when the CTEs are inlined. --- src/test/regress/expected/cte_inline.out | 12 ++++++------ src/test/regress/expected/cte_inline_0.out | 12 ++++++------ src/test/regress/sql/cte_inline.sql | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 39d48e915..7af842e29 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -654,12 +654,12 @@ WITH cte_1 AS ( WITH cte_1 AS ( WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) SELECT * FROM cte_1) - SELECT * FROM cte_1 WHERE key = 1) - SELECT * FROM cte_1 WHERE key = 2) - SELECT * FROM cte_1 WHERE key = 3) - SELECT * FROM cte_1 WHERE key = 4) - SELECT * FROM cte_1 WHERE key = 5) -SELECT * FROM cte_1 WHERE key = 6; + SELECT * FROM cte_1 WHERE key >= 1) + SELECT * FROM cte_1 WHERE key >= 2) + SELECT * FROM cte_1 WHERE key >= 3) + SELECT * FROM cte_1 WHERE key >= 4) + SELECT * FROM cte_1 WHERE key >= 5) +SELECT * FROM cte_1 WHERE key = 1; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning diff --git a/src/test/regress/expected/cte_inline_0.out b/src/test/regress/expected/cte_inline_0.out index a727d4d21..e5afa4ee3 100644 --- a/src/test/regress/expected/cte_inline_0.out +++ b/src/test/regress/expected/cte_inline_0.out @@ -654,12 +654,12 @@ WITH cte_1 AS ( WITH cte_1 AS ( WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) SELECT * FROM cte_1) - SELECT * FROM cte_1 WHERE key = 1) - SELECT * FROM cte_1 WHERE key = 2) - SELECT * FROM cte_1 WHERE key = 3) - SELECT * FROM cte_1 WHERE key = 4) - SELECT * FROM cte_1 WHERE key = 5) -SELECT * FROM cte_1 WHERE key = 6; + SELECT * FROM cte_1 WHERE key >= 1) + SELECT * FROM cte_1 WHERE key >= 2) + SELECT * FROM cte_1 WHERE key >= 3) + SELECT * FROM cte_1 WHERE key >= 4) + SELECT * FROM cte_1 WHERE key >= 5) +SELECT * FROM cte_1 WHERE key = 1; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning diff --git a/src/test/regress/sql/cte_inline.sql b/src/test/regress/sql/cte_inline.sql index 862a8510b..3f3e14c88 100644 --- a/src/test/regress/sql/cte_inline.sql +++ b/src/test/regress/sql/cte_inline.sql @@ -350,12 +350,12 @@ WITH cte_1 AS ( WITH cte_1 AS ( WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) SELECT * FROM cte_1) - SELECT * FROM cte_1 WHERE key = 1) - SELECT * FROM cte_1 WHERE key = 2) - SELECT * FROM cte_1 WHERE key = 3) - SELECT * FROM cte_1 WHERE key = 4) - SELECT * FROM cte_1 WHERE key = 5) -SELECT * FROM cte_1 WHERE key = 6; + SELECT * FROM cte_1 WHERE key >= 1) + SELECT * FROM cte_1 WHERE key >= 2) + SELECT * FROM cte_1 WHERE key >= 3) + SELECT * FROM cte_1 WHERE key >= 4) + SELECT * FROM cte_1 WHERE key >= 5) +SELECT * FROM cte_1 WHERE key = 1; From 680c23ffcf0089c6dc7658637a142a34d0bbc3df Mon Sep 17 00:00:00 2001 From: Colm Date: Wed, 20 Nov 2024 11:51:16 +0000 Subject: [PATCH 064/155] PG17 compatibility: add/fix tests with correlated subqueries that can be pulled to a join (#7745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix Test Failure in subquery_in_where, set_operations, dml_recursive in PG17 #7741 The test failures are caused by[ this commit in PG17](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639), which enables correlated subqueries to be pulled up to a join. Prior to this, the correlated subquery was implemented as a subplan. In citus, it is not possible to pushdown a correlated subplan, but with a different plan in PG17 the query can be executed, per the test diff from `subquery_in_where`: ``` 37,39c37,41 < DEBUG: generating subplan XXX_1 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM public.events_table < DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ... < ERROR: correlated subqueries are not supported when the FROM clause contains a CTE or subquery --- > count > --------------------------------------------------------------------- > 0 > (1 row) > ``` This is because with pg17 `= ANY subquery` in the queries can be implemented as a join, instead of as a subplan filter on a table scan. For example, `SELECT * FROM test a WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) ORDER BY 1,2` (from set_operations) has this plan in pg17; note that the subquery is the inner side of a nested loop join: ``` ┌───────────────────────────────────────────────────┐ │ QUERY PLAN │ ├───────────────────────────────────────────────────┤ │ Sort │ │ Sort Key: a.x, a.y │ │ -> Nested Loop │ │ -> Seq Scan on test a │ │ -> Subquery Scan on "ANY_subquery" │ │ Filter: (a.x = "ANY_subquery".x) │ │ -> HashAggregate │ │ Group Key: b.x │ │ -> Append │ │ -> Seq Scan on test b │ │ -> Seq Scan on test c │ │ Filter: (a.x = x) │ └───────────────────────────────────────────────────┘ ``` and this plan in pg16 (and previous pg versions); the subquery is a correlated subplan filter on a table scan: ``` ┌───────────────────────────────────────────────┐ │ QUERY PLAN │ ├───────────────────────────────────────────────┤ │ Sort │ │ Sort Key: a.x, a.y │ │ -> Seq Scan on test a │ │ Filter: (SubPlan 1) │ │ SubPlan 1 │ │ -> HashAggregate │ │ Group Key: b.x │ │ -> Append │ │ -> Seq Scan on test b │ │ -> Seq Scan on test c │ │ Filter: (a.x = x) │ └───────────────────────────────────────────────┘ ``` The fix Modifies the queries causing the test failures so that an ANY subquery is not folded to a join, preserving the expected output of the tests. A similar approach was taken for existing regress tests in the[ postgres commit](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639). See the `join `regress test, for example. We also add pg17 specific tests that leverage this improvement in Postgres with Citus distributed planning as well. --- src/test/regress/expected/dml_recursive.out | 3 +- src/test/regress/expected/pg17.out | 354 ++++++++++++++++++ src/test/regress/expected/pg17_0.out | 295 +++++++++++++++ src/test/regress/expected/set_operations.out | 4 +- .../regress/expected/subquery_in_where.out | 4 +- src/test/regress/multi_schedule | 1 + src/test/regress/sql/dml_recursive.sql | 3 +- src/test/regress/sql/pg17.sql | 182 +++++++++ src/test/regress/sql/set_operations.sql | 2 +- src/test/regress/sql/subquery_in_where.sql | 2 +- 10 files changed, 842 insertions(+), 8 deletions(-) create mode 100644 src/test/regress/expected/pg17.out create mode 100644 src/test/regress/expected/pg17_0.out create mode 100644 src/test/regress/sql/pg17.sql diff --git a/src/test/regress/expected/dml_recursive.out b/src/test/regress/expected/dml_recursive.out index cc4058def..be131f661 100644 --- a/src/test/regress/expected/dml_recursive.out +++ b/src/test/regress/expected/dml_recursive.out @@ -266,6 +266,7 @@ ERROR: complex joins are only supported when all distributed tables are co-loca -- again a correlated subquery -- this time distribution key eq. exists -- however recursive planning is prevented due to correlated subqueries +-- that cannot be folded to joins. UPDATE second_distributed_table SET @@ -285,7 +286,7 @@ FROM AND second_distributed_table.tenant_id IN ( - SELECT s2.tenant_id + SELECT s2.tenant_id || random()::text FROM second_distributed_table as s2 GROUP BY d1.tenant_id, s2.tenant_id ) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out new file mode 100644 index 000000000..8943e78e7 --- /dev/null +++ b/src/test/regress/expected/pg17.out @@ -0,0 +1,354 @@ +-- +-- PG17 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +-- PG17 has the capabilty to pull up a correlated ANY subquery to a join if +-- the subquery only refers to its immediate parent query. Previously, the +-- subquery needed to be implemented as a SubPlan node, typically as a +-- filter on a scan or join node. This PG17 capability enables Citus to +-- run queries with correlated subqueries in certain cases, as shown here. +-- Relevant PG commit: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639 +-- This feature is tested for all PG versions, not just PG17; each test query with +-- a correlated subquery should fail with PG version < 17.0, but the test query +-- rewritten to reflect how PG17 optimizes it should succeed with PG < 17.0 +CREATE SCHEMA pg17_corr_subq_folding; +SET search_path TO pg17_corr_subq_folding; +SET citus.next_shard_id TO 20240017; +SET citus.shard_count TO 2; +SET citus.shard_replication_factor TO 1; +CREATE TABLE test (x int, y int); +SELECT create_distributed_table('test', 'x'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test VALUES (1,1), (2,2); +-- Query 1: WHERE clause has a correlated subquery with a UNION. PG17 can plan +-- this as a nested loop join with the subquery as the inner. The correlation +-- is on the distribution column so the join can be pushed down by Citus. +explain (costs off) +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.x, remote_scan.y + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Nested Loop + -> Seq Scan on test_20240017 a + -> Subquery Scan on "ANY_subquery" + Filter: (a.x = "ANY_subquery".x) + -> HashAggregate + Group Key: b.x + -> Append + -> Seq Scan on test_20240017 b + -> Seq Scan on test_20240017 c + Filter: (a.x = x) +(17 rows) + +SET client_min_messages TO DEBUG2; +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; +DEBUG: Router planner cannot handle multi-shard select queries + x | y +--------------------------------------------------------------------- + 1 | 1 + 2 | 2 +(2 rows) + +RESET client_min_messages; +-- Query 1 rewritten with subquery pulled up to a join, as done by PG17 planner; +-- this query can be run without issues by Citus with older (pre PG17) PGs. +explain (costs off) +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.x, remote_scan.y + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Nested Loop + -> Seq Scan on test_20240017 a + -> Subquery Scan on dt1 + Filter: (a.x = dt1.x) + -> HashAggregate + Group Key: b.x + -> Append + -> Seq Scan on test_20240017 b + -> Seq Scan on test_20240017 c + Filter: (a.x = x) +(17 rows) + +SET client_min_messages TO DEBUG2; +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; +DEBUG: Router planner cannot handle multi-shard select queries + x | y +--------------------------------------------------------------------- + 1 | 1 + 2 | 2 +(2 rows) + +RESET client_min_messages; +CREATE TABLE users (user_id int, time int, dept int, info bigint); +CREATE TABLE events (user_id int, time int, event_type int, payload text); +select create_distributed_table('users', 'user_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_table('events', 'user_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into users +select i, 2021 + (i % 3), i % 5, 99999 * i from generate_series(1, 10) i; +insert into events +select i % 10 + 1, 2021 + (i % 3), i %11, md5((i*i)::text) from generate_series(1, 100) i; +-- Query 2. In Citus correlated subqueries can not be used in the WHERE +-- clause but if the subquery can be pulled up to a join it becomes possible +-- for Citus to run the query, per this example. Pre PG17 the suqbuery +-- was implemented as a SubPlan filter on the events table scan. +EXPLAIN (costs off) +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Hash Join + Hash Cond: ((events."time" = users."time") AND (events.user_id = users.user_id)) + -> Seq Scan on events_20240021 events + -> Hash + -> HashAggregate + Group Key: users."time", users.user_id + -> Seq Scan on users_20240019 users +(14 rows) + +SET client_min_messages TO DEBUG2; +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); +DEBUG: CTE event_id is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 31 +(1 row) + +RESET client_min_messages; +-- Query 2 rewritten with subquery pulled up to a join, as done by pg17 planner. Citus +-- Citus is able to run this query with previous pg versions. Note that the CTE can be +-- disregarded because it is inlined, being only referenced once. +EXPLAIN (COSTS OFF) +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Hash Join + Hash Cond: ((events.user_id = users.user_id) AND (events."time" = users."time")) + -> Seq Scan on events_20240021 events + -> Hash + -> HashAggregate + Group Key: users.user_id, users."time" + -> Seq Scan on users_20240019 users +(14 rows) + +SET client_min_messages TO DEBUG2; +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 31 +(1 row) + +RESET client_min_messages; +-- Query 3: another example where recursive planning was prevented due to +-- correlated subqueries, but with PG17 folding the subquery to a join it is +-- possible for Citus to plan and run the query. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Group Key: remote_scan.dept + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: users.dept + -> Sort + Sort Key: users.dept + -> Nested Loop Semi Join + -> Hash Join + Hash Cond: (d1.user_id = users.user_id) + -> Seq Scan on events_20240021 d1 + -> Hash + -> Seq Scan on users_20240019 users + Filter: (dept = ANY ('{3,4}'::integer[])) + -> Subquery Scan on "ANY_subquery" + Filter: (d1.user_id = "ANY_subquery".user_id) + -> HashAggregate + Group Key: s2.user_id + -> Seq Scan on users_20240019 s2 +(23 rows) + +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; +DEBUG: Router planner cannot handle multi-shard select queries + dept | sum +--------------------------------------------------------------------- + 3 | 110 + 4 | 130 +(2 rows) + +RESET client_min_messages; +-- Query 3 rewritten in a similar way to how the PG17 pulls up the subquery; +-- the join is on the distribution key so Citus can push down. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Group Key: remote_scan.dept + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: users.dept + -> Sort + Sort Key: users.dept + -> Nested Loop + -> Hash Join + Hash Cond: (d1.user_id = users.user_id) + -> Seq Scan on events_20240021 d1 + -> Hash + -> Seq Scan on users_20240019 users + Filter: (dept = ANY ('{3,4}'::integer[])) + -> Subquery Scan on d2 + Filter: (d1.user_id = d2.user_id) + -> HashAggregate + Group Key: s2.user_id + -> Result + One-Time Filter: (d1.user_id IS NOT NULL) + -> Seq Scan on users_20240019 s2 +(25 rows) + +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; +DEBUG: Router planner cannot handle multi-shard select queries + dept | sum +--------------------------------------------------------------------- + 3 | 110 + 4 | 130 +(2 rows) + +RESET client_min_messages; +RESET search_path; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; +DROP SCHEMA pg17_corr_subq_folding CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table pg17_corr_subq_folding.test +drop cascades to table pg17_corr_subq_folding.users +drop cascades to table pg17_corr_subq_folding.events +\if :server_version_ge_17 +\else +\q +\endif +-- PG17-specific tests go here. +-- diff --git a/src/test/regress/expected/pg17_0.out b/src/test/regress/expected/pg17_0.out new file mode 100644 index 000000000..66dba2c29 --- /dev/null +++ b/src/test/regress/expected/pg17_0.out @@ -0,0 +1,295 @@ +-- +-- PG17 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +-- PG17 has the capabilty to pull up a correlated ANY subquery to a join if +-- the subquery only refers to its immediate parent query. Previously, the +-- subquery needed to be implemented as a SubPlan node, typically as a +-- filter on a scan or join node. This PG17 capability enables Citus to +-- run queries with correlated subqueries in certain cases, as shown here. +-- Relevant PG commit: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639 +-- This feature is tested for all PG versions, not just PG17; each test query with +-- a correlated subquery should fail with PG version < 17.0, but the test query +-- rewritten to reflect how PG17 optimizes it should succeed with PG < 17.0 +CREATE SCHEMA pg17_corr_subq_folding; +SET search_path TO pg17_corr_subq_folding; +SET citus.next_shard_id TO 20240017; +SET citus.shard_count TO 2; +SET citus.shard_replication_factor TO 1; +CREATE TABLE test (x int, y int); +SELECT create_distributed_table('test', 'x'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test VALUES (1,1), (2,2); +-- Query 1: WHERE clause has a correlated subquery with a UNION. PG17 can plan +-- this as a nested loop join with the subquery as the inner. The correlation +-- is on the distribution column so the join can be pushed down by Citus. +explain (costs off) +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; +ERROR: cannot push down this subquery +DETAIL: Complex subqueries and CTEs are not supported within a UNION +SET client_min_messages TO DEBUG2; +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT x FROM pg17_corr_subq_folding.test b +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT x, y FROM pg17_corr_subq_folding.test a WHERE (x OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer) UNION SELECT c.y FROM pg17_corr_subq_folding.test c WHERE (a.x OPERATOR(pg_catalog.=) c.x))) ORDER BY x, y +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +ERROR: cannot push down this subquery +DETAIL: Complex subqueries and CTEs are not supported within a UNION +RESET client_min_messages; +-- Query 1 rewritten with subquery pulled up to a join, as done by PG17 planner; +-- this query can be run without issues by Citus with older (pre PG17) PGs. +explain (costs off) +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.x, remote_scan.y + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Nested Loop + -> Seq Scan on test_20240017 a + -> Subquery Scan on dt1 + Filter: (a.x = dt1.x) + -> HashAggregate + Group Key: b.x + -> Append + -> Seq Scan on test_20240017 b + -> Seq Scan on test_20240017 c + Filter: (a.x = x) +(17 rows) + +SET client_min_messages TO DEBUG2; +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; +DEBUG: Router planner cannot handle multi-shard select queries + x | y +--------------------------------------------------------------------- + 1 | 1 + 2 | 2 +(2 rows) + +RESET client_min_messages; +CREATE TABLE users (user_id int, time int, dept int, info bigint); +CREATE TABLE events (user_id int, time int, event_type int, payload text); +select create_distributed_table('users', 'user_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_table('events', 'user_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into users +select i, 2021 + (i % 3), i % 5, 99999 * i from generate_series(1, 10) i; +insert into events +select i % 10 + 1, 2021 + (i % 3), i %11, md5((i*i)::text) from generate_series(1, 100) i; +-- Query 2. In Citus correlated subqueries can not be used in the WHERE +-- clause but if the subquery can be pulled up to a join it becomes possible +-- for Citus to run the query, per this example. Pre PG17 the suqbuery +-- was implemented as a SubPlan filter on the events table scan. +EXPLAIN (costs off) +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); +ERROR: correlated subqueries are not supported when the FROM clause contains a CTE or subquery +SET client_min_messages TO DEBUG2; +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); +DEBUG: CTE event_id is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM pg17_corr_subq_folding.events +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time integer, event_type integer)) event_id WHERE (events_user_id OPERATOR(pg_catalog.=) ANY (SELECT users.user_id FROM pg17_corr_subq_folding.users WHERE (users."time" OPERATOR(pg_catalog.=) event_id.events_time))) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +ERROR: correlated subqueries are not supported when the FROM clause contains a CTE or subquery +RESET client_min_messages; +-- Query 2 rewritten with subquery pulled up to a join, as done by pg17 planner. Citus +-- Citus is able to run this query with previous pg versions. Note that the CTE can be +-- disregarded because it is inlined, being only referenced once. +EXPLAIN (COSTS OFF) +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Hash Join + Hash Cond: ((events.user_id = users.user_id) AND (events."time" = users."time")) + -> Seq Scan on events_20240021 events + -> Hash + -> HashAggregate + Group Key: users.user_id, users."time" + -> Seq Scan on users_20240019 users +(14 rows) + +SET client_min_messages TO DEBUG2; +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 31 +(1 row) + +RESET client_min_messages; +-- Query 3: another example where recursive planning was prevented due to +-- correlated subqueries, but with PG17 folding the subquery to a join it is +-- possible for Citus to plan and run the query. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; +ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +RESET client_min_messages; +-- Query 3 rewritten in a similar way to how the PG17 pulls up the subquery; +-- the join is on the distribution key so Citus can push down. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Group Key: remote_scan.dept + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: users.dept + -> Sort + Sort Key: users.dept + -> Nested Loop + -> Hash Join + Hash Cond: (d1.user_id = users.user_id) + -> Seq Scan on events_20240021 d1 + -> Hash + -> Seq Scan on users_20240019 users + Filter: (dept = ANY ('{3,4}'::integer[])) + -> Subquery Scan on d2 + Filter: (d1.user_id = d2.user_id) + -> HashAggregate + Group Key: s2.user_id + -> Result + One-Time Filter: (d1.user_id IS NOT NULL) + -> Seq Scan on users_20240019 s2 +(25 rows) + +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; +DEBUG: Router planner cannot handle multi-shard select queries + dept | sum +--------------------------------------------------------------------- + 3 | 110 + 4 | 130 +(2 rows) + +RESET client_min_messages; +RESET search_path; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; +DROP SCHEMA pg17_corr_subq_folding CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table pg17_corr_subq_folding.test +drop cascades to table pg17_corr_subq_folding.users +drop cascades to table pg17_corr_subq_folding.events +\if :server_version_ge_17 +\else +\q diff --git a/src/test/regress/expected/set_operations.out b/src/test/regress/expected/set_operations.out index f2e0616e7..15a0345b5 100644 --- a/src/test/regress/expected/set_operations.out +++ b/src/test/regress/expected/set_operations.out @@ -771,13 +771,13 @@ DEBUG: Router planner cannot handle multi-shard select queries (2 rows) -- correlated subquery with union in WHERE clause -SELECT * FROM test a WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) ORDER BY 1,2; +SELECT * FROM test a WHERE (x + random()) IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) ORDER BY 1,2; DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Router planner cannot handle multi-shard select queries DEBUG: generating subplan XXX_1 for subquery SELECT x FROM recursive_union.test b DEBUG: skipping recursive planning for the subquery since it contains references to outer queries DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT x, y FROM recursive_union.test a WHERE (x OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer) UNION SELECT c.y FROM recursive_union.test c WHERE (a.x OPERATOR(pg_catalog.=) c.x))) ORDER BY x, y +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT x, y FROM recursive_union.test a WHERE (((x)::double precision OPERATOR(pg_catalog.+) random()) OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer) UNION SELECT c.y FROM recursive_union.test c WHERE (a.x OPERATOR(pg_catalog.=) c.x))) ORDER BY x, y DEBUG: Router planner cannot handle multi-shard select queries DEBUG: skipping recursive planning for the subquery since it contains references to outer queries DEBUG: skipping recursive planning for the subquery since it contains references to outer queries diff --git a/src/test/regress/expected/subquery_in_where.out b/src/test/regress/expected/subquery_in_where.out index eb56acd87..990c29084 100644 --- a/src/test/regress/expected/subquery_in_where.out +++ b/src/test/regress/expected/subquery_in_where.out @@ -30,12 +30,12 @@ WITH event_id FROM events_table) SELECT Count(*) FROM event_id -WHERE events_user_id IN (SELECT user_id +WHERE (events_user_id, random()) IN (SELECT user_id, 1 FROM users_table WHERE users_table.time = events_time); DEBUG: CTE event_id is going to be inlined via distributed planning DEBUG: generating subplan XXX_1 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM public.events_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) event_id WHERE (events_user_id OPERATOR(pg_catalog.=) ANY (SELECT users_table.user_id FROM public.users_table WHERE (users_table."time" OPERATOR(pg_catalog.=) event_id.events_time))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) event_id WHERE ((events_user_id, random()) OPERATOR(pg_catalog.=) ANY (SELECT users_table.user_id, 1 FROM public.users_table WHERE (users_table."time" OPERATOR(pg_catalog.=) event_id.events_time))) ERROR: correlated subqueries are not supported when the FROM clause contains a CTE or subquery -- Recurring tuples as empty join tree SELECT * diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index d32757e48..32d1f9707 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -66,6 +66,7 @@ test: pg14 test: pg15 test: pg15_jsonpath detect_conn_close test: pg16 +test: pg17 test: drop_column_partitioned_table test: tableam diff --git a/src/test/regress/sql/dml_recursive.sql b/src/test/regress/sql/dml_recursive.sql index 89e654b6c..7337c9672 100644 --- a/src/test/regress/sql/dml_recursive.sql +++ b/src/test/regress/sql/dml_recursive.sql @@ -212,6 +212,7 @@ RETURNING *; -- again a correlated subquery -- this time distribution key eq. exists -- however recursive planning is prevented due to correlated subqueries +-- that cannot be folded to joins. UPDATE second_distributed_table SET @@ -231,7 +232,7 @@ FROM AND second_distributed_table.tenant_id IN ( - SELECT s2.tenant_id + SELECT s2.tenant_id || random()::text FROM second_distributed_table as s2 GROUP BY d1.tenant_id, s2.tenant_id ) diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql new file mode 100644 index 000000000..4fdde71ca --- /dev/null +++ b/src/test/regress/sql/pg17.sql @@ -0,0 +1,182 @@ +-- +-- PG17 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset + +-- PG17 has the capabilty to pull up a correlated ANY subquery to a join if +-- the subquery only refers to its immediate parent query. Previously, the +-- subquery needed to be implemented as a SubPlan node, typically as a +-- filter on a scan or join node. This PG17 capability enables Citus to +-- run queries with correlated subqueries in certain cases, as shown here. +-- Relevant PG commit: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639 + +-- This feature is tested for all PG versions, not just PG17; each test query with +-- a correlated subquery should fail with PG version < 17.0, but the test query +-- rewritten to reflect how PG17 optimizes it should succeed with PG < 17.0 + +CREATE SCHEMA pg17_corr_subq_folding; +SET search_path TO pg17_corr_subq_folding; +SET citus.next_shard_id TO 20240017; +SET citus.shard_count TO 2; +SET citus.shard_replication_factor TO 1; + +CREATE TABLE test (x int, y int); +SELECT create_distributed_table('test', 'x'); +INSERT INTO test VALUES (1,1), (2,2); + +-- Query 1: WHERE clause has a correlated subquery with a UNION. PG17 can plan +-- this as a nested loop join with the subquery as the inner. The correlation +-- is on the distribution column so the join can be pushed down by Citus. +explain (costs off) +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; + +SET client_min_messages TO DEBUG2; +SELECT * +FROM test a +WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) +ORDER BY 1,2; +RESET client_min_messages; + +-- Query 1 rewritten with subquery pulled up to a join, as done by PG17 planner; +-- this query can be run without issues by Citus with older (pre PG17) PGs. +explain (costs off) +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; + +SET client_min_messages TO DEBUG2; +SELECT a.* +FROM test a JOIN LATERAL (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) dt1 ON a.x = dt1.x +ORDER BY 1,2; +RESET client_min_messages; + +CREATE TABLE users (user_id int, time int, dept int, info bigint); +CREATE TABLE events (user_id int, time int, event_type int, payload text); +select create_distributed_table('users', 'user_id'); +select create_distributed_table('events', 'user_id'); + +insert into users +select i, 2021 + (i % 3), i % 5, 99999 * i from generate_series(1, 10) i; + +insert into events +select i % 10 + 1, 2021 + (i % 3), i %11, md5((i*i)::text) from generate_series(1, 100) i; + +-- Query 2. In Citus correlated subqueries can not be used in the WHERE +-- clause but if the subquery can be pulled up to a join it becomes possible +-- for Citus to run the query, per this example. Pre PG17 the suqbuery +-- was implemented as a SubPlan filter on the events table scan. +EXPLAIN (costs off) +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); + +SET client_min_messages TO DEBUG2; +WITH event_id + AS(SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events) +SELECT Count(*) +FROM event_id +WHERE (events_user_id) IN (SELECT user_id + FROM users + WHERE users.time = events_time); +RESET client_min_messages; + +-- Query 2 rewritten with subquery pulled up to a join, as done by pg17 planner. Citus +-- Citus is able to run this query with previous pg versions. Note that the CTE can be +-- disregarded because it is inlined, being only referenced once. +EXPLAIN (COSTS OFF) +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; + +SET client_min_messages TO DEBUG2; +SELECT Count(*) +FROM (SELECT user_id AS events_user_id, + time AS events_time, + event_type FROM events) dt1 +INNER JOIN (SELECT distinct user_id, time FROM users) dt + ON events_user_id = dt.user_id and events_time = dt.time; +RESET client_min_messages; + +-- Query 3: another example where recursive planning was prevented due to +-- correlated subqueries, but with PG17 folding the subquery to a join it is +-- possible for Citus to plan and run the query. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; + +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id IN + (SELECT s2.user_id FROM users as s2 + GROUP BY d1.user_id, s2.user_id)) dt +GROUP BY dept; +RESET client_min_messages; + +-- Query 3 rewritten in a similar way to how the PG17 pulls up the subquery; +-- the join is on the distribution key so Citus can push down. +EXPLAIN (costs off) +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; + +SET client_min_messages TO DEBUG2; +SELECT dept, sum(user_id) FROM +(SELECT users.dept, users.user_id +FROM users, events as d1 + JOIN LATERAL (SELECT s2.user_id FROM users as s2 + GROUP BY s2.user_id HAVING d1.user_id IS NOT NULL) as d2 ON 1=1 +WHERE d1.user_id = users.user_id + AND users.dept IN (3,4) + AND users.user_id = d2.user_id) dt +GROUP BY dept; +RESET client_min_messages; + +RESET search_path; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; +DROP SCHEMA pg17_corr_subq_folding CASCADE; + +\if :server_version_ge_17 +\else +\q +\endif + +-- PG17-specific tests go here. +-- diff --git a/src/test/regress/sql/set_operations.sql b/src/test/regress/sql/set_operations.sql index 633b5c0b5..58907a281 100644 --- a/src/test/regress/sql/set_operations.sql +++ b/src/test/regress/sql/set_operations.sql @@ -134,7 +134,7 @@ SELECT * FROM test a WHERE x NOT IN (SELECT x FROM test b WHERE y = 1 UNION SELE SELECT * FROM test a WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c) ORDER BY 1,2; -- correlated subquery with union in WHERE clause -SELECT * FROM test a WHERE x IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) ORDER BY 1,2; +SELECT * FROM test a WHERE (x + random()) IN (SELECT x FROM test b UNION SELECT y FROM test c WHERE a.x = c.x) ORDER BY 1,2; -- force unions to be planned while subqueries are being planned SELECT * FROM ((SELECT * FROM test) UNION (SELECT * FROM test) ORDER BY 1,2 LIMIT 5) as foo ORDER BY 1 DESC LIMIT 3; diff --git a/src/test/regress/sql/subquery_in_where.sql b/src/test/regress/sql/subquery_in_where.sql index 90386122f..8316508b7 100644 --- a/src/test/regress/sql/subquery_in_where.sql +++ b/src/test/regress/sql/subquery_in_where.sql @@ -25,7 +25,7 @@ WITH event_id FROM events_table) SELECT Count(*) FROM event_id -WHERE events_user_id IN (SELECT user_id +WHERE (events_user_id, random()) IN (SELECT user_id, 1 FROM users_table WHERE users_table.time = events_time); From 7e701befde6ab23523ad9569263e81daaf464cc7 Mon Sep 17 00:00:00 2001 From: Colm Date: Thu, 21 Nov 2024 19:22:30 +0000 Subject: [PATCH 065/155] PG17 compatibility: add helper function for EXPLAIN diffs in scalar subquery output (#7757) PG17 changed how scalar subquery outputs appear in EXPLAIN output (*). This commit changes impacted regress goldfiles to the PG17 format, and adds a helper function to covert pre-PG17 plans to the PG17 format. The conversion is required when testing Citus on pgversions prior to 17. The helper function can and should be removed when 17 becomes the minimum supported version. (*) https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb --- src/test/regress/expected/ch_bench_having.out | 42 ++++++++----- .../regress/expected/ch_bench_having_mx.out | 44 +++++++------ src/test/regress/expected/having_subquery.out | 8 ++- src/test/regress/expected/multi_explain.out | 18 +++--- src/test/regress/expected/multi_explain_0.out | 18 +++--- .../expected/multi_reference_table.out | 6 +- src/test/regress/expected/multi_subquery.out | 24 +++++--- .../regress/expected/multi_test_helpers.out | 59 ++++++++++++++++++ src/test/regress/sql/ch_bench_having.sql | 11 +++- src/test/regress/sql/ch_bench_having_mx.sql | 11 +++- src/test/regress/sql/having_subquery.sql | 2 + src/test/regress/sql/multi_explain.sql | 6 +- .../regress/sql/multi_reference_table.sql | 2 + src/test/regress/sql/multi_subquery.sql | 8 +++ src/test/regress/sql/multi_test_helpers.sql | 61 +++++++++++++++++++ 15 files changed, 257 insertions(+), 63 deletions(-) diff --git a/src/test/regress/expected/ch_bench_having.out b/src/test/regress/expected/ch_bench_having.out index 29feb0305..02fba11c8 100644 --- a/src/test/regress/expected/ch_bench_having.out +++ b/src/test/regress/expected/ch_bench_having.out @@ -12,6 +12,7 @@ SELECT create_distributed_table('stock','s_w_id'); (1 row) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock @@ -19,15 +20,16 @@ where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock) group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 -> Aggregate @@ -53,27 +55,29 @@ order by s_i_id; Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: stock.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Seq Scan on stock_1640000 stock - Filter: ((s_order_cnt)::numeric > $0) + Filter: ((s_order_cnt)::numeric > (InitPlan 1).col1) (36 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; - QUERY PLAN +$Q$) as "QUERY PLAN"; + QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 -> Aggregate @@ -93,17 +97,19 @@ order by s_i_id; -> Seq Scan on stock_1640000 stock (24 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock); - QUERY PLAN +$Q$) as "QUERY PLAN"; + QUERY PLAN --------------------------------------------------------------------- HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) - InitPlan 1 (returns $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 @@ -124,24 +130,26 @@ having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from st -> Seq Scan on stock_1640000 stock (22 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true) order by s_i_id; +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Result -> HashAggregate Group Key: remote_scan.s_i_id -> Result - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Custom Scan (Citus Adaptive) - Filter: $0 + Filter: (InitPlan 1).col1 Task Count: 4 Tasks Shown: One of 4 -> Task @@ -151,21 +159,23 @@ order by s_i_id; -> Seq Scan on stock_1640000 s (17 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- HashAggregate Group Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Result -> Result - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Custom Scan (Citus Adaptive) - Filter: $0 + Filter: (InitPlan 1).col1 Task Count: 4 Tasks Shown: One of 4 -> Task diff --git a/src/test/regress/expected/ch_bench_having_mx.out b/src/test/regress/expected/ch_bench_having_mx.out index 90c4334a0..f4664fba5 100644 --- a/src/test/regress/expected/ch_bench_having_mx.out +++ b/src/test/regress/expected/ch_bench_having_mx.out @@ -16,6 +16,7 @@ SELECT create_distributed_table('stock','s_w_id'); \c - - - :worker_1_port SET search_path = ch_bench_having; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock @@ -23,15 +24,16 @@ where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock) group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 -> Aggregate @@ -57,27 +59,29 @@ order by s_i_id; Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: stock.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Seq Scan on stock_1640000 stock - Filter: ((s_order_cnt)::numeric > $0) + Filter: ((s_order_cnt)::numeric > (InitPlan 1).col1) (36 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; - QUERY PLAN +$Q$) as "QUERY PLAN"; + QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 -> Aggregate @@ -97,17 +101,19 @@ order by s_i_id; -> Seq Scan on stock_1640000 stock (24 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock); - QUERY PLAN +$Q$) as "QUERY PLAN"; + QUERY PLAN --------------------------------------------------------------------- HashAggregate Group Key: remote_scan.s_i_id - Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > $0) - InitPlan 1 (returns $0) + Filter: ((pg_catalog.sum(remote_scan.worker_column_3))::bigint > (InitPlan 1).col1) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 @@ -128,24 +134,26 @@ having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from st -> Seq Scan on stock_1640000 stock (22 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true) order by s_i_id; - QUERY PLAN +$Q$) as "QUERY PLAN"; + QUERY PLAN --------------------------------------------------------------------- Sort Sort Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Result -> HashAggregate Group Key: remote_scan.s_i_id -> Result - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Custom Scan (Citus Adaptive) - Filter: $0 + Filter: (InitPlan 1).col1 Task Count: 4 Tasks Shown: One of 4 -> Task @@ -155,20 +163,22 @@ order by s_i_id; -> Seq Scan on stock_1640000 s (17 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ explain select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- HashAggregate (cost=500.01..503.01 rows=200 width=12) Group Key: remote_scan.s_i_id - InitPlan 1 (returns $0) + InitPlan 1 -> Result (cost=0.00..0.01 rows=1 width=1) -> Result (cost=0.00..0.00 rows=100000 width=12) - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=100000 width=12) - Filter: $0 + Filter: (InitPlan 1).col1 Task Count: 4 Tasks Shown: One of 4 -> Task diff --git a/src/test/regress/expected/having_subquery.out b/src/test/regress/expected/having_subquery.out index a67d441b3..83e582791 100644 --- a/src/test/regress/expected/having_subquery.out +++ b/src/test/regress/expected/having_subquery.out @@ -22,6 +22,7 @@ HAVING ( (3 rows) -- lets pin the plan in the test as well +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*), o_orderstatus @@ -31,16 +32,17 @@ HAVING ( SELECT count(*) FROM customer ) > 0; +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- HashAggregate Group Key: remote_scan.o_orderstatus - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Result - One-Time Filter: ($0 > 0) + One-Time Filter: ((InitPlan 1).col1 > 0) -> Custom Scan (Citus Adaptive) - Filter: ($0 > 0) + Filter: ((InitPlan 1).col1 > 0) -> Distributed Subplan XXX_1 -> Custom Scan (Citus Adaptive) Task Count: 1 diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 17b673607..906add24c 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -1456,25 +1456,27 @@ Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator -> Function Scan on generate_series s -- WHERE EXISTS forces pg12 to materialize cte +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) WITH cte1 AS (SELECT s FROM generate_series(1,10) s) INSERT INTO lineitem_hash_part WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); +$Q$); Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator -> Result - One-Time Filter: $3 + One-Time Filter: (InitPlan 4).col1 CTE cte1 -> Function Scan on generate_series s CTE cte1 -> Limit - InitPlan 2 (returns $1) + InitPlan 2 -> CTE Scan on cte1 cte1_1 -> Result - One-Time Filter: $1 + One-Time Filter: (InitPlan 2).col1 -> CTE Scan on cte1 cte1_2 - InitPlan 4 (returns $3) + InitPlan 4 -> CTE Scan on cte1 cte1_3 -> CTE Scan on cte1 EXPLAIN (COSTS OFF) @@ -2425,9 +2427,11 @@ Aggregate (actual rows=1 loops=1) Sort Key: ref_table.a Sort Method: quicksort Memory: 25kB -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) -EXPLAIN :default_analyze_flags +SELECT public.explain_with_pg17_initplan_format($Q$ +EXPLAIN (ANALYZE on, COSTS off, TIMING off, SUMMARY off) SELECT count(distinct a) FROM dist_table WHERE EXISTS(SELECT random() < 2 FROM dist_table NATURAL JOIN ref_table); +$Q$); Aggregate (actual rows=1 loops=1) -> Custom Scan (Citus Adaptive) (actual rows=4 loops=1) -> Distributed Subplan XXX_1 @@ -2457,13 +2461,13 @@ Aggregate (actual rows=1 loops=1) Tuple data received from node: 8 bytes Node: host=localhost port=xxxxx dbname=regression -> Aggregate (actual rows=1 loops=1) - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) -> Sort (actual rows=4 loops=1) Sort Key: dist_table.a Sort Method: quicksort Memory: 25kB -> Result (actual rows=4 loops=1) - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) BEGIN; EXPLAIN :default_analyze_flags diff --git a/src/test/regress/expected/multi_explain_0.out b/src/test/regress/expected/multi_explain_0.out index 9534cefb8..5ba5e056f 100644 --- a/src/test/regress/expected/multi_explain_0.out +++ b/src/test/regress/expected/multi_explain_0.out @@ -1456,25 +1456,27 @@ Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator -> Function Scan on generate_series s -- WHERE EXISTS forces pg12 to materialize cte +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) WITH cte1 AS (SELECT s FROM generate_series(1,10) s) INSERT INTO lineitem_hash_part WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); +$Q$); Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator -> Result - One-Time Filter: $3 + One-Time Filter: (InitPlan 4).col1 CTE cte1 -> Function Scan on generate_series s CTE cte1 -> Limit - InitPlan 2 (returns $1) + InitPlan 2 -> CTE Scan on cte1 cte1_1 -> Result - One-Time Filter: $1 + One-Time Filter: (InitPlan 2).col1 -> CTE Scan on cte1 cte1_2 - InitPlan 4 (returns $3) + InitPlan 4 -> CTE Scan on cte1 cte1_3 -> CTE Scan on cte1 EXPLAIN (COSTS OFF) @@ -2420,9 +2422,11 @@ Aggregate (actual rows=1 loops=1) Sort Key: ref_table.a Sort Method: quicksort Memory: 25kB -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) -EXPLAIN :default_analyze_flags +SELECT public.explain_with_pg17_initplan_format($Q$ +EXPLAIN (ANALYZE on, COSTS off, TIMING off, SUMMARY off) SELECT count(distinct a) FROM dist_table WHERE EXISTS(SELECT random() < 2 FROM dist_table NATURAL JOIN ref_table); +$Q$); Aggregate (actual rows=1 loops=1) -> Custom Scan (Citus Adaptive) (actual rows=4 loops=1) -> Distributed Subplan XXX_1 @@ -2452,10 +2456,10 @@ Aggregate (actual rows=1 loops=1) Tuple data received from node: 8 bytes Node: host=localhost port=xxxxx dbname=regression -> Aggregate (actual rows=1 loops=1) - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) -> Result (actual rows=4 loops=1) - One-Time Filter: $0 + One-Time Filter: (InitPlan 1).col1 -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) BEGIN; EXPLAIN :default_analyze_flags diff --git a/src/test/regress/expected/multi_reference_table.out b/src/test/regress/expected/multi_reference_table.out index 75a9c3b64..68835be40 100644 --- a/src/test/regress/expected/multi_reference_table.out +++ b/src/test/regress/expected/multi_reference_table.out @@ -1574,9 +1574,11 @@ ALTER TABLE reference_table_test ADD COLUMN value_dummy INT; INSERT INTO reference_table_test VALUES (2, 2.0, '2', '2016-12-02'); ROLLBACK; -- Previous issue failed to rename reference tables in subqueries +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT value_1, count(*) FROM colocated_table_test GROUP BY value_1 HAVING (SELECT rt.value_2 FROM reference_table_test rt where rt.value_2 = 2) > 0 ORDER BY 1; +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Sort @@ -1596,10 +1598,10 @@ ORDER BY 1; Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: colocated_table_test.value_1 - InitPlan 1 (returns $0) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Result - One-Time Filter: ($0 > '0'::double precision) + One-Time Filter: ((InitPlan 1).col1 > '0'::double precision) -> Seq Scan on colocated_table_test_1250005 colocated_table_test (22 rows) diff --git a/src/test/regress/expected/multi_subquery.out b/src/test/regress/expected/multi_subquery.out index 60f978f5e..0a074e882 100644 --- a/src/test/regress/expected/multi_subquery.out +++ b/src/test/regress/expected/multi_subquery.out @@ -973,8 +973,10 @@ SELECT create_reference_table('keyvalref'); (1 row) +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyvalref GROUP BY key); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) @@ -993,15 +995,17 @@ SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: keyval1.key - Filter: (sum(keyval1.value) > $0) - InitPlan 1 (returns $0) + Filter: (sum(keyval1.value) > (InitPlan 1).col1) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Seq Scan on keyval1_xxxxxxx keyval1 (20 rows) -- For some reason 'ORDER BY 1 DESC LIMIT 1' triggers recursive planning +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyvalref GROUP BY key ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) @@ -1023,14 +1027,16 @@ SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: keyval1.key - Filter: (sum(keyval1.value) > $0) - InitPlan 1 (returns $0) + Filter: (sum(keyval1.value) > (InitPlan 1).col1) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Seq Scan on keyval1_xxxxxxx keyval1 (23 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyval2 GROUP BY key ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) @@ -1055,14 +1061,16 @@ SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) Node: host=localhost port=xxxxx dbname=regression -> HashAggregate Group Key: keyval1.key - Filter: (sum(keyval1.value) > $0) - InitPlan 1 (returns $0) + Filter: (sum(keyval1.value) > (InitPlan 1).col1) + InitPlan 1 -> Function Scan on read_intermediate_result intermediate_result -> Seq Scan on keyval1_xxxxxxx keyval1 (26 rows) +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) @@ -1071,8 +1079,8 @@ SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 HAVING sum(value) > (SELECT sum -> Task Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Filter: (sum(k1.value) > $0) - InitPlan 1 (returns $0) + Filter: (sum(k1.value) > (InitPlan 1).col1) + InitPlan 1 -> Limit -> Sort Sort Key: (sum(k2.value)) DESC diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index b8758e561..f66ff1e47 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -526,3 +526,62 @@ BEGIN RETURN result; END; $func$ LANGUAGE plpgsql; +CREATE or REPLACE FUNCTION initplan_references_to_pg17(text) returns text AS $$ +DECLARE + expr_parts text[]; + initplan_refs text[]; + n_initplan_refs int = 0; + i int := 1; + rv text := ''; + expr_part text; +BEGIN + -- Split the line on each $x; there must be at least one + -- For example 'foo = $0 and bar < $1' is split to: [ 'foo =', 'bar <' ] + expr_parts := regexp_split_to_array($1, '\$\d+'); + + -- Construct the PG17 formatted names in the given text + -- for example 'foo = $0 and bar < $1' yields [ '(InitPlan1).col1', '(InitPlan2).col1' ] + initplan_refs := ARRAY(select '(InitPlan ' || substr(x[1],2)::int + 1 || ').col1' from regexp_matches($1, '\$\d', 'g') x); + n_initplan_refs := array_length(initplan_refs, 1); + + -- Combine expression parts with PG17 formatted names + FOREACH expr_part IN ARRAY expr_parts + LOOP + rv := rv || expr_part; + -- There should be more expr parts than init plan refs so + -- check init plan refs boundary each time + IF i <= n_initplan_refs THEN + rv := rv || initplan_refs[i]; + END IF; + i := i + 1; + END LOOP; + RETURN rv; +END; +$$ LANGUAGE plpgsql; +-- This function formats EXPLAIN output to conform to how PG17 EXPLAIN shows +-- scalar subquery outputs if the pg version is less than 17 (*). When 17 +-- becomes the minimum supported pgversion this function can be retired. +-- +-- (*) https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb +CREATE OR REPLACE FUNCTION explain_with_pg17_initplan_format(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + pgversion int = 0; +BEGIN + pgversion = substring(version(), '\d+')::int ; + FOR query_plan IN execute explain_command LOOP + IF pgversion < 17 THEN + -- Two types of format changes are needed: + -- 1) 'Init Plan 1 (returns $0)' becomes just 'Init Plan 1' + -- 2) 'foo = $0' becomes 'foo = (InitPlan 1).col1' + IF query_plan ~ 'InitPlan \d \(returns' THEN + query_plan = regexp_replace(query_plan, '\(returns \$\d\)', '', 'g'); + ELSIF query_plan ~ '\$\d' THEN + -- This line contains at least one InitPlan reference + -- Replace it to have PG17 style InitPlan references + query_plan = public.initplan_references_to_pg17(query_plan); + END IF; + END IF; + RETURN NEXT; + END LOOP; +END; $$ language plpgsql; diff --git a/src/test/regress/sql/ch_bench_having.sql b/src/test/regress/sql/ch_bench_having.sql index b6996cbe3..53838b9c0 100644 --- a/src/test/regress/sql/ch_bench_having.sql +++ b/src/test/regress/sql/ch_bench_having.sql @@ -10,6 +10,7 @@ CREATE TABLE stock ( SELECT create_distributed_table('stock','s_w_id'); +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock @@ -17,33 +18,41 @@ where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock) group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock); +$Q$) as "QUERY PLAN"; - +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true); +$Q$) as "QUERY PLAN"; select s_i_id, sum(s_order_cnt) as ordercount from stock diff --git a/src/test/regress/sql/ch_bench_having_mx.sql b/src/test/regress/sql/ch_bench_having_mx.sql index 7374ba443..798591a5f 100644 --- a/src/test/regress/sql/ch_bench_having_mx.sql +++ b/src/test/regress/sql/ch_bench_having_mx.sql @@ -15,6 +15,7 @@ SELECT create_distributed_table('stock','s_w_id'); \c - - - :worker_1_port SET search_path = ch_bench_having; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock @@ -22,32 +23,40 @@ where s_order_cnt > (select sum(s_order_cnt) * .005 as where_query from stock) group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false, summary false, timing false) select s_i_id, sum(s_order_cnt) as ordercount from stock group by s_i_id having sum(s_order_cnt) > (select max(s_order_cnt) - 3 as having_query from stock); +$Q$) as "QUERY PLAN"; - +SELECT public.explain_with_pg17_initplan_format($Q$ explain (costs false) select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true) order by s_i_id; +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ explain select s_i_id, sum(s_order_cnt) as ordercount from stock s group by s_i_id having (select true); +$Q$) as "QUERY PLAN"; select s_i_id, sum(s_order_cnt) as ordercount from stock diff --git a/src/test/regress/sql/having_subquery.sql b/src/test/regress/sql/having_subquery.sql index ced678d39..f1e1c6911 100644 --- a/src/test/regress/sql/having_subquery.sql +++ b/src/test/regress/sql/having_subquery.sql @@ -16,6 +16,7 @@ HAVING ( ) > 0; -- lets pin the plan in the test as well +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*), o_orderstatus @@ -25,3 +26,4 @@ HAVING ( SELECT count(*) FROM customer ) > 0; +$Q$) as "QUERY PLAN"; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index 7fa75c8be..4fc16fbd8 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -630,11 +630,13 @@ INSERT INTO lineitem_hash_part (l_orderkey) SELECT s FROM generate_series(1,5) s; -- WHERE EXISTS forces pg12 to materialize cte +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) WITH cte1 AS (SELECT s FROM generate_series(1,10) s) INSERT INTO lineitem_hash_part WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); +$Q$); EXPLAIN (COSTS OFF) INSERT INTO lineitem_hash_part @@ -949,9 +951,11 @@ SELECT count(distinct a) from r NATURAL JOIN ref_table; EXPLAIN :default_analyze_flags SELECT count(distinct a) FROM (SELECT GREATEST(random(), 2) r, a FROM dist_table) t NATURAL JOIN ref_table; -EXPLAIN :default_analyze_flags +SELECT public.explain_with_pg17_initplan_format($Q$ +EXPLAIN (ANALYZE on, COSTS off, TIMING off, SUMMARY off) SELECT count(distinct a) FROM dist_table WHERE EXISTS(SELECT random() < 2 FROM dist_table NATURAL JOIN ref_table); +$Q$); BEGIN; EXPLAIN :default_analyze_flags diff --git a/src/test/regress/sql/multi_reference_table.sql b/src/test/regress/sql/multi_reference_table.sql index 42e3c283b..d538effe6 100644 --- a/src/test/regress/sql/multi_reference_table.sql +++ b/src/test/regress/sql/multi_reference_table.sql @@ -993,9 +993,11 @@ INSERT INTO reference_table_test VALUES (2, 2.0, '2', '2016-12-02'); ROLLBACK; -- Previous issue failed to rename reference tables in subqueries +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT value_1, count(*) FROM colocated_table_test GROUP BY value_1 HAVING (SELECT rt.value_2 FROM reference_table_test rt where rt.value_2 = 2) > 0 ORDER BY 1; +$Q$) as "QUERY PLAN"; WITH a as (SELECT rt.value_2 FROM reference_table_test rt where rt.value_2 = 2) SELECT ct.value_1, count(*) FROM colocated_table_test ct join a on ct.value_1 = a.value_2 diff --git a/src/test/regress/sql/multi_subquery.sql b/src/test/regress/sql/multi_subquery.sql index e5d8aa17c..8dd2d4791 100644 --- a/src/test/regress/sql/multi_subquery.sql +++ b/src/test/regress/sql/multi_subquery.sql @@ -665,18 +665,26 @@ SELECT create_distributed_table('keyval2', 'key'); CREATE TABLE keyvalref (key int, value int); SELECT create_reference_table('keyvalref'); +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyvalref GROUP BY key); +$Q$) as "QUERY PLAN"; -- For some reason 'ORDER BY 1 DESC LIMIT 1' triggers recursive planning +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyvalref GROUP BY key ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyval2 GROUP BY key ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; +SELECT public.explain_with_pg17_initplan_format($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 ORDER BY 1 DESC LIMIT 1); +$Q$) as "QUERY PLAN"; -- Simple join subquery pushdown SELECT diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index f7c97f1b2..e39ae451b 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -550,3 +550,64 @@ BEGIN RETURN result; END; $func$ LANGUAGE plpgsql; + +CREATE or REPLACE FUNCTION initplan_references_to_pg17(text) returns text AS $$ +DECLARE + expr_parts text[]; + initplan_refs text[]; + n_initplan_refs int = 0; + i int := 1; + rv text := ''; + expr_part text; +BEGIN + -- Split the line on each $x; there must be at least one + -- For example 'foo = $0 and bar < $1' is split to: [ 'foo =', 'bar <' ] + expr_parts := regexp_split_to_array($1, '\$\d+'); + + -- Construct the PG17 formatted names in the given text + -- for example 'foo = $0 and bar < $1' yields [ '(InitPlan1).col1', '(InitPlan2).col1' ] + initplan_refs := ARRAY(select '(InitPlan ' || substr(x[1],2)::int + 1 || ').col1' from regexp_matches($1, '\$\d', 'g') x); + n_initplan_refs := array_length(initplan_refs, 1); + + -- Combine expression parts with PG17 formatted names + FOREACH expr_part IN ARRAY expr_parts + LOOP + rv := rv || expr_part; + -- There should be more expr parts than init plan refs so + -- check init plan refs boundary each time + IF i <= n_initplan_refs THEN + rv := rv || initplan_refs[i]; + END IF; + i := i + 1; + END LOOP; + RETURN rv; +END; +$$ LANGUAGE plpgsql; + +-- This function formats EXPLAIN output to conform to how PG17 EXPLAIN shows +-- scalar subquery outputs if the pg version is less than 17 (*). When 17 +-- becomes the minimum supported pgversion this function can be retired. +-- +-- (*) https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb +CREATE OR REPLACE FUNCTION explain_with_pg17_initplan_format(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + pgversion int = 0; +BEGIN + pgversion = substring(version(), '\d+')::int ; + FOR query_plan IN execute explain_command LOOP + IF pgversion < 17 THEN + -- Two types of format changes are needed: + -- 1) 'Init Plan 1 (returns $0)' becomes just 'Init Plan 1' + -- 2) 'foo = $0' becomes 'foo = (InitPlan 1).col1' + IF query_plan ~ 'InitPlan \d \(returns' THEN + query_plan = regexp_replace(query_plan, '\(returns \$\d\)', '', 'g'); + ELSIF query_plan ~ '\$\d' THEN + -- This line contains at least one InitPlan reference + -- Replace it to have PG17 style InitPlan references + query_plan = public.initplan_references_to_pg17(query_plan); + END IF; + END IF; + RETURN NEXT; + END LOOP; +END; $$ language plpgsql; From d7f04aa187805181be9316d708c7bee81863ddf4 Mon Sep 17 00:00:00 2001 From: Colm Date: Thu, 21 Nov 2024 21:45:04 +0000 Subject: [PATCH 066/155] PG17 compatibility: Normalize COPY error messages (#7759) A recent Postgres commit (*) that refactored error messages is the cause of the diffs in pg16 regress test when running Citus on Postgres 17. The fix changes the pg16 goldfile and includes a normalization rule for the error messages so pg16 will pass when running with version 16 of Postgres. (*) https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=498ee9ee2f --- src/test/regress/bin/normalize.sed | 10 ++++++++++ src/test/regress/expected/pg16.out | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 62e9de697..1cbd0b404 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -309,3 +309,13 @@ s/permission denied to terminate process/must be a superuser to terminate superu s/permission denied to cancel query/must be a superuser to cancel superuser query/g #endif /* PG_VERSION_NUM < PG_VERSION_16 */ + +# pg17 changes +# can be removed when dropping PG15&16 support +#if PG_VERSION_NUM < PG_VERSION_17 +# (This is not preprocessor directive, but a reminder for the developer that will drop PG15&16 support ) + +s/COPY DEFAULT only available using COPY FROM/COPY DEFAULT cannot be used with COPY TO/ +s/COPY delimiter must not appear in the DEFAULT specification/COPY delimiter character must not appear in the DEFAULT specification/ + +#endif /* PG_VERSION_NUM < PG_VERSION_17 */ diff --git a/src/test/regress/expected/pg16.out b/src/test/regress/expected/pg16.out index 0af010348..7689d21bd 100644 --- a/src/test/regress/expected/pg16.out +++ b/src/test/regress/expected/pg16.out @@ -517,7 +517,7 @@ COPY copy_default FROM stdin WITH (default E'\r'); ERROR: COPY default representation cannot use newline or carriage return -- DELIMITER cannot appear in DEFAULT spec COPY copy_default FROM stdin WITH (delimiter ';', default 'test;test'); -ERROR: COPY delimiter must not appear in the DEFAULT specification +ERROR: COPY delimiter character must not appear in the DEFAULT specification -- CSV quote cannot appear in DEFAULT spec COPY copy_default FROM stdin WITH (format csv, quote '"', default 'test"test'); ERROR: CSV quote character must not appear in the DEFAULT specification @@ -603,7 +603,7 @@ SET citus.shard_count TO 1; SET citus.shard_replication_factor TO 1; -- DEFAULT cannot be used in COPY TO COPY (select 1 as test) TO stdout WITH (default '\D'); -ERROR: COPY DEFAULT only available using COPY FROM +ERROR: COPY DEFAULT cannot be used with COPY TO -- Tests for SQL/JSON: JSON_ARRAYAGG and JSON_OBJECTAGG aggregates -- Relevant PG commit: -- https://github.com/postgres/postgres/commit/7081ac4 From 65dcc5904d88b5b231800a8155c3a015c5a5627e Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 22 Nov 2024 01:08:15 +0300 Subject: [PATCH 067/155] PG17 compatibility: fix backend type orders in test (#7760) This work was already done by @m3hm3t and approved as part of https://github.com/citusdata/citus/pull/7722 I separated it in this PR since the previous one contained other changes which we don't currently want to merge. Relevant PG commit: --------- Co-authored-by: Mehmet YILMAZ --- .../expected/multi_mx_hide_shard_names.out | 26 +++++++++++------- .../regress/sql/multi_mx_hide_shard_names.sql | 27 +++++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/test/regress/expected/multi_mx_hide_shard_names.out b/src/test/regress/expected/multi_mx_hide_shard_names.out index 762c6a30b..1f6e06ab1 100644 --- a/src/test/regress/expected/multi_mx_hide_shard_names.out +++ b/src/test/regress/expected/multi_mx_hide_shard_names.out @@ -473,19 +473,25 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- PG16 added one more backend type B_STANDALONE_BACKEND -- and also alphabetized the backend types, hence the orders changed --- Relevant PG commit: +-- Relevant PG16 commit: -- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690 +-- Relevant Pg17 commit: +-- https://github.com/postgres/postgres/commit/067701f57758f9baed5bd9d868539738d77bfa92#diff-afc0ebd67534b71b5b94b29a1387aa6eedffe342a5539f52d686428be323e802 SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 -\gset -\if :server_version_ge_16 -SELECT 4 AS client_backend \gset -SELECT 5 AS bgworker \gset -SELECT 12 AS walsender \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset +\if :server_version_ge_17 + SELECT 1 AS client_backend \gset + SELECT 4 AS bgworker \gset + SELECT 5 AS walsender \gset +\elif :server_version_ge_16 + SELECT 4 AS client_backend \gset + SELECT 5 AS bgworker \gset + SELECT 12 AS walsender \gset \else -SELECT 3 AS client_backend \gset -SELECT 4 AS bgworker \gset -SELECT 9 AS walsender \gset + SELECT 3 AS client_backend \gset + SELECT 4 AS bgworker \gset + SELECT 9 AS walsender \gset \endif -- say, we set it to bgworker -- the shards and indexes do not show up diff --git a/src/test/regress/sql/multi_mx_hide_shard_names.sql b/src/test/regress/sql/multi_mx_hide_shard_names.sql index addc7f90e..70f87a875 100644 --- a/src/test/regress/sql/multi_mx_hide_shard_names.sql +++ b/src/test/regress/sql/multi_mx_hide_shard_names.sql @@ -246,20 +246,25 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- PG16 added one more backend type B_STANDALONE_BACKEND -- and also alphabetized the backend types, hence the orders changed --- Relevant PG commit: +-- Relevant PG16 commit: -- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690 +-- Relevant Pg17 commit: +-- https://github.com/postgres/postgres/commit/067701f57758f9baed5bd9d868539738d77bfa92#diff-afc0ebd67534b71b5b94b29a1387aa6eedffe342a5539f52d686428be323e802 SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 -\gset - -\if :server_version_ge_16 -SELECT 4 AS client_backend \gset -SELECT 5 AS bgworker \gset -SELECT 12 AS walsender \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 \gset +\if :server_version_ge_17 + SELECT 1 AS client_backend \gset + SELECT 4 AS bgworker \gset + SELECT 5 AS walsender \gset +\elif :server_version_ge_16 + SELECT 4 AS client_backend \gset + SELECT 5 AS bgworker \gset + SELECT 12 AS walsender \gset \else -SELECT 3 AS client_backend \gset -SELECT 4 AS bgworker \gset -SELECT 9 AS walsender \gset + SELECT 3 AS client_backend \gset + SELECT 4 AS bgworker \gset + SELECT 9 AS walsender \gset \endif -- say, we set it to bgworker From 12dd9c1f6b1b8e9990e1cdd4c3ed4b16e280c0fa Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 22 Nov 2024 16:10:02 +0300 Subject: [PATCH 068/155] PG17 compatibility: Adjust print_extension_changes function for extra type outputs in PG17 (#7761) In PG17, Auto-generated array types, multirange types, and relation rowtypes are treated as dependent objects, hence changing the output of the print_extension_changes function. Relevant PG commit: e5bc9454e527b1cba97553531d8d4992892fdeef https://github.com/postgres/postgres/commit/e5bc9454e527b1cba97553531d8d4992892fdeef Here we create a table with only the basic extension types in order to avoid printing extra ones for now. This can be removed when we drop PG16 support. https://github.com/citusdata/citus/actions/runs/11960253650/attempts/1#summary-33343972656 ```diff | table pg_dist_rebalance_strategy + | type citus.distribution_type[] + | type citus.pg_dist_object + | type pg_dist_shard + | type pg_dist_shard[] + | type pg_dist_shard_placement + | type pg_dist_shard_placement[] + | type pg_dist_transaction + | type pg_dist_transaction[] | view citus_dist_stat_activity | view pg_dist_shard_placement ``` --- src/test/regress/expected/multi_extension.out | 30 +++++++++++++++-- .../expected/upgrade_list_citus_objects.out | 22 +++++++++++++ src/test/regress/sql/multi_extension.sql | 32 +++++++++++++++++-- .../sql/upgrade_list_citus_objects.sql | 24 ++++++++++++++ 4 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index b2badd878..627b0b72f 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -38,6 +38,24 @@ $definition$ create_function_test_maintenance_worker CREATE TABLE multi_extension.prev_objects(description text); CREATE TABLE multi_extension.extension_diff(previous_object text COLLATE "C", current_object text COLLATE "C"); +-- In PG17, Auto-generated array types, multirange types, and relation rowtypes +-- are treated as dependent objects, hence changing the output of the +-- print_extension_changes function. +-- Relevant PG commit: e5bc9454e527b1cba97553531d8d4992892fdeef +-- Here we create a table with only the basic extension types +-- in order to avoid printing extra ones for now +-- This can be removed when we drop PG16 support. +CREATE TABLE multi_extension.extension_basic_types (description text); +INSERT INTO multi_extension.extension_basic_types VALUES ('type citus.distribution_type'), + ('type citus.shard_transfer_mode'), + ('type citus_copy_format'), + ('type noderole'), + ('type citus_job_status'), + ('type citus_task_status'), + ('type replication_slot_info'), + ('type split_copy_info'), + ('type split_shard_info'), + ('type cluster_clock'); CREATE FUNCTION multi_extension.print_extension_changes() RETURNS TABLE(previous_object text, current_object text) AS $func$ @@ -53,7 +71,10 @@ BEGIN WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = e.oid AND deptype = 'e' - AND e.extname='citus'; + AND e.extname='citus' + AND (pg_catalog.pg_describe_object(classid, objid, 0) NOT LIKE 'type%' + OR + pg_catalog.pg_describe_object(classid, objid, 0) IN (SELECT * FROM extension_basic_types)); INSERT INTO extension_diff SELECT p.description previous_object, c.description current_object @@ -88,7 +109,8 @@ FROM pg_depend AS pgd, WHERE pgd.refclassid = 'pg_extension'::regclass AND pgd.refobjid = pge.oid AND pge.extname = 'citus' AND - pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') AND + pgio.type != 'type' ORDER BY 1, 2; type | identity --------------------------------------------------------------------- @@ -1424,7 +1446,8 @@ FROM pg_depend AS pgd, WHERE pgd.refclassid = 'pg_extension'::regclass AND pgd.refobjid = pge.oid AND pge.extname = 'citus' AND - pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') AND + pgio.type != 'type' ORDER BY 1, 2; type | identity --------------------------------------------------------------------- @@ -1914,6 +1937,7 @@ RESET citus.enable_schema_based_sharding; DROP EXTENSION citus; CREATE EXTENSION citus; DROP TABLE version_mismatch_table; +DROP TABLE multi_extension.extension_basic_types; DROP SCHEMA multi_extension; ERROR: cannot drop schema multi_extension because other objects depend on it DETAIL: function multi_extension.print_extension_changes() depends on schema multi_extension diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 36bd504e8..e848d5da7 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -1,3 +1,21 @@ +-- In PG17, Auto-generated array types, multirange types, and relation rowtypes +-- are treated as dependent objects, hence changing the output of the +-- print_extension_changes function. +-- Relevant PG commit: e5bc9454e527b1cba97553531d8d4992892fdeef +-- Here we create a table with only the basic extension types +-- in order to avoid printing extra ones for now +-- This can be removed when we drop PG16 support. +CREATE TABLE extension_basic_types (description text); +INSERT INTO extension_basic_types VALUES ('type citus.distribution_type'), + ('type citus.shard_transfer_mode'), + ('type citus_copy_format'), + ('type noderole'), + ('type citus_job_status'), + ('type citus_task_status'), + ('type replication_slot_info'), + ('type split_copy_info'), + ('type split_shard_info'), + ('type cluster_clock'); -- list all postgres objects belonging to the citus extension SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS description FROM pg_catalog.pg_depend, pg_catalog.pg_extension e @@ -5,6 +23,9 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = e.oid AND deptype = 'e' AND e.extname='citus' + AND (pg_catalog.pg_describe_object(classid, objid, 0) NOT LIKE 'type%' + OR + pg_catalog.pg_describe_object(classid, objid, 0) IN (SELECT * FROM extension_basic_types)) AND pg_catalog.pg_describe_object(classid, objid, 0) != 'function any_value(anyelement)' AND pg_catalog.pg_describe_object(classid, objid, 0) != 'function any_value_agg(anyelement,anyelement)' ORDER BY 1; @@ -345,3 +366,4 @@ ORDER BY 1; view time_partitions (333 rows) +DROP TABLE extension_basic_types; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index 1726c260f..4e917ef36 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -42,6 +42,26 @@ CREATE TABLE multi_extension.prev_objects(description text); CREATE TABLE multi_extension.extension_diff(previous_object text COLLATE "C", current_object text COLLATE "C"); +-- In PG17, Auto-generated array types, multirange types, and relation rowtypes +-- are treated as dependent objects, hence changing the output of the +-- print_extension_changes function. +-- Relevant PG commit: e5bc9454e527b1cba97553531d8d4992892fdeef +-- Here we create a table with only the basic extension types +-- in order to avoid printing extra ones for now +-- This can be removed when we drop PG16 support. + +CREATE TABLE multi_extension.extension_basic_types (description text); +INSERT INTO multi_extension.extension_basic_types VALUES ('type citus.distribution_type'), + ('type citus.shard_transfer_mode'), + ('type citus_copy_format'), + ('type noderole'), + ('type citus_job_status'), + ('type citus_task_status'), + ('type replication_slot_info'), + ('type split_copy_info'), + ('type split_shard_info'), + ('type cluster_clock'); + CREATE FUNCTION multi_extension.print_extension_changes() RETURNS TABLE(previous_object text, current_object text) AS $func$ @@ -57,7 +77,10 @@ BEGIN WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = e.oid AND deptype = 'e' - AND e.extname='citus'; + AND e.extname='citus' + AND (pg_catalog.pg_describe_object(classid, objid, 0) NOT LIKE 'type%' + OR + pg_catalog.pg_describe_object(classid, objid, 0) IN (SELECT * FROM extension_basic_types)); INSERT INTO extension_diff SELECT p.description previous_object, c.description current_object @@ -90,7 +113,8 @@ FROM pg_depend AS pgd, WHERE pgd.refclassid = 'pg_extension'::regclass AND pgd.refobjid = pge.oid AND pge.extname = 'citus' AND - pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') AND + pgio.type != 'type' ORDER BY 1, 2; @@ -647,7 +671,8 @@ FROM pg_depend AS pgd, WHERE pgd.refclassid = 'pg_extension'::regclass AND pgd.refobjid = pge.oid AND pge.extname = 'citus' AND - pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') AND + pgio.type != 'type' ORDER BY 1, 2; -- see incompatible version errors out @@ -1015,4 +1040,5 @@ DROP EXTENSION citus; CREATE EXTENSION citus; DROP TABLE version_mismatch_table; +DROP TABLE multi_extension.extension_basic_types; DROP SCHEMA multi_extension; diff --git a/src/test/regress/sql/upgrade_list_citus_objects.sql b/src/test/regress/sql/upgrade_list_citus_objects.sql index 47fafea05..fb761e852 100644 --- a/src/test/regress/sql/upgrade_list_citus_objects.sql +++ b/src/test/regress/sql/upgrade_list_citus_objects.sql @@ -1,3 +1,22 @@ +-- In PG17, Auto-generated array types, multirange types, and relation rowtypes +-- are treated as dependent objects, hence changing the output of the +-- print_extension_changes function. +-- Relevant PG commit: e5bc9454e527b1cba97553531d8d4992892fdeef +-- Here we create a table with only the basic extension types +-- in order to avoid printing extra ones for now +-- This can be removed when we drop PG16 support. +CREATE TABLE extension_basic_types (description text); +INSERT INTO extension_basic_types VALUES ('type citus.distribution_type'), + ('type citus.shard_transfer_mode'), + ('type citus_copy_format'), + ('type noderole'), + ('type citus_job_status'), + ('type citus_task_status'), + ('type replication_slot_info'), + ('type split_copy_info'), + ('type split_shard_info'), + ('type cluster_clock'); + -- list all postgres objects belonging to the citus extension SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS description FROM pg_catalog.pg_depend, pg_catalog.pg_extension e @@ -5,6 +24,11 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = e.oid AND deptype = 'e' AND e.extname='citus' + AND (pg_catalog.pg_describe_object(classid, objid, 0) NOT LIKE 'type%' + OR + pg_catalog.pg_describe_object(classid, objid, 0) IN (SELECT * FROM extension_basic_types)) AND pg_catalog.pg_describe_object(classid, objid, 0) != 'function any_value(anyelement)' AND pg_catalog.pg_describe_object(classid, objid, 0) != 'function any_value_agg(anyelement,anyelement)' ORDER BY 1; + +DROP TABLE extension_basic_types; From 1d0111aed61ca08d60f7fa91c99f9abe976ae520 Mon Sep 17 00:00:00 2001 From: Colm Date: Fri, 22 Nov 2024 16:25:22 +0000 Subject: [PATCH 069/155] PG17 regress test sanity: fix diffs in union_pushdown. (#7762) Preserve the test error message by adjusting the query so that PG17 cannot pull it up to a join. Another instance of a subquery that can be pulled up to a join with PG17 (#7745) This should have been fixed in, but slipped by, #7745 --- src/test/regress/expected/union_pushdown.out | 2 +- src/test/regress/sql/union_pushdown.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/union_pushdown.out b/src/test/regress/expected/union_pushdown.out index 040535b75..4ae83c972 100644 --- a/src/test/regress/expected/union_pushdown.out +++ b/src/test/regress/expected/union_pushdown.out @@ -1409,7 +1409,7 @@ $$); -- we hit https://github.com/citusdata/citus/blob/f00c63c33daf3d16f06462626ca14732b141ae7a/src/backend/distributed/planner/relation_restriction_equivalence.c#L235-L242 SELECT public.explain_has_distributed_subplan($$ -EXPLAIN SELECT * FROM users_table_part u1 WHERE (value_1, user_id) IN +EXPLAIN SELECT * FROM users_table_part u1 WHERE (value_1 + random(), user_id) IN ( SELECT u1.user_id, user_id FROM users_table_part UNION diff --git a/src/test/regress/sql/union_pushdown.sql b/src/test/regress/sql/union_pushdown.sql index 57099f060..1bb63eb62 100644 --- a/src/test/regress/sql/union_pushdown.sql +++ b/src/test/regress/sql/union_pushdown.sql @@ -1068,7 +1068,7 @@ $$); -- we hit https://github.com/citusdata/citus/blob/f00c63c33daf3d16f06462626ca14732b141ae7a/src/backend/distributed/planner/relation_restriction_equivalence.c#L235-L242 SELECT public.explain_has_distributed_subplan($$ -EXPLAIN SELECT * FROM users_table_part u1 WHERE (value_1, user_id) IN +EXPLAIN SELECT * FROM users_table_part u1 WHERE (value_1 + random(), user_id) IN ( SELECT u1.user_id, user_id FROM users_table_part UNION From d5f067a03f729f85f20614378a7edc215349c302 Mon Sep 17 00:00:00 2001 From: Colm Date: Mon, 25 Nov 2024 21:11:34 +0000 Subject: [PATCH 070/155] PG17 regress sanity: fix error unrecognized alter database option tablespace seen in database vanilla test (#7764) Disable DDL propagation for the vanilla test suite. This enables the vanilla `database ` test to pass, where previously it was correctly returning `ERROR: unrecognized ALTER DATABASE option: tablespace` because release-13.0 does not propagate this ALTER DATABASE variant. We (Citus team) discussed cherry picking [#7253](https://github.com/citusdata/citus/pull/7253) from main to release-13.0 because it does propagate ALTER DATABASE tablespace option (as well as a couple of others) but decided fixing the regress test was not the proper context for that. The fix disables `citus.enable_metadata_sync` when running vanilla, we discussed disabling `citus.enable_create_database_propagation` but this is not in release-13.0. --- src/test/regress/pg_regress_multi.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index adcb431e1..b9ed5d03e 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -511,6 +511,9 @@ if($vanillatest) # we disable some restrictions for local objects like local views to not break postgres vanilla test behaviour. push(@pgOptions, "citus.enforce_object_restrictions_for_local_objects=false"); + + # we disable metadata synchronization to enable postgres vanilla DDL tests to pass. + push(@pgOptions, "citus.enable_metadata_sync=false"); } if ($useMitmproxy) From e9110de7e1d2c0c32e454b8b8963249a9d328de4 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Mon, 2 Dec 2024 13:08:21 +0300 Subject: [PATCH 071/155] PG17 compatibility: Fix Test Failure in multi_name_lengths multi_create_table_constraints (#7726) PG 17 Removes outer parentheses from CHECK constraints we add them back for pg15,pg16 compatibility e.g. change CHECK other_col >= 100 to CHECK (other_col >= 100) Relevant PG commit: e59fcbd712c777eb2987d7c9ad542a7e817954ec https://github.com/postgres/postgres/commit/e59fcbd712c777eb2987d7c9ad542a7e817954ec CI link https://github.com/citusdata/citus/actions/runs/11844794788 ```difft SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.check_example_365068'::regclass; Constraint | Definition -------------------------------------+----------------------------------- - check_example_other_col_check | CHECK (other_col >= 100) - check_example_other_other_col_check | CHECK (abs(other_other_col) >= 100) + check_example_other_col_check | CHECK other_col >= 100 + check_example_other_other_col_check | CHECK abs(other_other_col) >= 100 ``` Co-authored-by: Mehmet YILMAZ --- src/test/regress/bin/normalize.sed | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 1cbd0b404..19ac512c5 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -319,3 +319,8 @@ s/COPY DEFAULT only available using COPY FROM/COPY DEFAULT cannot be used with C s/COPY delimiter must not appear in the DEFAULT specification/COPY delimiter character must not appear in the DEFAULT specification/ #endif /* PG_VERSION_NUM < PG_VERSION_17 */ + +# PG 17 Removes outer parentheses from CHECK constraints +# we add them back for pg15,pg16 compatibility +# e.g. change CHECK other_col >= 100 to CHECK (other_col >= 100) +s/\| CHECK ([a-zA-Z])(.*)/| CHECK \(\1\2\)/g From 089555c19742ae83daacb4a6d613e8b30f5903f8 Mon Sep 17 00:00:00 2001 From: Colm Date: Mon, 2 Dec 2024 13:47:19 +0000 Subject: [PATCH 072/155] PG17 compatibility: fix diff in tableam (#7771) Test `tableam` expects that this CREATE TABLE statement: `CREATE TABLE test_partitioned(id int, p int, val int) PARTITION BY RANGE (p) USING fake_am;` will produce this error: `specifying a table access method is not supported on a partitioned table` but as of [this PG commit](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a229) it is possible to specify an access method on a partitioned table. This fix moves the CREATE TABLE statement to pg17, and adds an additional test to show parent access method is inherited. --- src/test/regress/expected/pg17.out | 71 +++++++++++++++++++++++++++ src/test/regress/expected/tableam.out | 6 --- src/test/regress/sql/pg17.sql | 48 ++++++++++++++++++ src/test/regress/sql/tableam.sql | 6 --- 4 files changed, 119 insertions(+), 12 deletions(-) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 8943e78e7..563808787 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -352,3 +352,74 @@ drop cascades to table pg17_corr_subq_folding.events \endif -- PG17-specific tests go here. -- +CREATE SCHEMA pg17; +SET search_path TO pg17; +-- Test specifying access method on partitioned tables. PG17 feature, added by: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a229 +-- The following tests were failing tests in tableam but will pass on PG >= 17. +-- There is some set-up duplication of tableam, and this test can be returned +-- to tableam when 17 is the minimum supported PG version. +SELECT public.run_command_on_coordinator_and_workers($Q$ + SET citus.enable_ddl_propagation TO off; + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; +$Q$); + run_command_on_coordinator_and_workers +--------------------------------------------------------------------- + +(1 row) + +-- Since Citus assumes access methods are part of the extension, make fake_am +-- owned manually to be able to pass checks on Citus while distributing tables. +ALTER EXTENSION citus ADD ACCESS METHOD fake_am; +CREATE TABLE test_partitioned(id int, p int, val int) +PARTITION BY RANGE (p) USING fake_am; +-- Test that children inherit access method from parent +CREATE TABLE test_partitioned_p1 PARTITION OF test_partitioned + FOR VALUES FROM (1) TO (10); +CREATE TABLE test_partitioned_p2 PARTITION OF test_partitioned + FOR VALUES FROM (11) TO (20); +INSERT INTO test_partitioned VALUES (1, 5, -1), (2, 15, -2); +WARNING: fake_tuple_insert +WARNING: fake_tuple_insert +INSERT INTO test_partitioned VALUES (3, 6, -6), (4, 16, -4); +WARNING: fake_tuple_insert +WARNING: fake_tuple_insert +SELECT count(1) FROM test_partitioned_p1; +WARNING: fake_scan_getnextslot +WARNING: fake_scan_getnextslot +WARNING: fake_scan_getnextslot + count +--------------------------------------------------------------------- + 2 +(1 row) + +SELECT count(1) FROM test_partitioned_p2; +WARNING: fake_scan_getnextslot +WARNING: fake_scan_getnextslot +WARNING: fake_scan_getnextslot + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Both child table partitions inherit fake_am +SELECT c.relname, am.amname FROM pg_class c, pg_am am +WHERE c.relam = am.oid AND c.oid IN ('test_partitioned_p1'::regclass, 'test_partitioned_p2'::regclass) +ORDER BY c.relname; + relname | amname +--------------------------------------------------------------------- + test_partitioned_p1 | fake_am + test_partitioned_p2 | fake_am +(2 rows) + +DROP TABLE test_partitioned; +ALTER EXTENSION citus DROP ACCESS METHOD fake_am; +-- End of testing specifying access method on partitioned tables. +DROP SCHEMA pg17 CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to function fake_am_handler(internal) +drop cascades to access method fake_am diff --git a/src/test/regress/expected/tableam.out b/src/test/regress/expected/tableam.out index 8e6fe5205..a24fed14f 100644 --- a/src/test/regress/expected/tableam.out +++ b/src/test/regress/expected/tableam.out @@ -281,12 +281,6 @@ DETAIL: from localhost:xxxxx (1 row) DROP TABLE test_partitioned; --- Specifying access method in parent is not supported. --- If the below statement ever succeeds, add more tests for --- the case where children inherit access method from parent. -CREATE TABLE test_partitioned(id int, p int, val int) -PARTITION BY RANGE (p) USING fake_am; -ERROR: specifying a table access method is not supported on a partitioned table \set VERBOSITY terse ALTER EXTENSION citus DROP ACCESS METHOD fake_am; NOTICE: Citus does not propagate adding/dropping member objects diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 4fdde71ca..e297a0c58 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -180,3 +180,51 @@ DROP SCHEMA pg17_corr_subq_folding CASCADE; -- PG17-specific tests go here. -- +CREATE SCHEMA pg17; +SET search_path TO pg17; + +-- Test specifying access method on partitioned tables. PG17 feature, added by: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a229 +-- The following tests were failing tests in tableam but will pass on PG >= 17. +-- There is some set-up duplication of tableam, and this test can be returned +-- to tableam when 17 is the minimum supported PG version. + +SELECT public.run_command_on_coordinator_and_workers($Q$ + SET citus.enable_ddl_propagation TO off; + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; +$Q$); + +-- Since Citus assumes access methods are part of the extension, make fake_am +-- owned manually to be able to pass checks on Citus while distributing tables. +ALTER EXTENSION citus ADD ACCESS METHOD fake_am; + +CREATE TABLE test_partitioned(id int, p int, val int) +PARTITION BY RANGE (p) USING fake_am; + +-- Test that children inherit access method from parent +CREATE TABLE test_partitioned_p1 PARTITION OF test_partitioned + FOR VALUES FROM (1) TO (10); +CREATE TABLE test_partitioned_p2 PARTITION OF test_partitioned + FOR VALUES FROM (11) TO (20); + +INSERT INTO test_partitioned VALUES (1, 5, -1), (2, 15, -2); +INSERT INTO test_partitioned VALUES (3, 6, -6), (4, 16, -4); + +SELECT count(1) FROM test_partitioned_p1; +SELECT count(1) FROM test_partitioned_p2; + +-- Both child table partitions inherit fake_am +SELECT c.relname, am.amname FROM pg_class c, pg_am am +WHERE c.relam = am.oid AND c.oid IN ('test_partitioned_p1'::regclass, 'test_partitioned_p2'::regclass) +ORDER BY c.relname; + +DROP TABLE test_partitioned; +ALTER EXTENSION citus DROP ACCESS METHOD fake_am; + +-- End of testing specifying access method on partitioned tables. + +DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/tableam.sql b/src/test/regress/sql/tableam.sql index dd1976f70..d92b1d141 100644 --- a/src/test/regress/sql/tableam.sql +++ b/src/test/regress/sql/tableam.sql @@ -138,12 +138,6 @@ SELECT count(*) FROM test_partitioned; DROP TABLE test_partitioned; --- Specifying access method in parent is not supported. --- If the below statement ever succeeds, add more tests for --- the case where children inherit access method from parent. -CREATE TABLE test_partitioned(id int, p int, val int) -PARTITION BY RANGE (p) USING fake_am; - \set VERBOSITY terse ALTER EXTENSION citus DROP ACCESS METHOD fake_am; drop schema test_tableam cascade; From 26c09f340fa3e02cc02d387a5e1e3639de6c2468 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:03:38 +0300 Subject: [PATCH 073/155] PG17 compatibility: fix some tests outputs (#7765) There are two commits in this PR: 1) Remove domain_default column since it has been removed from PG17 Relevant PG commit: https://github.com/postgres/postgres/commit/78806a95095c4fb9230a441925244690d9c07d23 78806a95095c4fb9230a441925244690d9c07d23 2) pg_stat_statements reset output diff fix pg_stat_statements reset output changed in PG17, fix idea from Relevant PG commits: https://github.com/postgres/postgres/commit/6ab1dbd26bbf307055d805feaaca16dc3e750d36 6ab1dbd26bbf307055d805feaaca16dc3e750d36 --- src/test/regress/expected/issue_5248.out | 1 - src/test/regress/expected/issue_5248_0.out | 1 - src/test/regress/expected/stat_statements.out | 12 ++++++------ src/test/regress/sql/issue_5248.sql | 1 - src/test/regress/sql/stat_statements.sql | 4 ++-- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/issue_5248.out b/src/test/regress/expected/issue_5248.out index 4639f24ed..db1ae26c7 100644 --- a/src/test/regress/expected/issue_5248.out +++ b/src/test/regress/expected/issue_5248.out @@ -201,7 +201,6 @@ FROM ( sample_6.info AS c1, subq_0.c2 AS c2, subq_0.c3 AS c3, - ref_3.domain_default AS c4, sample_6.user_id AS c5, ref_3.collation_name AS c6 FROM orders AS sample_6 TABLESAMPLE system (3.8) diff --git a/src/test/regress/expected/issue_5248_0.out b/src/test/regress/expected/issue_5248_0.out index cb3c4562f..d7fe8020c 100644 --- a/src/test/regress/expected/issue_5248_0.out +++ b/src/test/regress/expected/issue_5248_0.out @@ -201,7 +201,6 @@ FROM ( sample_6.info AS c1, subq_0.c2 AS c2, subq_0.c3 AS c3, - ref_3.domain_default AS c4, sample_6.user_id AS c5, ref_3.collation_name AS c6 FROM orders AS sample_6 TABLESAMPLE system (3.8) diff --git a/src/test/regress/expected/stat_statements.out b/src/test/regress/expected/stat_statements.out index a3e2f673f..7ad87ff5f 100644 --- a/src/test/regress/expected/stat_statements.out +++ b/src/test/regress/expected/stat_statements.out @@ -179,10 +179,10 @@ ORDER BY 1, 2, 3, 4; SET citus.stat_statements_track TO 'all'; -- reset pg_stat_statements and verify it also cleans citus_stat_statements output -- verify that entries are actually removed from citus_stat_statements -SELECT pg_stat_statements_reset(); - pg_stat_statements_reset +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t --------------------------------------------------------------------- - + t (1 row) SELECT * FROM citus_stat_statements; @@ -251,10 +251,10 @@ ORDER BY 1, 2, 3, 4; SELECT l_orderkey FROM lineitem_hash_part WHERE l_orderkey > ? | adaptive | | 1 (6 rows) -SELECT pg_stat_statements_reset(); - pg_stat_statements_reset +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t --------------------------------------------------------------------- - + t (1 row) SELECT count(*) FROM lineitem_hash_part; diff --git a/src/test/regress/sql/issue_5248.sql b/src/test/regress/sql/issue_5248.sql index 321e9df14..2248f1493 100644 --- a/src/test/regress/sql/issue_5248.sql +++ b/src/test/regress/sql/issue_5248.sql @@ -184,7 +184,6 @@ FROM ( sample_6.info AS c1, subq_0.c2 AS c2, subq_0.c3 AS c3, - ref_3.domain_default AS c4, sample_6.user_id AS c5, ref_3.collation_name AS c6 FROM orders AS sample_6 TABLESAMPLE system (3.8) diff --git a/src/test/regress/sql/stat_statements.sql b/src/test/regress/sql/stat_statements.sql index 5afed9215..148b2fd7c 100644 --- a/src/test/regress/sql/stat_statements.sql +++ b/src/test/regress/sql/stat_statements.sql @@ -91,7 +91,7 @@ SET citus.stat_statements_track TO 'all'; -- reset pg_stat_statements and verify it also cleans citus_stat_statements output -- verify that entries are actually removed from citus_stat_statements -SELECT pg_stat_statements_reset(); +SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM citus_stat_statements; -- run some queries @@ -109,7 +109,7 @@ SELECT normalize_query_string(query), executor, partition_key, calls FROM citus_stat_statements ORDER BY 1, 2, 3, 4; -SELECT pg_stat_statements_reset(); +SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT count(*) FROM lineitem_hash_part; SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = 4; From 5f479d5f472d1271fe1ea6482042ecac9a1df3c8 Mon Sep 17 00:00:00 2001 From: Colm Date: Tue, 3 Dec 2024 08:44:56 +0000 Subject: [PATCH 074/155] PG17 compatibility: revert #7764 (#7775) Revert PG17 compatibility fix #7764 --- src/test/regress/pg_regress_multi.pl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index b9ed5d03e..adcb431e1 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -511,9 +511,6 @@ if($vanillatest) # we disable some restrictions for local objects like local views to not break postgres vanilla test behaviour. push(@pgOptions, "citus.enforce_object_restrictions_for_local_objects=false"); - - # we disable metadata synchronization to enable postgres vanilla DDL tests to pass. - push(@pgOptions, "citus.enable_metadata_sync=false"); } if ($useMitmproxy) From 698699d89efafe321ebf502845923bc347ecebfc Mon Sep 17 00:00:00 2001 From: Colm Date: Tue, 3 Dec 2024 09:14:47 +0000 Subject: [PATCH 075/155] PG17 compatibility (#7653): Fix test diffs in columnar schedule (#7768) This PR fixes diffs in `columnnar_chunk_filtering` and `columnar_paths` tests. In `columnnar_chunk_filtering` an expression `(NOT (SubPlan 1))` changed to `(NOT (ANY (a = (SubPlan 1).col1)))`. This is due to [aPG17 commit](https://github.com/postgres/postgres/commit/fd0398fc) that improved how scalar subqueries (InitPlans) and ANY subqueries (SubPlans) are EXPLAINed in expressions. The fix uses a helper function which converts the PG17 format to the pre-PG17 format. It is done this way because pre-PG17 EXPLAIN does not provide enough context to convert to the PG17 format. The helper function can (and should) be retired when 17 becomes the minimum supported PG. In `columnar_paths`, a merge join changed to a hash join. This is due to [this PG17 commit](https://github.com/postgres/postgres/commit/f7816aec23eed1dc1da5f9a53cb6507d30b7f0a2), which improved the PG optimizer's ability to estimate the size of a CTE scan. The impacted query involves a CTE scan with a point predicate `(a=123)` and before the change the CTE size was estimated to be 5000, but with the change it is correctly (given the data in the table) estimated to be 1, making hash join a more attractive join method. The fix is to have an alternative goldfile for pre-PG17. I tried, but was unable, to force a specific kind of join method using the GUCs (`enable_nestloop`, `enable_hashjoin`, `enable_mergejoin`), but it was not possible to obtain a consistent plan across all supported PG versions (in some cases the join inputs switched sides). --- .../expected/columnar_chunk_filtering.out | 18 +- .../expected/columnar_chunk_filtering_0.out | 6 + src/test/regress/expected/columnar_paths.out | 19 +- .../regress/expected/columnar_paths_0.out | 620 ++++++++++++++++++ .../regress/expected/multi_test_helpers.out | 20 + .../regress/sql/columnar_chunk_filtering.sql | 2 + src/test/regress/sql/columnar_paths.sql | 6 + src/test/regress/sql/multi_test_helpers.sql | 21 + 8 files changed, 697 insertions(+), 15 deletions(-) create mode 100644 src/test/regress/expected/columnar_paths_0.out diff --git a/src/test/regress/expected/columnar_chunk_filtering.out b/src/test/regress/expected/columnar_chunk_filtering.out index 3acdd957d..f952eb27b 100644 --- a/src/test/regress/expected/columnar_chunk_filtering.out +++ b/src/test/regress/expected/columnar_chunk_filtering.out @@ -977,6 +977,7 @@ DETAIL: unparameterized; 1 clauses pushed down (1 row) SET hash_mem_multiplier = 1.0; +SELECT public.explain_with_pg16_subplan_format($Q$ EXPLAIN (analyze on, costs off, timing off, summary off) SELECT sum(a) FROM pushdown_test where ( @@ -989,13 +990,18 @@ SELECT sum(a) FROM pushdown_test where ) or (a > 200000-2010); +$Q$) as "QUERY PLAN"; NOTICE: columnar planner: adding CustomScan path for pushdown_test DETAIL: unparameterized; 0 clauses pushed down +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' HINT: Var must only reference this rel, and Expr must not reference this rel +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: cannot push down clause: must not contain a subplan +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: adding CustomScan path for pushdown_test DETAIL: unparameterized; 1 clauses pushed down +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement QUERY PLAN --------------------------------------------------------------------- Aggregate (actual rows=1 loops=1) @@ -1092,14 +1098,14 @@ BEGIN; END; EXPLAIN (analyze on, costs off, timing off, summary off) SELECT id FROM pushdown_test WHERE country IN ('USA', 'BR', 'ZW'); - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1) - Filter: (country = ANY ('{USA,BR,ZW}'::text[])) - Rows Removed by Filter: 1 - Columnar Projected Columns: id, country - Columnar Chunk Group Filters: (country = ANY ('{USA,BR,ZW}'::text[])) - Columnar Chunk Groups Removed by Filter: 2 + Filter: (country = ANY ('{USA,BR,ZW}'::text[])) + Rows Removed by Filter: 1 + Columnar Projected Columns: id, country + Columnar Chunk Group Filters: (country = ANY ('{USA,BR,ZW}'::text[])) + Columnar Chunk Groups Removed by Filter: 2 (6 rows) SELECT id FROM pushdown_test WHERE country IN ('USA', 'BR', 'ZW'); diff --git a/src/test/regress/expected/columnar_chunk_filtering_0.out b/src/test/regress/expected/columnar_chunk_filtering_0.out index 746f3406f..57b30b8b1 100644 --- a/src/test/regress/expected/columnar_chunk_filtering_0.out +++ b/src/test/regress/expected/columnar_chunk_filtering_0.out @@ -977,6 +977,7 @@ DETAIL: unparameterized; 1 clauses pushed down (1 row) SET hash_mem_multiplier = 1.0; +SELECT public.explain_with_pg16_subplan_format($Q$ EXPLAIN (analyze on, costs off, timing off, summary off) SELECT sum(a) FROM pushdown_test where ( @@ -989,13 +990,18 @@ SELECT sum(a) FROM pushdown_test where ) or (a > 200000-2010); +$Q$) as "QUERY PLAN"; NOTICE: columnar planner: adding CustomScan path for pushdown_test DETAIL: unparameterized; 0 clauses pushed down +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' HINT: Var must only reference this rel, and Expr must not reference this rel +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: cannot push down clause: must not contain a subplan +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement NOTICE: columnar planner: adding CustomScan path for pushdown_test DETAIL: unparameterized; 1 clauses pushed down +CONTEXT: PL/pgSQL function explain_with_pg16_subplan_format(text) line XX at FOR over EXECUTE statement QUERY PLAN --------------------------------------------------------------------- Aggregate (actual rows=1 loops=1) diff --git a/src/test/regress/expected/columnar_paths.out b/src/test/regress/expected/columnar_paths.out index 07b91a42e..1c4bfc608 100644 --- a/src/test/regress/expected/columnar_paths.out +++ b/src/test/regress/expected/columnar_paths.out @@ -1,5 +1,10 @@ CREATE SCHEMA columnar_paths; SET search_path TO columnar_paths; +-- columnar_paths has an alternative test output file because PG17 improved +-- the optimizer's ability to use statistics to estimate the size of a CTE +-- scan. +-- The relevant PG commit is: +-- https://github.com/postgres/postgres/commit/f7816aec23eed1dc1da5f9a53cb6507d30b7f0a2 CREATE TABLE full_correlated (a int, b text, c int, d int) USING columnar; INSERT INTO full_correlated SELECT i, i::text FROM generate_series(1, 1000000) i; CREATE INDEX full_correlated_btree ON full_correlated (a); @@ -296,20 +301,16 @@ SELECT * FROM w AS w1 JOIN w AS w2 ON w1.a = w2.d WHERE w2.a = 123; QUERY PLAN --------------------------------------------------------------------- - Merge Join - Merge Cond: (w2.d = w1.a) + Hash Join + Hash Cond: (w1.a = w2.d) CTE w -> Custom Scan (ColumnarScan) on full_correlated Columnar Projected Columns: a, b, c, d - -> Sort - Sort Key: w2.d + -> CTE Scan on w w1 + -> Hash -> CTE Scan on w w2 Filter: (a = 123) - -> Materialize - -> Sort - Sort Key: w1.a - -> CTE Scan on w w1 -(13 rows) +(9 rows) -- use index EXPLAIN (COSTS OFF) WITH w AS NOT MATERIALIZED (SELECT * FROM full_correlated) diff --git a/src/test/regress/expected/columnar_paths_0.out b/src/test/regress/expected/columnar_paths_0.out new file mode 100644 index 000000000..2b7349e42 --- /dev/null +++ b/src/test/regress/expected/columnar_paths_0.out @@ -0,0 +1,620 @@ +CREATE SCHEMA columnar_paths; +SET search_path TO columnar_paths; +-- columnar_paths has an alternative test output file because PG17 improved +-- the optimizer's ability to use statistics to estimate the size of a CTE +-- scan. +-- The relevant PG commit is: +-- https://github.com/postgres/postgres/commit/f7816aec23eed1dc1da5f9a53cb6507d30b7f0a2 +CREATE TABLE full_correlated (a int, b text, c int, d int) USING columnar; +INSERT INTO full_correlated SELECT i, i::text FROM generate_series(1, 1000000) i; +CREATE INDEX full_correlated_btree ON full_correlated (a); +ANALYZE full_correlated; +-- Prevent qual pushdown from competing with index scans. +SET columnar.enable_qual_pushdown = false; +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a=200; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a<0; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a>10 AND a<20; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a>1000000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a>900000; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a FROM full_correlated WHERE a>900000; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a<1000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a,b FROM full_correlated WHERE a<3000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a<9000; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_index_scan ( + $$ + SELECT a FROM full_correlated WHERE a<9000; + $$ + ); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +BEGIN; + TRUNCATE full_correlated; + INSERT INTO full_correlated SELECT i, i::text FROM generate_series(1, 1000) i; + -- Since we have much smaller number of rows, selectivity of below + -- query should be much higher. So we would choose columnar custom scan. + SELECT columnar_test_helpers.uses_custom_scan ( + $$ + SELECT a FROM full_correlated WHERE a=200; + $$ + ); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a FROM full_correlated WHERE a=200; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same filter used in above, but choosing multiple columns would increase +-- custom scan cost, so we would prefer index scan this time +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a,b,c,d FROM full_correlated WHERE a<9000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_index_scan ( + $$ + SELECT a,b,c,d FROM full_correlated WHERE a<9000; + $$ + ); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- again same filter used in above, but we would choose custom scan this +-- time since it would read three less columns from disk +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT c FROM full_correlated WHERE a<10000; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_index_scan ( + $$ + SELECT c FROM full_correlated WHERE a<10000; + $$ + ); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a>200; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a=0 OR a=5; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a FROM full_correlated WHERE a=0 OR a=5; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- +-- some tests with joins / subqueries etc. +-- +CREATE TABLE heap_table (a int, b text, c int, d int); +INSERT INTO heap_table SELECT i, i::text, (i+1000)*7, (i+900)*5 FROM generate_series(1, 1000000) i; +CREATE INDEX heap_table_btree ON heap_table (a); +ANALYZE heap_table; +EXPLAIN (COSTS OFF) +WITH cte AS MATERIALIZED (SELECT d FROM full_correlated WHERE a > 1) +SELECT SUM(ht_1.a), MIN(ct_1.c) +FROM heap_table AS ht_1 +LEFT JOIN full_correlated AS ct_1 ON ht_1.a=ct_1.d +LEFT JOIN heap_table AS ht_2 ON ht_2.a=ct_1.c +JOIN cte ON cte.d=ht_1.a +WHERE ct_1.a < 3000; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + CTE cte + -> Custom Scan (ColumnarScan) on full_correlated + Filter: (a > 1) + Columnar Projected Columns: a, d + -> Nested Loop Left Join + -> Hash Join + Hash Cond: (cte.d = ht_1.a) + -> CTE Scan on cte + -> Hash + -> Nested Loop + -> Index Scan using full_correlated_btree on full_correlated ct_1 + Index Cond: (a < 3000) + -> Index Only Scan using heap_table_btree on heap_table ht_1 + Index Cond: (a = ct_1.d) + -> Index Only Scan using heap_table_btree on heap_table ht_2 + Index Cond: (a = ct_1.c) +(17 rows) + +-- same query but columnar custom scan is disabled +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + EXPLAIN (COSTS OFF) + WITH cte AS MATERIALIZED (SELECT d FROM full_correlated WHERE a > 1) + SELECT SUM(ht_1.a), MIN(ct_1.c) + FROM heap_table AS ht_1 + LEFT JOIN full_correlated AS ct_1 ON ht_1.a=ct_1.d + LEFT JOIN heap_table AS ht_2 ON ht_2.a=ct_1.c + JOIN cte ON cte.d=ht_1.a + WHERE ct_1.a < 3000; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + CTE cte + -> Seq Scan on full_correlated + Filter: (a > 1) + -> Nested Loop Left Join + -> Hash Join + Hash Cond: (cte.d = ht_1.a) + -> CTE Scan on cte + -> Hash + -> Nested Loop + -> Index Scan using full_correlated_btree on full_correlated ct_1 + Index Cond: (a < 3000) + -> Index Only Scan using heap_table_btree on heap_table ht_1 + Index Cond: (a = ct_1.d) + -> Index Only Scan using heap_table_btree on heap_table ht_2 + Index Cond: (a = ct_1.c) +(16 rows) + +ROLLBACK; +-- use custom scan +EXPLAIN (COSTS OFF) WITH w AS (SELECT * FROM full_correlated) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.a = w2.d +WHERE w2.a = 123; + QUERY PLAN +--------------------------------------------------------------------- + Merge Join + Merge Cond: (w2.d = w1.a) + CTE w + -> Custom Scan (ColumnarScan) on full_correlated + Columnar Projected Columns: a, b, c, d + -> Sort + Sort Key: w2.d + -> CTE Scan on w w2 + Filter: (a = 123) + -> Materialize + -> Sort + Sort Key: w1.a + -> CTE Scan on w w1 +(13 rows) + +-- use index +EXPLAIN (COSTS OFF) WITH w AS NOT MATERIALIZED (SELECT * FROM full_correlated) +SELECT * FROM w AS w1 JOIN w AS w2 ON w1.a = w2.d +WHERE w2.a = 123; + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop + -> Index Scan using full_correlated_btree on full_correlated full_correlated_1 + Index Cond: (a = 123) + -> Index Scan using full_correlated_btree on full_correlated + Index Cond: (a = full_correlated_1.d) +(5 rows) + +EXPLAIN (COSTS OFF) SELECT sub_1.b, sub_2.a, sub_3.avg +FROM + (SELECT b FROM full_correlated WHERE (a > 2) GROUP BY b ORDER BY 1 DESC LIMIT 5) AS sub_1, + (SELECT a FROM full_correlated WHERE (a > 10) GROUP BY a HAVING count(DISTINCT a) >= 1 ORDER BY 1 DESC LIMIT 3) AS sub_2, + (SELECT avg(a) AS AVG FROM full_correlated WHERE (a > 2) GROUP BY a HAVING sum(a) > 10 ORDER BY (sum(d) - avg(a) - COALESCE(array_upper(ARRAY[max(a)],1) * 5, 0)) DESC LIMIT 3) AS sub_3 +WHERE sub_2.a < sub_1.b::integer +ORDER BY 3 DESC, 2 DESC, 1 DESC +LIMIT 100; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: sub_3.avg DESC, full_correlated_1.a DESC, full_correlated.b DESC + -> Nested Loop + -> Nested Loop + Join Filter: (full_correlated_1.a < (full_correlated.b)::integer) + -> Limit + -> Sort + Sort Key: full_correlated.b DESC + -> HashAggregate + Group Key: full_correlated.b + -> Custom Scan (ColumnarScan) on full_correlated + Filter: (a > 2) + Columnar Projected Columns: a, b + -> Materialize + -> Limit + -> GroupAggregate + Group Key: full_correlated_1.a + Filter: (count(DISTINCT full_correlated_1.a) >= 1) + -> Index Scan Backward using full_correlated_btree on full_correlated full_correlated_1 + Index Cond: (a > 10) + -> Materialize + -> Subquery Scan on sub_3 + -> Limit + -> Sort + Sort Key: ((((sum(full_correlated_2.d))::numeric - avg(full_correlated_2.a)) - (COALESCE((array_upper(ARRAY[max(full_correlated_2.a)], 1) * 5), 0))::numeric)) DESC + -> GroupAggregate + Group Key: full_correlated_2.a + Filter: (sum(full_correlated_2.a) > 10) + -> Index Scan using full_correlated_btree on full_correlated full_correlated_2 + Index Cond: (a > 2) +(31 rows) + +DROP INDEX full_correlated_btree; +CREATE INDEX full_correlated_hash ON full_correlated USING hash(a); +ANALYZE full_correlated; +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a<10; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a>1 AND a<10; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_correlated WHERE a=0 OR a=5; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_correlated WHERE a=1000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a,c FROM full_correlated WHERE a=1000; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_index_scan ( + $$ + SELECT a,c FROM full_correlated WHERE a=1000; + $$ + ); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +CREATE TABLE full_anti_correlated (a int, b text) USING columnar; +INSERT INTO full_anti_correlated SELECT i, i::text FROM generate_series(1, 500000) i; +CREATE INDEX full_anti_correlated_hash ON full_anti_correlated USING hash(b); +ANALYZE full_anti_correlated; +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_anti_correlated WHERE b='600'; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a,b FROM full_anti_correlated WHERE b='600'; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a,b FROM full_anti_correlated WHERE b='600' OR b='10'; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a,b FROM full_anti_correlated WHERE b='600' OR b='10'; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +DROP INDEX full_anti_correlated_hash; +CREATE INDEX full_anti_correlated_btree ON full_anti_correlated (a,b); +ANALYZE full_anti_correlated; +SELECT columnar_test_helpers.uses_index_scan ( +$$ +SELECT a FROM full_anti_correlated WHERE a>6500 AND a<7000 AND b<'10000'; +$$ +); + uses_index_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_anti_correlated WHERE a>2000 AND a<7000; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM full_anti_correlated WHERE a<7000 AND b<'10000'; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a FROM full_anti_correlated WHERE a<7000 AND b<'10000'; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +CREATE TABLE no_correlation (a int, b text) USING columnar; +INSERT INTO no_correlation SELECT random()*5000, (random()*5000)::int::text FROM generate_series(1, 500000) i; +CREATE INDEX no_correlation_btree ON no_correlation (a); +ANALYZE no_correlation; +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM no_correlation WHERE a < 2; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +SELECT columnar_test_helpers.uses_custom_scan ( +$$ +SELECT a FROM no_correlation WHERE a = 200; +$$ +); + uses_custom_scan +--------------------------------------------------------------------- + t +(1 row) + +BEGIN; + SET LOCAL columnar.enable_custom_scan TO 'OFF'; + SELECT columnar_test_helpers.uses_seq_scan ( + $$ + SELECT a FROM no_correlation WHERE a = 200; + $$ + ); + uses_seq_scan +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +SET columnar.enable_qual_pushdown TO DEFAULT; +BEGIN; +SET LOCAL columnar.stripe_row_limit = 2000; +SET LOCAL columnar.chunk_group_row_limit = 1000; +CREATE TABLE correlated(x int) using columnar; +INSERT INTO correlated + SELECT g FROM generate_series(1,100000) g; +CREATE TABLE uncorrelated(x int) using columnar; +INSERT INTO uncorrelated + SELECT (g * 19) % 100000 FROM generate_series(1,100000) g; +COMMIT; +CREATE INDEX correlated_idx ON correlated(x); +CREATE INDEX uncorrelated_idx ON uncorrelated(x); +ANALYZE correlated, uncorrelated; +-- should choose chunk group filtering; selective and correlated +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM correlated WHERE x = 78910; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on correlated (actual rows=1 loops=1) + Filter: (x = 78910) + Rows Removed by Filter: 999 + Columnar Projected Columns: x + Columnar Chunk Group Filters: (x = 78910) + Columnar Chunk Groups Removed by Filter: 99 +(6 rows) + +SELECT * FROM correlated WHERE x = 78910; + x +--------------------------------------------------------------------- + 78910 +(1 row) + +-- should choose index scan; selective but uncorrelated +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM uncorrelated WHERE x = 78910; + QUERY PLAN +--------------------------------------------------------------------- + Index Scan using uncorrelated_idx on uncorrelated (actual rows=1 loops=1) + Index Cond: (x = 78910) +(2 rows) + +SELECT * FROM uncorrelated WHERE x = 78910; + x +--------------------------------------------------------------------- + 78910 +(1 row) + +SET client_min_messages TO WARNING; +DROP SCHEMA columnar_paths CASCADE; diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index f66ff1e47..8797f56b5 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -585,3 +585,23 @@ BEGIN RETURN NEXT; END LOOP; END; $$ language plpgsql; +-- This function formats EXPLAIN output to conform to how pg <= 16 EXPLAIN +-- shows ANY in an expression the pg version >= 17. When 17 is +-- the minimum supported pgversion this function can be retired. The commit +-- that changed how ANY exrpressions appear in EXPLAIN is: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb +CREATE OR REPLACE FUNCTION explain_with_pg16_subplan_format(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + pgversion int = 0; +BEGIN + pgversion = substring(version(), '\d+')::int ; + FOR query_plan IN execute explain_command LOOP + IF pgversion >= 17 THEN + IF query_plan ~ 'SubPlan \d+\).col' THEN + query_plan = regexp_replace(query_plan, '\(ANY \(\w+ = \(SubPlan (\d+)\).col1\)\)', '(SubPlan \1)', 'g'); + END IF; + END IF; + RETURN NEXT; + END LOOP; +END; $$ language plpgsql; diff --git a/src/test/regress/sql/columnar_chunk_filtering.sql b/src/test/regress/sql/columnar_chunk_filtering.sql index d37a8d8b6..6c90e1943 100644 --- a/src/test/regress/sql/columnar_chunk_filtering.sql +++ b/src/test/regress/sql/columnar_chunk_filtering.sql @@ -415,6 +415,7 @@ SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 2000 SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010); SET hash_mem_multiplier = 1.0; +SELECT public.explain_with_pg16_subplan_format($Q$ EXPLAIN (analyze on, costs off, timing off, summary off) SELECT sum(a) FROM pushdown_test where ( @@ -427,6 +428,7 @@ SELECT sum(a) FROM pushdown_test where ) or (a > 200000-2010); +$Q$) as "QUERY PLAN"; RESET hash_mem_multiplier; SELECT sum(a) FROM pushdown_test where ( diff --git a/src/test/regress/sql/columnar_paths.sql b/src/test/regress/sql/columnar_paths.sql index 3c92d4a21..d56443a03 100644 --- a/src/test/regress/sql/columnar_paths.sql +++ b/src/test/regress/sql/columnar_paths.sql @@ -1,6 +1,12 @@ CREATE SCHEMA columnar_paths; SET search_path TO columnar_paths; +-- columnar_paths has an alternative test output file because PG17 improved +-- the optimizer's ability to use statistics to estimate the size of a CTE +-- scan. +-- The relevant PG commit is: +-- https://github.com/postgres/postgres/commit/f7816aec23eed1dc1da5f9a53cb6507d30b7f0a2 + CREATE TABLE full_correlated (a int, b text, c int, d int) USING columnar; INSERT INTO full_correlated SELECT i, i::text FROM generate_series(1, 1000000) i; CREATE INDEX full_correlated_btree ON full_correlated (a); diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index e39ae451b..d0a393a8e 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -611,3 +611,24 @@ BEGIN RETURN NEXT; END LOOP; END; $$ language plpgsql; + +-- This function formats EXPLAIN output to conform to how pg <= 16 EXPLAIN +-- shows ANY in an expression the pg version >= 17. When 17 is +-- the minimum supported pgversion this function can be retired. The commit +-- that changed how ANY exrpressions appear in EXPLAIN is: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb +CREATE OR REPLACE FUNCTION explain_with_pg16_subplan_format(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + pgversion int = 0; +BEGIN + pgversion = substring(version(), '\d+')::int ; + FOR query_plan IN execute explain_command LOOP + IF pgversion >= 17 THEN + IF query_plan ~ 'SubPlan \d+\).col' THEN + query_plan = regexp_replace(query_plan, '\(ANY \(\w+ = \(SubPlan (\d+)\).col1\)\)', '(SubPlan \1)', 'g'); + END IF; + END IF; + RETURN NEXT; + END LOOP; +END; $$ language plpgsql; From 063be4644487ef611db388e92dddc15d51d58595 Mon Sep 17 00:00:00 2001 From: Colm Date: Wed, 4 Dec 2024 15:29:39 +0000 Subject: [PATCH 076/155] =?UTF-8?q?PG17=20compatibility:=20Fix=20check-sty?= =?UTF-8?q?le,=20broken=20by=20PG17=20columnar=20test=20fix=E2=80=A6=20(#7?= =?UTF-8?q?776)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … (698699d89efafe3) --------- Co-authored-by: naisila --- src/test/regress/sql/columnar_paths.sql | 2 +- src/test/regress/sql/multi_test_helpers.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/regress/sql/columnar_paths.sql b/src/test/regress/sql/columnar_paths.sql index d56443a03..748b9006a 100644 --- a/src/test/regress/sql/columnar_paths.sql +++ b/src/test/regress/sql/columnar_paths.sql @@ -3,7 +3,7 @@ SET search_path TO columnar_paths; -- columnar_paths has an alternative test output file because PG17 improved -- the optimizer's ability to use statistics to estimate the size of a CTE --- scan. +-- scan. -- The relevant PG commit is: -- https://github.com/postgres/postgres/commit/f7816aec23eed1dc1da5f9a53cb6507d30b7f0a2 diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index d0a393a8e..1cdcf4b39 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -612,9 +612,9 @@ BEGIN END LOOP; END; $$ language plpgsql; --- This function formats EXPLAIN output to conform to how pg <= 16 EXPLAIN +-- This function formats EXPLAIN output to conform to how pg <= 16 EXPLAIN -- shows ANY in an expression the pg version >= 17. When 17 is --- the minimum supported pgversion this function can be retired. The commit +-- the minimum supported pgversion this function can be retired. The commit -- that changed how ANY exrpressions appear in EXPLAIN is: -- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=fd0398fcb CREATE OR REPLACE FUNCTION explain_with_pg16_subplan_format(explain_command text, out query_plan text) From 30a75eadf166f6fc63d0dd122c7e8aaca3909b0c Mon Sep 17 00:00:00 2001 From: Colm Date: Thu, 5 Dec 2024 10:03:28 +0000 Subject: [PATCH 077/155] PG17 compatibility: fix diffs in create_index, privileges vanilla tests (#7766) PG17 regress sanity (#7653) fix; address diffs in vanilla tests `create_index` and `privileges`. There is a change from `permission denied` to `must be owner of`, seen in create_index: ``` @@ -2970,21 +2970,21 @@ REINDEX TABLE pg_toast.pg_toast_1260; ERROR: permission denied for table pg_toast_1260 REINDEX INDEX pg_toast.pg_toast_1260_index; -ERROR: permission denied for index pg_toast_1260_index +ERROR: must be owner of index pg_toast_1260_index ``` and privileges: ``` @@ -2945,41 +2945,43 @@ ERROR: permission denied for table maintain_test REINDEX INDEX maintain_test_a_idx; -ERROR: permission denied for index maintain_test_a_idx +ERROR: must be owner of index maintain_test_a_idx REINDEX SCHEMA reindex_test; REINDEX INDEX maintain_test_a_idx; +ERROR: must be owner of index maintain_test_a_idx REINDEX SCHEMA reindex_test; ``` The fix updates function `RangeVarCallbackForReindexIndex()` in `index.c` with changes made by the introduction of the [MAINTAIN privilege in PG17](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337) to the function `RangeVarCallbackForReindexIndex()` in `indexcmds.c`. The code is under a Postgres 17 version directive, which can be removed when 17 becomes the oldest supported Postgres version. --- src/backend/distributed/commands/index.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 2b6941393..6598b8a7c 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -1109,6 +1109,7 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelI char relkind; struct ReindexIndexCallbackState *state = arg; LOCKMODE table_lockmode; + Oid table_oid; /* * Lock level here should match table lock in reindex_index() for @@ -1146,13 +1147,24 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelI errmsg("\"%s\" is not an index", relation->relname))); /* Check permissions */ + + #if PG_VERSION_NUM >= PG_VERSION_17 + table_oid = IndexGetRelation(relId, true); + if (OidIsValid(table_oid)) + { + AclResult aclresult = pg_class_aclcheck(table_oid, GetUserId(), ACL_MAINTAIN); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, OBJECT_INDEX, relation->relname); + } + #else if (!object_ownercheck(RelationRelationId, relId, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relation->relname); + #endif /* Lock heap before index to avoid deadlock. */ if (relId != oldRelId) { - Oid table_oid = IndexGetRelation(relId, true); + table_oid = IndexGetRelation(relId, true); /* * If the OID isn't valid, it means the index was concurrently From 90d76e809764789ba04b9802c27ecb28ac82626f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:47:51 +0300 Subject: [PATCH 078/155] Fix bug: alter database shouldn't propagate for undistributed database (#7777) Before this fix, the following would happen for a database that is not distributed: ```sql CREATE DATABASE db_to_test; NOTICE: Citus partially supports CREATE DATABASE for distributed databases DETAIL: Citus does not propagate CREATE DATABASE command to workers HINT: You can manually create a database and its extensions on workers. ALTER DATABASE db_to_test with CONNECTION LIMIT 100; NOTICE: issuing ALTER DATABASE db_to_test WITH CONNECTION LIMIT 100; DETAIL: on server postgres@localhost:57638 connectionId: 2 NOTICE: issuing ALTER DATABASE db_to_test WITH CONNECTION LIMIT 100; DETAIL: on server postgres@localhost:57637 connectionId: 1 ERROR: database "db_to_test" does not exist ``` With this fix, we also take care of https://github.com/citusdata/citus/issues/7763 Fixes #7763 --- src/backend/distributed/commands/database.c | 25 ++++++++++++++++--- .../expected/alter_database_propagation.out | 7 ++++++ .../sql/alter_database_propagation.sql | 5 ++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index a668c9ba6..c92191774 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -24,6 +24,7 @@ #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" +#include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" @@ -32,6 +33,8 @@ static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); static Oid get_database_owner(Oid db_oid); +static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, + bool missingOk); List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); @@ -161,13 +164,15 @@ List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { - if (!ShouldPropagate()) + AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); + bool missingOk = false; + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, + missingOk); + if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } - AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); - EnsureCoordinator(); char *sql = DeparseTreeNode((Node *) stmt); @@ -213,3 +218,17 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, #endif + + +/* + * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress + * of the database. + */ +static ObjectAddress * +GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) +{ + Oid databaseOid = get_database_oid(databaseName, missingOk); + ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, databaseOid); + return dbObjectAddress; +} diff --git a/src/test/regress/expected/alter_database_propagation.out b/src/test/regress/expected/alter_database_propagation.out index b7d04c50f..b8a3a0ae9 100644 --- a/src/test/regress/expected/alter_database_propagation.out +++ b/src/test/regress/expected/alter_database_propagation.out @@ -33,4 +33,11 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- this statement will get error since we don't have a multiple database support for now alter database regression rename to regression2; ERROR: current database cannot be renamed +-- this database is not distributed so we will not see any remote commands +CREATE DATABASE db_to_test; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +alter database db_to_test with CONNECTION LIMIT 100; +DROP DATABASE db_to_test; set citus.log_remote_commands = false; diff --git a/src/test/regress/sql/alter_database_propagation.sql b/src/test/regress/sql/alter_database_propagation.sql index 748a66ddd..ace558cec 100644 --- a/src/test/regress/sql/alter_database_propagation.sql +++ b/src/test/regress/sql/alter_database_propagation.sql @@ -15,4 +15,9 @@ alter database regression with IS_TEMPLATE false; -- this statement will get error since we don't have a multiple database support for now alter database regression rename to regression2; +-- this database is not distributed so we will not see any remote commands +CREATE DATABASE db_to_test; +alter database db_to_test with CONNECTION LIMIT 100; +DROP DATABASE db_to_test; + set citus.log_remote_commands = false; From 7c9280f6e3419088f23b5b262bea0e50d2a5067e Mon Sep 17 00:00:00 2001 From: Colm Date: Fri, 6 Dec 2024 11:55:12 +0000 Subject: [PATCH 079/155] PG17 compatibility: fix multi-1 diffs caused by PG17 optimizer enhancements (#7769) This fix ensures that the expected DEBUG error messages from the router planner in `multi_router_planner`, `multi_router_planner_fast_path` and `query_single_shard_table` are present with PG17. In `query_single_shard_table` the diff: ``` SELECT COUNT(*) FROM citus_local_table t1 WHERE t1.b IN ( SELECT b+1 FROM nullkey_c1_t1 t2 WHERE t2.b = t1.a ); -DEBUG: router planner does not support queries that reference non-colocated distributed tables +DEBUG: Local tables cannot be used in distributed queries. ``` occurred because of[ this PG17 commit](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9f1337639) which enables the optimizer to pull up a correlated ANY subquery to a join. The fix inhibits subquery pull up by including a volatile function in the predicate involving the ANY subquery, preserving the pre-PG17 optimizer treatment of the query. In the case of `multi_router_planner` and `multi_router_planner_fast_path` the diffs: ``` -- partition_column is null clause does not prune out any shards, -- all shards remain after shard pruning, not router plannable SELECT * FROM articles_hash a WHERE a.author_id is null; -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan ``` are because of [this PG17 commit](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=b262ad440), which enables the optimizer to detect and remove redundant IS (NOT) NULL expressions. The fix is to adjust the table definition so the column used for distribution is not marked NOT NULL, thus preserving the pre-PG17 query planning behavior. Finallly, a rule is added to `normalize.sed` to ignore DEBUG logging in CREATE MATERIALIZED VIEW AS statements introduced by [this PG17 commit](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=b4da732fd64); _when creating materialized views, use REFRESH logic to load data_, a consequence of which is that with `client_min_messages` at `DEBUG2` Postgres emits extra detail for CREATE MATERIALIZED VIEW AS statements. ``` CREATE MATERIALIZED VIEW mv_articles_hash_empty AS SELECT * FROM articles_hash WHERE author_id = 1; DEBUG: Creating router plan DEBUG: query has a single distribution column value: 1 +DEBUG: drop auto-cascades to type multi_router_planner.pg_temp_61391 +DEBUG: drop auto-cascades to type multi_router_planner.pg_temp_61391[] ``` The rule can be changed to a normalization, or possibly dropped, when 17 becomes the minimum supported version. --- src/test/regress/bin/normalize.sed | 7 +++++++ src/test/regress/expected/multi_router_planner.out | 2 +- .../regress/expected/multi_router_planner_fast_path.out | 2 +- src/test/regress/expected/query_single_shard_table.out | 8 ++++---- src/test/regress/sql/multi_router_planner.sql | 2 +- src/test/regress/sql/multi_router_planner_fast_path.sql | 3 +-- src/test/regress/sql/query_single_shard_table.sql | 8 ++++---- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 19ac512c5..59bf6d97c 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -324,3 +324,10 @@ s/COPY delimiter must not appear in the DEFAULT specification/COPY delimiter cha # we add them back for pg15,pg16 compatibility # e.g. change CHECK other_col >= 100 to CHECK (other_col >= 100) s/\| CHECK ([a-zA-Z])(.*)/| CHECK \(\1\2\)/g + +# pg17 change: this is a rule that ignores additional DEBUG logging +# for CREATE MATERIALIZED VIEW (commit b4da732fd64). This could be +# changed to a normalization rule when 17 becomes the minimum +# supported Postgres version. + +/DEBUG: drop auto-cascades to type [a-zA-Z_]*.pg_temp_[0-9]*/d diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index fee821a7d..ce68d133d 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -10,7 +10,7 @@ CREATE SCHEMA multi_router_planner; SET search_path TO multi_router_planner; CREATE TABLE articles_hash ( id bigint NOT NULL, - author_id bigint NOT NULL, + author_id bigint, title varchar(20) NOT NULL, word_count integer ); diff --git a/src/test/regress/expected/multi_router_planner_fast_path.out b/src/test/regress/expected/multi_router_planner_fast_path.out index 25cc8a1a7..e483660ee 100644 --- a/src/test/regress/expected/multi_router_planner_fast_path.out +++ b/src/test/regress/expected/multi_router_planner_fast_path.out @@ -12,7 +12,7 @@ SET citus.enable_fast_path_router_planner TO true; -- =================================================================== CREATE TABLE articles_hash ( id bigint NOT NULL, - author_id bigint NOT NULL, + author_id bigint, title varchar(20) NOT NULL, word_count integer ); diff --git a/src/test/regress/expected/query_single_shard_table.out b/src/test/regress/expected/query_single_shard_table.out index 5f551a988..0945bc1d7 100644 --- a/src/test/regress/expected/query_single_shard_table.out +++ b/src/test/regress/expected/query_single_shard_table.out @@ -1183,7 +1183,7 @@ DEBUG: Local tables cannot be used in distributed queries. DEBUG: skipping recursive planning for the subquery since it contains references to outer queries ERROR: direct joins between distributed and local tables are not supported SELECT COUNT(*) FROM nullkey_c1_t1 t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM citus_local_table t2 WHERE t2.b = t1.a ); DEBUG: router planner does not support queries that reference non-colocated distributed tables @@ -1258,7 +1258,7 @@ DEBUG: Local tables cannot be used in distributed queries. DEBUG: skipping recursive planning for the subquery since it contains references to outer queries ERROR: direct joins between distributed and local tables are not supported SELECT COUNT(*) FROM citus_local_table t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM nullkey_c1_t1 t2 WHERE t2.b = t1.a ); DEBUG: router planner does not support queries that reference non-colocated distributed tables @@ -1312,7 +1312,7 @@ DEBUG: skipping recursive planning for the subquery since it contains reference ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins SELECT COUNT(*) FROM nullkey_c1_t1 t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM postgres_local_table t2 WHERE t2.b = t1.a ); DEBUG: found no worker with all shard placements @@ -1344,7 +1344,7 @@ DEBUG: skipping recursive planning for the subquery since it contains reference ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins SELECT COUNT(*) FROM postgres_local_table t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM nullkey_c1_t1 t2 WHERE t2.b = t1.a ); DEBUG: found no worker with all shard placements diff --git a/src/test/regress/sql/multi_router_planner.sql b/src/test/regress/sql/multi_router_planner.sql index 2ccd43ea3..20b8a5956 100644 --- a/src/test/regress/sql/multi_router_planner.sql +++ b/src/test/regress/sql/multi_router_planner.sql @@ -15,7 +15,7 @@ SET search_path TO multi_router_planner; CREATE TABLE articles_hash ( id bigint NOT NULL, - author_id bigint NOT NULL, + author_id bigint, title varchar(20) NOT NULL, word_count integer ); diff --git a/src/test/regress/sql/multi_router_planner_fast_path.sql b/src/test/regress/sql/multi_router_planner_fast_path.sql index 1fd1f6ce0..56684c075 100644 --- a/src/test/regress/sql/multi_router_planner_fast_path.sql +++ b/src/test/regress/sql/multi_router_planner_fast_path.sql @@ -18,7 +18,7 @@ SET citus.enable_fast_path_router_planner TO true; CREATE TABLE articles_hash ( id bigint NOT NULL, - author_id bigint NOT NULL, + author_id bigint, title varchar(20) NOT NULL, word_count integer ); @@ -803,7 +803,6 @@ CREATE MATERIALIZED VIEW mv_articles_hash_empty AS SELECT * FROM articles_hash WHERE author_id = 1; SELECT * FROM mv_articles_hash_empty; - SELECT id FROM articles_hash WHERE author_id = 1; diff --git a/src/test/regress/sql/query_single_shard_table.sql b/src/test/regress/sql/query_single_shard_table.sql index 96de2705c..4abda0bea 100644 --- a/src/test/regress/sql/query_single_shard_table.sql +++ b/src/test/regress/sql/query_single_shard_table.sql @@ -502,7 +502,7 @@ WHERE NOT EXISTS ( ); SELECT COUNT(*) FROM nullkey_c1_t1 t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM citus_local_table t2 WHERE t2.b = t1.a ); @@ -543,7 +543,7 @@ WHERE EXISTS ( ); SELECT COUNT(*) FROM citus_local_table t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM nullkey_c1_t1 t2 WHERE t2.b = t1.a ); @@ -573,7 +573,7 @@ WHERE NOT EXISTS ( ); SELECT COUNT(*) FROM nullkey_c1_t1 t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM postgres_local_table t2 WHERE t2.b = t1.a ); @@ -593,7 +593,7 @@ WHERE EXISTS ( ); SELECT COUNT(*) FROM postgres_local_table t1 -WHERE t1.b IN ( +WHERE t1.b + random() IN ( SELECT b+1 FROM nullkey_c1_t1 t2 WHERE t2.b = t1.a ); From 1c2e78405b1e483a4ef92e9b2ee7bc68f61688ce Mon Sep 17 00:00:00 2001 From: Colm Date: Fri, 6 Dec 2024 13:03:51 +0000 Subject: [PATCH 080/155] PG17 compatibility: account for MAINTAIN privilege in regress tests (#7774) This PR addresses regress tests impacted by the introduction of [the MAINTAIN privilege in PG17](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337). The impacted tests include `generated_identity`, `create_single_shard_table`, `grant_on_sequence_propagation`, `grant_on_foreign_server_propagation`, `single_node_enterprise`, `multi_multiuser_master_protocol`, `multi_alter_table_row_level_security`, `shard_move_constraints` which show the following error: ``` SELECT start_metadata_sync_to_node('localhost', :worker_2_port); - start_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - +ERROR: unrecognized aclright: 16384 ``` and `multi_multiuser_master_protocol`, where the `pg_class.relacl` column has 'm' for MAINTAIN if applicable: ``` relname | rolname | relacl ---------------------+-------------+------------------------------------------------------------ trivial_full_access | full_access | - trivial_postgres | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} ``` The PR updates function `convert_aclright_to_string()` in citus_ruleutils.c to include a case for `ACL_MAINTAIN`. Per the comment on `convert_aclright_to_string()` in citus_ruleutils.c, it is a copy of `convert_aclright_to_string()` in Postgres (where it is in `src/backend/utils/adt/acl.c`), so requires updating to be consistent with Postgres. With this change Citus can recognize the MAINTAIN privilege, and will not emit the `unrecognized aclright` error. The PR also adds an alternative goldfile for `multi_multiuser_master_protocol`. Note that `convert_aclright_to_string()` in Postgres includes access types SET and ALTER SYSTEM on system parameters (aka GUCs), added by [this PG16 commit](https://github.com/postgres/postgres/commit/a0ffa885e). If Citus were to have a requirement to support granting SET and ALTER SYSTEM we would need to update `convert_aclright_to_string()` in citus_ruleutils.c with SET and ALTER SYSTEM. --- .../distributed/deparser/citus_ruleutils.c | 4 + .../multi_multiuser_master_protocol.out | 44 +- .../multi_multiuser_master_protocol_0.out | 498 ++++++++++++++++++ src/test/regress/expected/pg17.out | 106 +++- src/test/regress/expected/pg17_0.out | 3 - .../sql/multi_multiuser_master_protocol.sql | 6 + src/test/regress/sql/pg17.sql | 78 ++- 7 files changed, 696 insertions(+), 43 deletions(-) create mode 100644 src/test/regress/expected/multi_multiuser_master_protocol_0.out diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 530f6e720..e5445b3df 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -1359,6 +1359,10 @@ convert_aclright_to_string(int aclright) return "TEMPORARY"; case ACL_CONNECT: return "CONNECT"; +#if PG_VERSION_NUM >= PG_VERSION_17 + case ACL_MAINTAIN: + return "MAINTAIN"; +#endif default: elog(ERROR, "unrecognized aclright: %d", aclright); return NULL; diff --git a/src/test/regress/expected/multi_multiuser_master_protocol.out b/src/test/regress/expected/multi_multiuser_master_protocol.out index a6bddb7f2..9d08bf454 100644 --- a/src/test/regress/expected/multi_multiuser_master_protocol.out +++ b/src/test/regress/expected/multi_multiuser_master_protocol.out @@ -1,6 +1,11 @@ -- -- MULTI_MULTIUSER_MASTER_PROTOCOL -- +-- Test multi_multiuser_master_protocol has an alternative output file because +-- PG17's support for the MAINTAIN privilege: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337 +-- means that calls of master_get_table_ddl_events() can show MAINTAIN and the +-- pg_class.relacl column may have 'm' for MAINTAIN ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 109079; -- Tests that check the metadata returned by the master node. At the -- same time ensure that any user, not just a superuser, can call @@ -19,6 +24,8 @@ SELECT * FROM master_get_table_ddl_events('lineitem') order by 1; GRANT DELETE ON public.lineitem TO postgres GRANT INSERT ON public.lineitem TO full_access GRANT INSERT ON public.lineitem TO postgres + GRANT MAINTAIN ON public.lineitem TO full_access + GRANT MAINTAIN ON public.lineitem TO postgres GRANT REFERENCES ON public.lineitem TO full_access GRANT REFERENCES ON public.lineitem TO postgres GRANT SELECT ON public.lineitem TO full_access @@ -31,7 +38,7 @@ SELECT * FROM master_get_table_ddl_events('lineitem') order by 1; GRANT UPDATE ON public.lineitem TO full_access GRANT UPDATE ON public.lineitem TO postgres REVOKE ALL ON public.lineitem FROM PUBLIC -(20 rows) +(22 rows) SELECT * FROM master_get_new_shardid(); master_get_new_shardid @@ -75,8 +82,9 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO postgres GRANT REFERENCES ON public.checkperm TO postgres GRANT TRIGGER ON public.checkperm TO postgres + GRANT MAINTAIN ON public.checkperm TO postgres ALTER TABLE public.checkperm OWNER TO postgres -(10 rows) +(11 rows) GRANT SELECT ON checkperm TO read_access; GRANT ALL ON checkperm TO full_access; @@ -92,6 +100,7 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO postgres GRANT REFERENCES ON public.checkperm TO postgres GRANT TRIGGER ON public.checkperm TO postgres + GRANT MAINTAIN ON public.checkperm TO postgres GRANT SELECT ON public.checkperm TO read_access GRANT INSERT ON public.checkperm TO full_access GRANT SELECT ON public.checkperm TO full_access @@ -100,8 +109,9 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO full_access GRANT REFERENCES ON public.checkperm TO full_access GRANT TRIGGER ON public.checkperm TO full_access + GRANT MAINTAIN ON public.checkperm TO full_access ALTER TABLE public.checkperm OWNER TO postgres -(18 rows) +(20 rows) REVOKE ALL ON checkperm FROM read_access; GRANT SELECT ON checkperm TO PUBLIC; @@ -117,6 +127,7 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO postgres GRANT REFERENCES ON public.checkperm TO postgres GRANT TRIGGER ON public.checkperm TO postgres + GRANT MAINTAIN ON public.checkperm TO postgres GRANT INSERT ON public.checkperm TO full_access GRANT SELECT ON public.checkperm TO full_access GRANT UPDATE ON public.checkperm TO full_access @@ -124,9 +135,10 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO full_access GRANT REFERENCES ON public.checkperm TO full_access GRANT TRIGGER ON public.checkperm TO full_access + GRANT MAINTAIN ON public.checkperm TO full_access GRANT SELECT ON public.checkperm TO PUBLIC ALTER TABLE public.checkperm OWNER TO postgres -(18 rows) +(20 rows) GRANT ALL ON checkperm TO full_access WITH GRANT OPTION; SELECT * FROM master_get_table_ddl_events('checkperm'); @@ -141,6 +153,7 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO postgres GRANT REFERENCES ON public.checkperm TO postgres GRANT TRIGGER ON public.checkperm TO postgres + GRANT MAINTAIN ON public.checkperm TO postgres GRANT INSERT ON public.checkperm TO full_access WITH GRANT OPTION GRANT SELECT ON public.checkperm TO full_access WITH GRANT OPTION GRANT UPDATE ON public.checkperm TO full_access WITH GRANT OPTION @@ -148,9 +161,10 @@ SELECT * FROM master_get_table_ddl_events('checkperm'); GRANT TRUNCATE ON public.checkperm TO full_access WITH GRANT OPTION GRANT REFERENCES ON public.checkperm TO full_access WITH GRANT OPTION GRANT TRIGGER ON public.checkperm TO full_access WITH GRANT OPTION + GRANT MAINTAIN ON public.checkperm TO full_access WITH GRANT OPTION GRANT SELECT ON public.checkperm TO PUBLIC ALTER TABLE public.checkperm OWNER TO postgres -(18 rows) +(20 rows) -- create table as superuser/postgres CREATE TABLE trivial_postgres (id int); @@ -172,10 +186,10 @@ SELECT create_distributed_table('trivial_full_access', 'id', 'append'); RESET ROLE; SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; - relname | rolname | relacl + relname | rolname | relacl --------------------------------------------------------------------- trivial_full_access | full_access | - trivial_postgres | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} (2 rows) SET citus.shard_replication_factor = 2; -- on all workers... @@ -222,26 +236,26 @@ SELECT master_create_empty_shard('trivial_full_access'); RESET ROLE; \c - - - :worker_1_port SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; - relname | rolname | relacl + relname | rolname | relacl --------------------------------------------------------------------- trivial_full_access_109081 | full_access | trivial_full_access_109083 | full_access | trivial_full_access_109085 | full_access | - trivial_postgres_109080 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} - trivial_postgres_109082 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} - trivial_postgres_109084 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109080 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} + trivial_postgres_109082 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} + trivial_postgres_109084 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} (6 rows) \c - - - :worker_2_port SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; - relname | rolname | relacl + relname | rolname | relacl --------------------------------------------------------------------- trivial_full_access_109081 | full_access | trivial_full_access_109083 | full_access | trivial_full_access_109085 | full_access | - trivial_postgres_109080 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} - trivial_postgres_109082 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} - trivial_postgres_109084 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109080 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} + trivial_postgres_109082 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} + trivial_postgres_109084 | postgres | {postgres=arwdDxtm/postgres,full_access=arwdDxtm/postgres} (6 rows) \c - - - :master_port diff --git a/src/test/regress/expected/multi_multiuser_master_protocol_0.out b/src/test/regress/expected/multi_multiuser_master_protocol_0.out new file mode 100644 index 000000000..f8422ad67 --- /dev/null +++ b/src/test/regress/expected/multi_multiuser_master_protocol_0.out @@ -0,0 +1,498 @@ +-- +-- MULTI_MULTIUSER_MASTER_PROTOCOL +-- +-- Test multi_multiuser_master_protocol has an alternative output file because +-- PG17's support for the MAINTAIN privilege: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337 +-- means that calls of master_get_table_ddl_events() can show MAINTAIN and the +-- pg_class.relacl column may have 'm' for MAINTAIN +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 109079; +-- Tests that check the metadata returned by the master node. At the +-- same time ensure that any user, not just a superuser, can call +-- these. Note that, for now at least, any user can call these. That's +-- OK-ish, since the schema is visible from the catalogs anyway, and +-- exhausting shardids doesn't seem like a super viable attack path. +SET ROLE no_access; +SELECT * FROM master_get_table_ddl_events('lineitem') order by 1; + master_get_table_ddl_events +--------------------------------------------------------------------- + ALTER TABLE public.lineitem ADD CONSTRAINT lineitem_pkey PRIMARY KEY (l_orderkey, l_linenumber) + ALTER TABLE public.lineitem OWNER TO postgres + CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate) + CREATE TABLE public.lineitem (l_orderkey bigint NOT NULL, l_partkey integer NOT NULL, l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity numeric(15,2) NOT NULL, l_extendedprice numeric(15,2) NOT NULL, l_discount numeric(15,2) NOT NULL, l_tax numeric(15,2) NOT NULL, l_returnflag character(1) NOT NULL, l_linestatus character(1) NOT NULL, l_shipdate date NOT NULL, l_commitdate date NOT NULL, l_receiptdate date NOT NULL, l_shipinstruct character(25) NOT NULL, l_shipmode character(10) NOT NULL, l_comment character varying(44) NOT NULL) USING heap + GRANT DELETE ON public.lineitem TO full_access + GRANT DELETE ON public.lineitem TO postgres + GRANT INSERT ON public.lineitem TO full_access + GRANT INSERT ON public.lineitem TO postgres + GRANT REFERENCES ON public.lineitem TO full_access + GRANT REFERENCES ON public.lineitem TO postgres + GRANT SELECT ON public.lineitem TO full_access + GRANT SELECT ON public.lineitem TO postgres + GRANT SELECT ON public.lineitem TO read_access + GRANT TRIGGER ON public.lineitem TO full_access + GRANT TRIGGER ON public.lineitem TO postgres + GRANT TRUNCATE ON public.lineitem TO full_access + GRANT TRUNCATE ON public.lineitem TO postgres + GRANT UPDATE ON public.lineitem TO full_access + GRANT UPDATE ON public.lineitem TO postgres + REVOKE ALL ON public.lineitem FROM PUBLIC +(20 rows) + +SELECT * FROM master_get_new_shardid(); + master_get_new_shardid +--------------------------------------------------------------------- + 109079 +(1 row) + +SELECT * FROM master_get_active_worker_nodes(); + node_name | node_port +--------------------------------------------------------------------- + localhost | 57638 + localhost | 57637 +(2 rows) + +RESET ROLE; +-- ensure GRANT/REVOKE's do something sane for creating shards of +CREATE TABLE checkperm(key int); +SELECT create_distributed_table('checkperm', 'key', 'append'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT * FROM master_get_table_ddl_events('checkperm'); + master_get_table_ddl_events +--------------------------------------------------------------------- + CREATE TABLE public.checkperm (key integer) USING heap + ALTER TABLE public.checkperm OWNER TO postgres +(2 rows) + +REVOKE ALL ON checkperm FROM PUBLIC; +SELECT * FROM master_get_table_ddl_events('checkperm'); + master_get_table_ddl_events +--------------------------------------------------------------------- + CREATE TABLE public.checkperm (key integer) USING heap + REVOKE ALL ON public.checkperm FROM PUBLIC + GRANT INSERT ON public.checkperm TO postgres + GRANT SELECT ON public.checkperm TO postgres + GRANT UPDATE ON public.checkperm TO postgres + GRANT DELETE ON public.checkperm TO postgres + GRANT TRUNCATE ON public.checkperm TO postgres + GRANT REFERENCES ON public.checkperm TO postgres + GRANT TRIGGER ON public.checkperm TO postgres + ALTER TABLE public.checkperm OWNER TO postgres +(10 rows) + +GRANT SELECT ON checkperm TO read_access; +GRANT ALL ON checkperm TO full_access; +SELECT * FROM master_get_table_ddl_events('checkperm'); + master_get_table_ddl_events +--------------------------------------------------------------------- + CREATE TABLE public.checkperm (key integer) USING heap + REVOKE ALL ON public.checkperm FROM PUBLIC + GRANT INSERT ON public.checkperm TO postgres + GRANT SELECT ON public.checkperm TO postgres + GRANT UPDATE ON public.checkperm TO postgres + GRANT DELETE ON public.checkperm TO postgres + GRANT TRUNCATE ON public.checkperm TO postgres + GRANT REFERENCES ON public.checkperm TO postgres + GRANT TRIGGER ON public.checkperm TO postgres + GRANT SELECT ON public.checkperm TO read_access + GRANT INSERT ON public.checkperm TO full_access + GRANT SELECT ON public.checkperm TO full_access + GRANT UPDATE ON public.checkperm TO full_access + GRANT DELETE ON public.checkperm TO full_access + GRANT TRUNCATE ON public.checkperm TO full_access + GRANT REFERENCES ON public.checkperm TO full_access + GRANT TRIGGER ON public.checkperm TO full_access + ALTER TABLE public.checkperm OWNER TO postgres +(18 rows) + +REVOKE ALL ON checkperm FROM read_access; +GRANT SELECT ON checkperm TO PUBLIC; +SELECT * FROM master_get_table_ddl_events('checkperm'); + master_get_table_ddl_events +--------------------------------------------------------------------- + CREATE TABLE public.checkperm (key integer) USING heap + REVOKE ALL ON public.checkperm FROM PUBLIC + GRANT INSERT ON public.checkperm TO postgres + GRANT SELECT ON public.checkperm TO postgres + GRANT UPDATE ON public.checkperm TO postgres + GRANT DELETE ON public.checkperm TO postgres + GRANT TRUNCATE ON public.checkperm TO postgres + GRANT REFERENCES ON public.checkperm TO postgres + GRANT TRIGGER ON public.checkperm TO postgres + GRANT INSERT ON public.checkperm TO full_access + GRANT SELECT ON public.checkperm TO full_access + GRANT UPDATE ON public.checkperm TO full_access + GRANT DELETE ON public.checkperm TO full_access + GRANT TRUNCATE ON public.checkperm TO full_access + GRANT REFERENCES ON public.checkperm TO full_access + GRANT TRIGGER ON public.checkperm TO full_access + GRANT SELECT ON public.checkperm TO PUBLIC + ALTER TABLE public.checkperm OWNER TO postgres +(18 rows) + +GRANT ALL ON checkperm TO full_access WITH GRANT OPTION; +SELECT * FROM master_get_table_ddl_events('checkperm'); + master_get_table_ddl_events +--------------------------------------------------------------------- + CREATE TABLE public.checkperm (key integer) USING heap + REVOKE ALL ON public.checkperm FROM PUBLIC + GRANT INSERT ON public.checkperm TO postgres + GRANT SELECT ON public.checkperm TO postgres + GRANT UPDATE ON public.checkperm TO postgres + GRANT DELETE ON public.checkperm TO postgres + GRANT TRUNCATE ON public.checkperm TO postgres + GRANT REFERENCES ON public.checkperm TO postgres + GRANT TRIGGER ON public.checkperm TO postgres + GRANT INSERT ON public.checkperm TO full_access WITH GRANT OPTION + GRANT SELECT ON public.checkperm TO full_access WITH GRANT OPTION + GRANT UPDATE ON public.checkperm TO full_access WITH GRANT OPTION + GRANT DELETE ON public.checkperm TO full_access WITH GRANT OPTION + GRANT TRUNCATE ON public.checkperm TO full_access WITH GRANT OPTION + GRANT REFERENCES ON public.checkperm TO full_access WITH GRANT OPTION + GRANT TRIGGER ON public.checkperm TO full_access WITH GRANT OPTION + GRANT SELECT ON public.checkperm TO PUBLIC + ALTER TABLE public.checkperm OWNER TO postgres +(18 rows) + +-- create table as superuser/postgres +CREATE TABLE trivial_postgres (id int); +SELECT create_distributed_table('trivial_postgres', 'id', 'append'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +GRANT ALL ON trivial_postgres TO full_access; +GRANT CREATE ON SCHEMA public TO full_access; +SET ROLE full_access; +CREATE TABLE trivial_full_access (id int); +SELECT create_distributed_table('trivial_full_access', 'id', 'append'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +RESET ROLE; +SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; + relname | rolname | relacl +--------------------------------------------------------------------- + trivial_full_access | full_access | + trivial_postgres | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} +(2 rows) + +SET citus.shard_replication_factor = 2; -- on all workers... +-- create shards as each user, verify ownership +SELECT master_create_empty_shard('trivial_postgres'); + master_create_empty_shard +--------------------------------------------------------------------- + 109080 +(1 row) + +SELECT master_create_empty_shard('trivial_full_access'); + master_create_empty_shard +--------------------------------------------------------------------- + 109081 +(1 row) + +SET ROLE full_access; +SELECT master_create_empty_shard('trivial_postgres'); + master_create_empty_shard +--------------------------------------------------------------------- + 109082 +(1 row) + +SELECT master_create_empty_shard('trivial_full_access'); + master_create_empty_shard +--------------------------------------------------------------------- + 109083 +(1 row) + +RESET ROLE; +SET ROLE full_access; +SELECT master_create_empty_shard('trivial_postgres'); + master_create_empty_shard +--------------------------------------------------------------------- + 109084 +(1 row) + +SELECT master_create_empty_shard('trivial_full_access'); + master_create_empty_shard +--------------------------------------------------------------------- + 109085 +(1 row) + +RESET ROLE; +\c - - - :worker_1_port +SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; + relname | rolname | relacl +--------------------------------------------------------------------- + trivial_full_access_109081 | full_access | + trivial_full_access_109083 | full_access | + trivial_full_access_109085 | full_access | + trivial_postgres_109080 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109082 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109084 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} +(6 rows) + +\c - - - :worker_2_port +SELECT relname, rolname, relacl FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) WHERE relname LIKE 'trivial%' ORDER BY relname; + relname | rolname | relacl +--------------------------------------------------------------------- + trivial_full_access_109081 | full_access | + trivial_full_access_109083 | full_access | + trivial_full_access_109085 | full_access | + trivial_postgres_109080 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109082 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} + trivial_postgres_109084 | postgres | {postgres=arwdDxt/postgres,full_access=arwdDxt/postgres} +(6 rows) + +\c - - - :master_port +-- ensure COPY into append tables works +CREATE TABLE stage_postgres(id) AS SELECT 2; +GRANT ALL ON stage_postgres TO full_access; +SET ROLE full_access; +CREATE TABLE stage_full_access(id) AS SELECT 1; +RESET ROLE; +SELECT master_create_empty_shard('trivial_postgres') AS shardid \gset +COPY trivial_postgres FROM STDIN WITH (append_to_shard :shardid); +SELECT master_create_empty_shard('trivial_full_access') AS shardid \gset +COPY trivial_full_access FROM STDIN WITH (append_to_shard :shardid); +SET ROLE full_access; +SELECT master_create_empty_shard('trivial_postgres') AS shardid \gset +COPY trivial_postgres FROM STDIN WITH (append_to_shard :shardid); +SELECT master_create_empty_shard('trivial_full_access') AS shardid \gset +COPY trivial_full_access FROM STDIN WITH (append_to_shard :shardid); +RESET ROLE; +SELECT * FROM trivial_postgres ORDER BY id; + id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +SELECT * FROM trivial_full_access ORDER BY id; + id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +SET ROLE full_access; +SELECT * FROM trivial_postgres ORDER BY id; + id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +SELECT * FROM trivial_full_access ORDER BY id; + id +--------------------------------------------------------------------- + 1 + 1 + 2 + 2 +(4 rows) + +RESET ROLE; +-- verify column level grants are not supported +GRANT UPDATE (id) ON trivial_postgres TO read_access; +ERROR: grant/revoke on column list is currently unsupported +DROP TABLE trivial_full_access; +DROP TABLE trivial_postgres; +DROP TABLE stage_full_access; +DROP TABLE stage_postgres; +-- test GRANT/REVOKE on all tables in schema +CREATE SCHEMA multiuser_schema; +CREATE TABLE multiuser_schema.hash_table(a int, b int); +CREATE TABLE multiuser_schema.reference_table(a int, b int); +SET citus.shard_replication_factor TO 1; +SELECT create_distributed_table('multiuser_schema.hash_table', 'a', colocate_with => 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- usage right must be granted to user +GRANT USAGE ON SCHEMA multiuser_schema TO read_access; +-- verify test user (read_access) does not have select privilege on both tables +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109090 | t | f + localhost | 57637 | 109092 | t | f + localhost | 57638 | 109091 | t | f + localhost | 57638 | 109093 | t | f +(4 rows) + +-- grant select +GRANT SELECT ON ALL TABLES IN SCHEMA multiuser_schema TO read_access; +-- verify select is granted +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109090 | t | t + localhost | 57637 | 109092 | t | t + localhost | 57638 | 109091 | t | t + localhost | 57638 | 109093 | t | t +(4 rows) + +-- distribute the second table +SELECT create_reference_table('multiuser_schema.reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- verify select is also granted +SELECT * FROM run_command_on_placements('multiuser_schema.reference_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57636 | 109094 | t | t + localhost | 57637 | 109094 | t | t + localhost | 57638 | 109094 | t | t +(3 rows) + +-- create another table in the schema, verify select is not granted +CREATE TABLE multiuser_schema.another_table(a int, b int); +SELECT create_distributed_table('multiuser_schema.another_table', 'a', colocate_with => 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT * FROM run_command_on_placements('multiuser_schema.another_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109095 | t | f + localhost | 57637 | 109097 | t | f + localhost | 57638 | 109096 | t | f + localhost | 57638 | 109098 | t | f +(4 rows) + +-- grant select again, verify it is granted +GRANT SELECT ON ALL TABLES IN SCHEMA multiuser_schema TO read_access; +SELECT * FROM run_command_on_placements('multiuser_schema.another_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109095 | t | t + localhost | 57637 | 109097 | t | t + localhost | 57638 | 109096 | t | t + localhost | 57638 | 109098 | t | t +(4 rows) + +-- verify isolate tenant carries grants +SELECT isolate_tenant_to_new_shard('multiuser_schema.hash_table', 5, shard_transfer_mode => 'block_writes'); + isolate_tenant_to_new_shard +--------------------------------------------------------------------- + 109100 +(1 row) + +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109092 | t | t + localhost | 57637 | 109099 | t | t + localhost | 57637 | 109100 | t | t + localhost | 57637 | 109101 | t | t + localhost | 57638 | 109091 | t | t + localhost | 57638 | 109093 | t | t +(6 rows) + +-- revoke select +REVOKE SELECT ON ALL TABLES IN SCHEMA multiuser_schema FROM read_access; +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109092 | t | f + localhost | 57637 | 109099 | t | f + localhost | 57637 | 109100 | t | f + localhost | 57637 | 109101 | t | f + localhost | 57638 | 109091 | t | f + localhost | 57638 | 109093 | t | f +(6 rows) + +-- test multi-schema grants +CREATE SCHEMA multiuser_second_schema; +CREATE TABLE multiuser_second_schema.hash_table(a int, b int); +SELECT create_distributed_table('multiuser_second_schema.hash_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +GRANT ALL ON ALL TABLES IN SCHEMA multiuser_schema, multiuser_second_schema TO read_access; +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109092 | t | t + localhost | 57637 | 109099 | t | t + localhost | 57637 | 109100 | t | t + localhost | 57637 | 109101 | t | t + localhost | 57638 | 109091 | t | t + localhost | 57638 | 109093 | t | t +(6 rows) + +SELECT * FROM run_command_on_placements('multiuser_second_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109102 | t | t + localhost | 57637 | 109103 | t | t + localhost | 57637 | 109104 | t | t + localhost | 57637 | 109106 | t | t + localhost | 57638 | 109105 | t | t + localhost | 57638 | 109107 | t | t +(6 rows) + +-- revoke from multiple schemas, verify result +REVOKE SELECT ON ALL TABLES IN SCHEMA multiuser_schema, multiuser_second_schema FROM read_access; +SELECT * FROM run_command_on_placements('multiuser_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109092 | t | f + localhost | 57637 | 109099 | t | f + localhost | 57637 | 109100 | t | f + localhost | 57637 | 109101 | t | f + localhost | 57638 | 109091 | t | f + localhost | 57638 | 109093 | t | f +(6 rows) + +SELECT * FROM run_command_on_placements('multiuser_second_schema.hash_table', $$ select has_table_privilege('read_access', '%s', 'select') $$) +ORDER BY nodename, nodeport, shardid; + nodename | nodeport | shardid | success | result +--------------------------------------------------------------------- + localhost | 57637 | 109102 | t | f + localhost | 57637 | 109103 | t | f + localhost | 57637 | 109104 | t | f + localhost | 57637 | 109106 | t | f + localhost | 57638 | 109105 | t | f + localhost | 57638 | 109107 | t | f +(6 rows) + +DROP SCHEMA multiuser_schema CASCADE; +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table multiuser_schema.hash_table +drop cascades to table multiuser_schema.reference_table +drop cascades to table multiuser_schema.reference_table_109094 +drop cascades to table multiuser_schema.another_table +DROP SCHEMA multiuser_second_schema CASCADE; +NOTICE: drop cascades to table multiuser_second_schema.hash_table diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 563808787..aa4ccb64c 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -338,9 +338,6 @@ DEBUG: Router planner cannot handle multi-shard select queries RESET client_min_messages; RESET search_path; -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; DROP SCHEMA pg17_corr_subq_folding CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table pg17_corr_subq_folding.test @@ -353,19 +350,19 @@ drop cascades to table pg17_corr_subq_folding.events -- PG17-specific tests go here. -- CREATE SCHEMA pg17; -SET search_path TO pg17; +SET search_path to pg17; -- Test specifying access method on partitioned tables. PG17 feature, added by: -- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a229 -- The following tests were failing tests in tableam but will pass on PG >= 17. -- There is some set-up duplication of tableam, and this test can be returned -- to tableam when 17 is the minimum supported PG version. SELECT public.run_command_on_coordinator_and_workers($Q$ - SET citus.enable_ddl_propagation TO off; - CREATE FUNCTION fake_am_handler(internal) - RETURNS table_am_handler - AS 'citus' - LANGUAGE C; - CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; + SET citus.enable_ddl_propagation TO off; + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; $Q$); run_command_on_coordinator_and_workers --------------------------------------------------------------------- @@ -379,9 +376,9 @@ CREATE TABLE test_partitioned(id int, p int, val int) PARTITION BY RANGE (p) USING fake_am; -- Test that children inherit access method from parent CREATE TABLE test_partitioned_p1 PARTITION OF test_partitioned - FOR VALUES FROM (1) TO (10); + FOR VALUES FROM (1) TO (10); CREATE TABLE test_partitioned_p2 PARTITION OF test_partitioned - FOR VALUES FROM (11) TO (20); + FOR VALUES FROM (11) TO (20); INSERT INTO test_partitioned VALUES (1, 5, -1), (2, 15, -2); WARNING: fake_tuple_insert WARNING: fake_tuple_insert @@ -416,10 +413,93 @@ ORDER BY c.relname; test_partitioned_p2 | fake_am (2 rows) +-- Clean up DROP TABLE test_partitioned; ALTER EXTENSION citus DROP ACCESS METHOD fake_am; +SELECT public.run_command_on_coordinator_and_workers($Q$ + RESET citus.enable_ddl_propagation; +$Q$); + run_command_on_coordinator_and_workers +--------------------------------------------------------------------- + +(1 row) + -- End of testing specifying access method on partitioned tables. +-- MAINTAIN privilege tests +CREATE ROLE regress_maintain; +CREATE ROLE regress_no_maintain; +ALTER ROLE regress_maintain WITH login; +GRANT USAGE ON SCHEMA pg17 TO regress_maintain; +ALTER ROLE regress_no_maintain WITH login; +GRANT USAGE ON SCHEMA pg17 TO regress_no_maintain; +SET citus.shard_count TO 1; -- For consistent remote command logging +CREATE TABLE dist_test(a int, b int); +SELECT create_distributed_table('dist_test', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO dist_test SELECT i % 10, i FROM generate_series(1, 100) t(i); +SET citus.log_remote_commands TO on; +SET citus.grep_remote_commands = '%maintain%'; +GRANT MAINTAIN ON dist_test TO regress_maintain; +NOTICE: issuing GRANT maintain ON dist_test TO regress_maintain +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing GRANT maintain ON dist_test TO regress_maintain +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240023, 'pg17', 'GRANT maintain ON dist_test TO regress_maintain') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +RESET citus.grep_remote_commands; +SET ROLE regress_no_maintain; +-- Current role does not have MAINTAIN privileges on dist_test +ANALYZE dist_test; +WARNING: permission denied to analyze "dist_test", skipping it +NOTICE: issuing ANALYZE pg17.dist_test_20240023 +DETAIL: on server regress_no_maintain@localhost:xxxxx connectionId: xxxxxxx +VACUUM dist_test; +WARNING: permission denied to vacuum "dist_test", skipping it +NOTICE: issuing VACUUM pg17.dist_test_20240023 +DETAIL: on server regress_no_maintain@localhost:xxxxx connectionId: xxxxxxx +SET ROLE regress_maintain; +-- Current role has MAINTAIN privileges on dist_test +ANALYZE dist_test; +NOTICE: issuing ANALYZE pg17.dist_test_20240023 +DETAIL: on server regress_maintain@localhost:xxxxx connectionId: xxxxxxx +VACUUM dist_test; +NOTICE: issuing VACUUM pg17.dist_test_20240023 +DETAIL: on server regress_maintain@localhost:xxxxx connectionId: xxxxxxx +-- Take away regress_maintain's MAINTAIN privileges on dist_test +RESET ROLE; +SET citus.grep_remote_commands = '%maintain%'; +REVOKE MAINTAIN ON dist_test FROM regress_maintain; +NOTICE: issuing REVOKE maintain ON dist_test FROM regress_maintain +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REVOKE maintain ON dist_test FROM regress_maintain +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240023, 'pg17', 'REVOKE maintain ON dist_test FROM regress_maintain') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +RESET citus.grep_remote_commands; +SET ROLE regress_maintain; +-- Current role does not have MAINTAIN privileges on dist_test +ANALYZE dist_test; +WARNING: permission denied to analyze "dist_test", skipping it +NOTICE: issuing ANALYZE pg17.dist_test_20240023 +DETAIL: on server regress_maintain@localhost:xxxxx connectionId: xxxxxxx +VACUUM dist_test; +WARNING: permission denied to vacuum "dist_test", skipping it +NOTICE: issuing VACUUM pg17.dist_test_20240023 +DETAIL: on server regress_maintain@localhost:xxxxx connectionId: xxxxxxx +RESET ROLE; +-- End of MAINTAIN privilege tests +RESET citus.log_remote_commands; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; DROP SCHEMA pg17 CASCADE; -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to function fake_am_handler(internal) drop cascades to access method fake_am +drop cascades to table dist_test +DROP ROLE regress_maintain; +DROP ROLE regress_no_maintain; diff --git a/src/test/regress/expected/pg17_0.out b/src/test/regress/expected/pg17_0.out index 66dba2c29..09db03e4c 100644 --- a/src/test/regress/expected/pg17_0.out +++ b/src/test/regress/expected/pg17_0.out @@ -282,9 +282,6 @@ DEBUG: Router planner cannot handle multi-shard select queries RESET client_min_messages; RESET search_path; -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; DROP SCHEMA pg17_corr_subq_folding CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table pg17_corr_subq_folding.test diff --git a/src/test/regress/sql/multi_multiuser_master_protocol.sql b/src/test/regress/sql/multi_multiuser_master_protocol.sql index 535500004..e13605fff 100644 --- a/src/test/regress/sql/multi_multiuser_master_protocol.sql +++ b/src/test/regress/sql/multi_multiuser_master_protocol.sql @@ -2,6 +2,12 @@ -- MULTI_MULTIUSER_MASTER_PROTOCOL -- +-- Test multi_multiuser_master_protocol has an alternative output file because +-- PG17's support for the MAINTAIN privilege: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337 +-- means that calls of master_get_table_ddl_events() can show MAINTAIN and the +-- pg_class.relacl column may have 'm' for MAINTAIN + ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 109079; -- Tests that check the metadata returned by the master node. At the diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index e297a0c58..3c9a2541c 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -168,9 +168,6 @@ GROUP BY dept; RESET client_min_messages; RESET search_path; -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; DROP SCHEMA pg17_corr_subq_folding CASCADE; \if :server_version_ge_17 @@ -181,7 +178,7 @@ DROP SCHEMA pg17_corr_subq_folding CASCADE; -- PG17-specific tests go here. -- CREATE SCHEMA pg17; -SET search_path TO pg17; +SET search_path to pg17; -- Test specifying access method on partitioned tables. PG17 feature, added by: -- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a229 @@ -190,12 +187,12 @@ SET search_path TO pg17; -- to tableam when 17 is the minimum supported PG version. SELECT public.run_command_on_coordinator_and_workers($Q$ - SET citus.enable_ddl_propagation TO off; - CREATE FUNCTION fake_am_handler(internal) - RETURNS table_am_handler - AS 'citus' - LANGUAGE C; - CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; + SET citus.enable_ddl_propagation TO off; + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; $Q$); -- Since Citus assumes access methods are part of the extension, make fake_am @@ -207,9 +204,9 @@ PARTITION BY RANGE (p) USING fake_am; -- Test that children inherit access method from parent CREATE TABLE test_partitioned_p1 PARTITION OF test_partitioned - FOR VALUES FROM (1) TO (10); + FOR VALUES FROM (1) TO (10); CREATE TABLE test_partitioned_p2 PARTITION OF test_partitioned - FOR VALUES FROM (11) TO (20); + FOR VALUES FROM (11) TO (20); INSERT INTO test_partitioned VALUES (1, 5, -1), (2, 15, -2); INSERT INTO test_partitioned VALUES (3, 6, -6), (4, 16, -4); @@ -222,9 +219,66 @@ SELECT c.relname, am.amname FROM pg_class c, pg_am am WHERE c.relam = am.oid AND c.oid IN ('test_partitioned_p1'::regclass, 'test_partitioned_p2'::regclass) ORDER BY c.relname; +-- Clean up DROP TABLE test_partitioned; ALTER EXTENSION citus DROP ACCESS METHOD fake_am; +SELECT public.run_command_on_coordinator_and_workers($Q$ + RESET citus.enable_ddl_propagation; +$Q$); -- End of testing specifying access method on partitioned tables. +-- MAINTAIN privilege tests + +CREATE ROLE regress_maintain; +CREATE ROLE regress_no_maintain; + +ALTER ROLE regress_maintain WITH login; +GRANT USAGE ON SCHEMA pg17 TO regress_maintain; +ALTER ROLE regress_no_maintain WITH login; +GRANT USAGE ON SCHEMA pg17 TO regress_no_maintain; + +SET citus.shard_count TO 1; -- For consistent remote command logging +CREATE TABLE dist_test(a int, b int); +SELECT create_distributed_table('dist_test', 'a'); +INSERT INTO dist_test SELECT i % 10, i FROM generate_series(1, 100) t(i); + +SET citus.log_remote_commands TO on; + +SET citus.grep_remote_commands = '%maintain%'; +GRANT MAINTAIN ON dist_test TO regress_maintain; +RESET citus.grep_remote_commands; + +SET ROLE regress_no_maintain; +-- Current role does not have MAINTAIN privileges on dist_test +ANALYZE dist_test; +VACUUM dist_test; + +SET ROLE regress_maintain; +-- Current role has MAINTAIN privileges on dist_test +ANALYZE dist_test; +VACUUM dist_test; + +-- Take away regress_maintain's MAINTAIN privileges on dist_test +RESET ROLE; +SET citus.grep_remote_commands = '%maintain%'; +REVOKE MAINTAIN ON dist_test FROM regress_maintain; +RESET citus.grep_remote_commands; + +SET ROLE regress_maintain; +-- Current role does not have MAINTAIN privileges on dist_test +ANALYZE dist_test; +VACUUM dist_test; + +RESET ROLE; + +-- End of MAINTAIN privilege tests + +RESET citus.log_remote_commands; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; + DROP SCHEMA pg17 CASCADE; +DROP ROLE regress_maintain; +DROP ROLE regress_no_maintain; From 89e0b53644cd46265f8064cc6b1b20bfd9889ae0 Mon Sep 17 00:00:00 2001 From: Colm Date: Tue, 17 Dec 2024 21:42:15 +0000 Subject: [PATCH 081/155] PG17 compatibility: fix plan diffs in multi_explain (#7780) Regress test `multi_explain` has two queries that have a different query plan with PG17. Here is part of the plan diff for the query labelled _Union and left join subquery pushdown_ in `multi_explain.sql` (for the complete diff, search for `multi_explain` [here](https://github.com/citusdata/citus/actions/runs/12158205599/attempts/1)): ``` -> Sort Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone, events.event_time - -> Hash Left Join - Hash Cond: (users.composite_id = subquery_2.composite_id) - -> HashAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Nested Loop Left Join + Join Filter: (users.composite_id = subquery_2.composite_id) + -> Unique + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time -> Append ``` The change is the same in both queries; a hash left join with subquery_1 on the outer and subquery_2 on the inner side of the join is now a nested loop left join with subquery_1 on the outer and subquery_2 on the inner; additionally, the chosen method of uniquifying the UNION in subquery_1 has changed from hashed grouping to sort followed by unique, as shown in the diff above. The PG17 commit that caused this plan change is likely _[Fix MergeAppend to more accurately compute the number of rows that need to be sorted](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=9d1a5354f)_ because it impacts the estimated rows counts of UNION paths. Comparing a costed plan of the query between PG16 and PG17 I noticed that with PG16 the rows estimate for the UNION in subquery_1 is 4, whereas with PG17 the rows estimate is 2. A lower rows estimate in the outer side of the join may result in nested loop looking cheaper than hash join for the left outer join, hence the plan change in the two queries where there is a UNION on the outer side of a left outer join. The proposed fix achieves a consistent plan across all supported postgres versions by temporarily disabling nested loop join and sort for the two impacted queries; the postgres optimizer selects hash join for the outer left join and hashed aggregation for the UNION operation. I investigated tweaking the queries, but was not able to arrive at a consistent plan, and I believe the SQL operator (e.g. join, group by, union) implementations are orthogonal to the intent of the test, so this should be a satisfactory solution, particularly as it avoids introducing a second alternative output file for `multi_explain`. --- src/test/regress/expected/multi_explain.out | 154 ++++++++++-------- src/test/regress/expected/multi_explain_0.out | 154 ++++++++++-------- src/test/regress/sql/multi_explain.sql | 8 + 3 files changed, 172 insertions(+), 144 deletions(-) diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 906add24c..bfcf29c4d 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -671,6 +671,15 @@ Aggregate -> Hash -> Seq Scan on events_1400285 events Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); +t +t +SELECT success FROM run_command_on_workers('alter system set enable_sort to off'); +t +t +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); +t +t -- Union and left join subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -741,41 +750,38 @@ HashAggregate Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate - Group Key: subquery_top.hasdone - -> Sort - Sort Key: subquery_top.hasdone - -> Subquery Scan on subquery_top - -> GroupAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone - -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone, events.event_time - -> Hash Left Join - Hash Cond: (users.composite_id = subquery_2.composite_id) - -> HashAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time - -> Append - -> Hash Join - Hash Cond: (users.composite_id = events.composite_id) - -> Seq Scan on users_1400289 users - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events - Filter: ((event_type)::text = 'click'::text) - -> Hash Join - Hash Cond: (users_1.composite_id = events_1.composite_id) - -> Seq Scan on users_1400289 users_1 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events_1 - Filter: ((event_type)::text = 'submit'::text) - -> Hash - -> Subquery Scan on subquery_2 - -> Unique - -> Sort - Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) - -> Seq Scan on events_1400285 events_2 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + -> HashAggregate + Group Key: COALESCE(subquery_2.hasdone, 'Has not done paying'::text) + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone, events.event_time + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 + -> Unique + -> Sort + Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) -- Union, left join and having subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -856,44 +862,48 @@ Sort Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate - Group Key: subquery_top.count_pay - -> Sort - Sort Key: subquery_top.count_pay - -> Subquery Scan on subquery_top - -> GroupAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay - Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) - -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay, events.event_time - -> Hash Left Join - Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: COALESCE(subquery_2.count_pay, '0'::bigint) + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay, events.event_time + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 -> HashAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time - -> Append - -> Hash Join - Hash Cond: (users.composite_id = events.composite_id) - -> Seq Scan on users_1400289 users - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events - Filter: ((event_type)::text = 'click'::text) - -> Hash Join - Hash Cond: (users_1.composite_id = events_1.composite_id) - -> Seq Scan on users_1400289 users_1 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events_1 - Filter: ((event_type)::text = 'submit'::text) - -> Hash - -> Subquery Scan on subquery_2 - -> GroupAggregate - Group Key: events_2.composite_id - Filter: (count(*) > 2) - -> Sort - Sort Key: events_2.composite_id - -> Seq Scan on events_1400285 events_2 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + Group Key: events_2.composite_id + Filter: (count(*) > 2) + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); +t +t +SELECT success FROM run_command_on_workers('alter system reset enable_sort'); +t +t +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); +t +t -- Lateral join subquery pushdown -- set subquery_pushdown due to limit in the query SET citus.subquery_pushdown to ON; diff --git a/src/test/regress/expected/multi_explain_0.out b/src/test/regress/expected/multi_explain_0.out index 5ba5e056f..4d3acd14d 100644 --- a/src/test/regress/expected/multi_explain_0.out +++ b/src/test/regress/expected/multi_explain_0.out @@ -671,6 +671,15 @@ Aggregate -> Hash -> Seq Scan on events_1400285 events Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); +t +t +SELECT success FROM run_command_on_workers('alter system set enable_sort to off'); +t +t +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); +t +t -- Union and left join subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -741,41 +750,38 @@ HashAggregate Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate - Group Key: subquery_top.hasdone - -> Sort - Sort Key: subquery_top.hasdone - -> Subquery Scan on subquery_top - -> GroupAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone - -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone - -> Hash Left Join - Hash Cond: (users.composite_id = subquery_2.composite_id) - -> HashAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time - -> Append - -> Hash Join - Hash Cond: (users.composite_id = events.composite_id) - -> Seq Scan on users_1400289 users - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events - Filter: ((event_type)::text = 'click'::text) - -> Hash Join - Hash Cond: (users_1.composite_id = events_1.composite_id) - -> Seq Scan on users_1400289 users_1 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events_1 - Filter: ((event_type)::text = 'submit'::text) - -> Hash - -> Subquery Scan on subquery_2 - -> Unique - -> Sort - Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) - -> Seq Scan on events_1400285 events_2 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + -> HashAggregate + Group Key: COALESCE(subquery_2.hasdone, 'Has not done paying'::text) + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 + -> Unique + -> Sort + Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) -- Union, left join and having subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -856,44 +862,48 @@ Sort Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate - Group Key: subquery_top.count_pay - -> Sort - Sort Key: subquery_top.count_pay - -> Subquery Scan on subquery_top - -> GroupAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay - Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) - -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay - -> Hash Left Join - Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: COALESCE(subquery_2.count_pay, '0'::bigint) + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 -> HashAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time - -> Append - -> Hash Join - Hash Cond: (users.composite_id = events.composite_id) - -> Seq Scan on users_1400289 users - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events - Filter: ((event_type)::text = 'click'::text) - -> Hash Join - Hash Cond: (users_1.composite_id = events_1.composite_id) - -> Seq Scan on users_1400289 users_1 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) - -> Hash - -> Seq Scan on events_1400285 events_1 - Filter: ((event_type)::text = 'submit'::text) - -> Hash - -> Subquery Scan on subquery_2 - -> GroupAggregate - Group Key: events_2.composite_id - Filter: (count(*) > 2) - -> Sort - Sort Key: events_2.composite_id - -> Seq Scan on events_1400285 events_2 - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + Group Key: events_2.composite_id + Filter: (count(*) > 2) + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); +t +t +SELECT success FROM run_command_on_workers('alter system reset enable_sort'); +t +t +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); +t +t -- Lateral join subquery pushdown -- set subquery_pushdown due to limit in the query SET citus.subquery_pushdown to ON; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index 4fc16fbd8..65ca6f5da 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -260,6 +260,10 @@ FROM tenant_id, user_id) AS subquery; +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); +SELECT success FROM run_command_on_workers('alter system set enable_sort to off'); +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + -- Union and left join subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -396,6 +400,10 @@ GROUP BY ORDER BY count_pay; +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); +SELECT success FROM run_command_on_workers('alter system reset enable_sort'); +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + -- Lateral join subquery pushdown -- set subquery_pushdown due to limit in the query SET citus.subquery_pushdown to ON; From e91ee245ac0dc6a5b44e4bab0392d925492e3e94 Mon Sep 17 00:00:00 2001 From: Colm Date: Wed, 18 Dec 2024 13:18:53 +0000 Subject: [PATCH 082/155] PG17 compatibility: account for identity columns in partitioned tables. (#7785) PG17 added support for identity columns in partitioned tables: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=699586315 A consequence is that a table with an identity column cannot be attached as a partition. But Citus on Postgres 17 will generate identity column for the partitions if the parent table has one (or more) identity columns when propagating distributed table DDL to worker nodes, as happens in the `generated_identity` regress test in #7768: ``` CREATE TABLE partitioned_table ( a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), c int ) PARTITION BY RANGE (c); CREATE TABLE partitioned_table_1_50 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); CREATE TABLE partitioned_table_50_500 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); SELECT create_distributed_table('partitioned_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - +ERROR: table "partitioned_table_1_50" being attached contains an identity column "a" +DETAIL: The new partition may not contain an identity column. ``` It is the Citus-generated ATTACH PARTITION statement that errors out, because the Citus-generated CREATE TABLE for the partitions included identity column definitions. The fix is straightforward - when propagating the CREATE TABLE ddl for a partition of a table with an identity column, don't include the identity column(s), they will be inherited on attaching the partition. In Citus on Postgres 16 (or less) partitions do not inherit identity; the partitions in the example would not have any identity columns so it was not an issue previously. --- .../distributed/deparser/citus_ruleutils.c | 12 +- src/test/regress/expected/pg17.out | 462 ++++++++++++++++++ src/test/regress/sql/pg17.sql | 189 +++++++ 3 files changed, 662 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index e5445b3df..d138f8859 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -315,6 +315,7 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults AttrNumber defaultValueIndex = 0; AttrNumber constraintIndex = 0; AttrNumber constraintCount = 0; + bool relIsPartition = false; StringInfoData buffer = { NULL, 0, 0, 0 }; /* @@ -342,6 +343,8 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults } appendStringInfo(&buffer, "TABLE %s (", relationName); + + relIsPartition = relation->rd_rel->relispartition; } else { @@ -392,7 +395,14 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults GetCompressionMethodName(attributeForm->attcompression)); } - if (attributeForm->attidentity && includeIdentityDefaults) + /* + * If this is an identity column include its identity definition in the + * DDL only if its relation is not a partition. If it is a partition, any + * identity is inherited from the parent table by ATTACH PARTITION. This + * is Postgres 17+ behavior (commit 699586315); prior PG versions did not + * support identity columns in partitioned tables. + */ + if (attributeForm->attidentity && includeIdentityDefaults && !relIsPartition) { bool missing_ok = false; Oid seqOid = getIdentitySequence(identitySequenceRelation_compat( diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index aa4ccb64c..35b4aa326 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -492,7 +492,469 @@ NOTICE: issuing VACUUM pg17.dist_test_20240023 DETAIL: on server regress_maintain@localhost:xxxxx connectionId: xxxxxxx RESET ROLE; -- End of MAINTAIN privilege tests +-- Partitions inherit identity column RESET citus.log_remote_commands; +-- PG17 added support for identity columns in partioned tables: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=699586315 +-- In particular, partitions with their own identity columns are not allowed. +-- Citus does not need to propagate identity columns in partitions; the identity +-- is inherited by PG17 behavior, as shown in this test. +CREATE TABLE partitioned_table ( + a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE pt_1 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); +SELECT create_distributed_table('partitioned_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE pt_2 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); +-- (1) The partitioned table has pt_1 and pt_2 as its partitions +\d+ partitioned_table; + Partitioned table "pg17.partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: pt_1 FOR VALUES FROM (1) TO (50), + pt_2 FOR VALUES FROM (50) TO (1000) + +-- (2) The partitions have the same identity column as the parent table; +-- This is PG17 behavior for support for identity in partitioned tables. +\d pt_1; + Table "pg17.pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (1) TO (50) + +\d pt_2; + Table "pg17.pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (50) TO (1000) + +-- Attaching a partition inherits the identity column from the parent table +CREATE TABLE pt_3 (a bigint not null, c int); +ALTER TABLE partitioned_table ATTACH PARTITION pt_3 FOR VALUES FROM (1000) TO (2000); +\d+ partitioned_table; + Partitioned table "pg17.partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: pt_1 FOR VALUES FROM (1) TO (50), + pt_2 FOR VALUES FROM (50) TO (1000), + pt_3 FOR VALUES FROM (1000) TO (2000) + +\d pt_3; + Table "pg17.pt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (1000) TO (2000) + +-- Partition pt_4 has its own identity column, which is not allowed in PG17 +-- and will produce an error on attempting to attach it to the partitioned table +CREATE TABLE pt_4 (a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), c int); +ALTER TABLE partitioned_table ATTACH PARTITION pt_4 FOR VALUES FROM (2000) TO (3000); +ERROR: table "pt_4" being attached contains an identity column "a" +DETAIL: The new partition may not contain an identity column. +\c - - - :worker_1_port +SET search_path TO pg17; +-- Show that DDL for partitioned_table has correctly propagated to the worker node; +-- (1) The partitioned table has pt_1, pt_2 and pt_3 as its partitions +\d+ partitioned_table; + Partitioned table "pg17.partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: pt_1 FOR VALUES FROM (1) TO (50), + pt_2 FOR VALUES FROM (50) TO (1000), + pt_3 FOR VALUES FROM (1000) TO (2000) + +-- (2) The partititions have the same identity column as the parent table +\d pt_1; + Table "pg17.pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (1) TO (50) + +\d pt_2; + Table "pg17.pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (50) TO (1000) + +\d pt_3; + Table "pg17.pt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: partitioned_table FOR VALUES FROM (1000) TO (2000) + +\c - - - :master_port +SET search_path TO pg17; +-- Test detaching a partition with an identity column +ALTER TABLE partitioned_table DETACH PARTITION pt_3; +-- partitioned_table has pt_1, pt_2 as its partitions +-- and pt_3 does not have an identity column +\d+ partitioned_table; + Partitioned table "pg17.partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: pt_1 FOR VALUES FROM (1) TO (50), + pt_2 FOR VALUES FROM (50) TO (1000) + +\d pt_3; + Table "pg17.pt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | + c | integer | | | + +-- Verify that the detach has propagated to the worker node +\c - - - :worker_1_port +SET search_path TO pg17; +\d+ partitioned_table; + Partitioned table "pg17.partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: pt_1 FOR VALUES FROM (1) TO (50), + pt_2 FOR VALUES FROM (50) TO (1000) + +\d pt_3; + Table "pg17.pt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | + c | integer | | | + +\c - - - :master_port +SET search_path TO pg17; +CREATE TABLE alt_test (a int, b date, c int) PARTITION BY RANGE(c); +SELECT create_distributed_table('alt_test', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE alt_test_pt_1 PARTITION OF alt_test FOR VALUES FROM (1) TO (50); +CREATE TABLE alt_test_pt_2 PARTITION OF alt_test FOR VALUES FROM (50) TO (100); +-- Citus does not support adding an identity column for a distributed table (#6738) +-- Attempting to add a column with identity produces an error +ALTER TABLE alt_test ADD COLUMN d bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10); +ERROR: cannot execute ADD COLUMN commands involving identity columns when metadata is synchronized to workers +-- alter table set identity is currently not supported, so adding identity to +-- an existing column generates an error +ALTER TABLE alt_test ALTER COLUMN a SET GENERATED BY DEFAULT SET INCREMENT BY 2 SET START WITH 75 RESTART; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +-- Verify that the identity column was not added, on coordinator and worker nodes +\d+ alt_test; + Partitioned table "pg17.alt_test" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | integer | | | | plain | | + b | date | | | | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: alt_test_pt_1 FOR VALUES FROM (1) TO (50), + alt_test_pt_2 FOR VALUES FROM (50) TO (100) + +\d alt_test_pt_1; + Table "pg17.alt_test_pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | integer | | | + b | date | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (1) TO (50) + +\d alt_test_pt_2; + Table "pg17.alt_test_pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | integer | | | + b | date | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (50) TO (100) + +\c - - - :worker_1_port +SET search_path TO pg17; +\d+ alt_test; + Partitioned table "pg17.alt_test" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | integer | | | | plain | | + b | date | | | | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: alt_test_pt_1 FOR VALUES FROM (1) TO (50), + alt_test_pt_2 FOR VALUES FROM (50) TO (100) + +\d alt_test_pt_1; + Table "pg17.alt_test_pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | integer | | | + b | date | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (1) TO (50) + +\d alt_test_pt_2; + Table "pg17.alt_test_pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | integer | | | + b | date | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (50) TO (100) + +\c - - - :master_port +SET search_path TO pg17; +DROP TABLE alt_test; +CREATE TABLE alt_test (a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b int, + c int) +PARTITION BY RANGE(c); +SELECT create_distributed_table('alt_test', 'b'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE alt_test_pt_1 PARTITION OF alt_test FOR VALUES FROM (1) TO (50); +CREATE TABLE alt_test_pt_2 PARTITION OF alt_test FOR VALUES FROM (50) TO (100); +-- Dropping of the identity property from a column is currently not supported; +-- Attempting to drop identity produces an error +ALTER TABLE alt_test ALTER COLUMN a DROP IDENTITY; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +-- Verify that alt_test still has identity on column a +\d+ alt_test; + Partitioned table "pg17.alt_test" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + b | integer | | | | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: alt_test_pt_1 FOR VALUES FROM (1) TO (50), + alt_test_pt_2 FOR VALUES FROM (50) TO (100) + +\d alt_test_pt_1; + Table "pg17.alt_test_pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (1) TO (50) + +\d alt_test_pt_2; + Table "pg17.alt_test_pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (50) TO (100) + +\c - - - :worker_1_port +SET search_path TO pg17; +\d+ alt_test; + Partitioned table "pg17.alt_test" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + b | integer | | | | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: alt_test_pt_1 FOR VALUES FROM (1) TO (50), + alt_test_pt_2 FOR VALUES FROM (50) TO (100) + +\d alt_test_pt_1; + Table "pg17.alt_test_pt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (1) TO (50) + +\d alt_test_pt_2 + Table "pg17.alt_test_pt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + c | integer | | | +Partition of: alt_test FOR VALUES FROM (50) TO (100) + +\c - - - :master_port +SET search_path TO pg17; +-- Repeat testing of partitions with identity column on a citus local table +CREATE TABLE local_partitioned_table ( + a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE lpt_1 PARTITION OF local_partitioned_table FOR VALUES FROM (1) TO (50); +SELECT citus_add_local_table_to_metadata('local_partitioned_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- Can create tables as partitions and attach tables as partitions to a citus local table: +CREATE TABLE lpt_2 PARTITION OF local_partitioned_table FOR VALUES FROM (50) TO (1000); +CREATE TABLE lpt_3 (a bigint not null, c int); +ALTER TABLE local_partitioned_table ATTACH PARTITION lpt_3 FOR VALUES FROM (1000) TO (2000); +-- The partitions have the same identity column as the parent table, on coordinator and worker nodes +\d+ local_partitioned_table; + Partitioned table "pg17.local_partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: lpt_1 FOR VALUES FROM (1) TO (50), + lpt_2 FOR VALUES FROM (50) TO (1000), + lpt_3 FOR VALUES FROM (1000) TO (2000) + +\d lpt_1; + Table "pg17.lpt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (1) TO (50) + +\d lpt_2; + Table "pg17.lpt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (50) TO (1000) + +\d lpt_3; + Table "pg17.lpt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (1000) TO (2000) + +\c - - - :worker_1_port +SET search_path TO pg17; +\d+ local_partitioned_table; + Partitioned table "pg17.local_partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: lpt_1 FOR VALUES FROM (1) TO (50), + lpt_2 FOR VALUES FROM (50) TO (1000), + lpt_3 FOR VALUES FROM (1000) TO (2000) + +\d lpt_1; + Table "pg17.lpt_1" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (1) TO (50) + +\d lpt_2; + Table "pg17.lpt_2" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (50) TO (1000) + +\d lpt_3; + Table "pg17.lpt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + c | integer | | | +Partition of: local_partitioned_table FOR VALUES FROM (1000) TO (2000) + +\c - - - :master_port +SET search_path TO pg17; +-- Test detaching a partition with an identity column from a citus local table +ALTER TABLE local_partitioned_table DETACH PARTITION lpt_3; +\d+ local_partitioned_table; + Partitioned table "pg17.local_partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: lpt_1 FOR VALUES FROM (1) TO (50), + lpt_2 FOR VALUES FROM (50) TO (1000) + +\d lpt_3; + Table "pg17.lpt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | + c | integer | | | + +\c - - - :worker_1_port +SET search_path TO pg17; +\d+ local_partitioned_table; + Partitioned table "pg17.local_partitioned_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity | plain | | + c | integer | | | | plain | | +Partition key: RANGE (c) +Partitions: lpt_1 FOR VALUES FROM (1) TO (50), + lpt_2 FOR VALUES FROM (50) TO (1000) + +\d lpt_3; + Table "pg17.lpt_3" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | + c | integer | | | + +\c - - - :master_port +SET search_path TO pg17; +DROP TABLE partitioned_table; +DROP TABLE local_partitioned_table; +DROP TABLE lpt_3; +DROP TABLE pt_3; +DROP TABLE pt_4; +DROP TABLE alt_test; +-- End of partition with identity columns testing RESET citus.next_shard_id; RESET citus.shard_count; RESET citus.shard_replication_factor; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 3c9a2541c..4730426eb 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -274,7 +274,196 @@ RESET ROLE; -- End of MAINTAIN privilege tests +-- Partitions inherit identity column + RESET citus.log_remote_commands; + +-- PG17 added support for identity columns in partioned tables: +-- https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=699586315 +-- In particular, partitions with their own identity columns are not allowed. +-- Citus does not need to propagate identity columns in partitions; the identity +-- is inherited by PG17 behavior, as shown in this test. + +CREATE TABLE partitioned_table ( + a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE pt_1 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); + +SELECT create_distributed_table('partitioned_table', 'a'); + +CREATE TABLE pt_2 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); + +-- (1) The partitioned table has pt_1 and pt_2 as its partitions +\d+ partitioned_table; + +-- (2) The partitions have the same identity column as the parent table; +-- This is PG17 behavior for support for identity in partitioned tables. +\d pt_1; +\d pt_2; + +-- Attaching a partition inherits the identity column from the parent table +CREATE TABLE pt_3 (a bigint not null, c int); +ALTER TABLE partitioned_table ATTACH PARTITION pt_3 FOR VALUES FROM (1000) TO (2000); + +\d+ partitioned_table; +\d pt_3; + +-- Partition pt_4 has its own identity column, which is not allowed in PG17 +-- and will produce an error on attempting to attach it to the partitioned table +CREATE TABLE pt_4 (a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), c int); +ALTER TABLE partitioned_table ATTACH PARTITION pt_4 FOR VALUES FROM (2000) TO (3000); + +\c - - - :worker_1_port + +SET search_path TO pg17; +-- Show that DDL for partitioned_table has correctly propagated to the worker node; +-- (1) The partitioned table has pt_1, pt_2 and pt_3 as its partitions +\d+ partitioned_table; + +-- (2) The partititions have the same identity column as the parent table +\d pt_1; +\d pt_2; +\d pt_3; + +\c - - - :master_port +SET search_path TO pg17; + +-- Test detaching a partition with an identity column +ALTER TABLE partitioned_table DETACH PARTITION pt_3; + +-- partitioned_table has pt_1, pt_2 as its partitions +-- and pt_3 does not have an identity column +\d+ partitioned_table; +\d pt_3; + +-- Verify that the detach has propagated to the worker node +\c - - - :worker_1_port +SET search_path TO pg17; + +\d+ partitioned_table; +\d pt_3; + +\c - - - :master_port +SET search_path TO pg17; + +CREATE TABLE alt_test (a int, b date, c int) PARTITION BY RANGE(c); +SELECT create_distributed_table('alt_test', 'a'); + +CREATE TABLE alt_test_pt_1 PARTITION OF alt_test FOR VALUES FROM (1) TO (50); +CREATE TABLE alt_test_pt_2 PARTITION OF alt_test FOR VALUES FROM (50) TO (100); + +-- Citus does not support adding an identity column for a distributed table (#6738) +-- Attempting to add a column with identity produces an error +ALTER TABLE alt_test ADD COLUMN d bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10); + +-- alter table set identity is currently not supported, so adding identity to +-- an existing column generates an error +ALTER TABLE alt_test ALTER COLUMN a SET GENERATED BY DEFAULT SET INCREMENT BY 2 SET START WITH 75 RESTART; + +-- Verify that the identity column was not added, on coordinator and worker nodes +\d+ alt_test; +\d alt_test_pt_1; +\d alt_test_pt_2; + +\c - - - :worker_1_port +SET search_path TO pg17; + +\d+ alt_test; +\d alt_test_pt_1; +\d alt_test_pt_2; + +\c - - - :master_port +SET search_path TO pg17; + +DROP TABLE alt_test; +CREATE TABLE alt_test (a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b int, + c int) +PARTITION BY RANGE(c); + +SELECT create_distributed_table('alt_test', 'b'); +CREATE TABLE alt_test_pt_1 PARTITION OF alt_test FOR VALUES FROM (1) TO (50); +CREATE TABLE alt_test_pt_2 PARTITION OF alt_test FOR VALUES FROM (50) TO (100); + +-- Dropping of the identity property from a column is currently not supported; +-- Attempting to drop identity produces an error +ALTER TABLE alt_test ALTER COLUMN a DROP IDENTITY; + +-- Verify that alt_test still has identity on column a +\d+ alt_test; +\d alt_test_pt_1; +\d alt_test_pt_2; + +\c - - - :worker_1_port +SET search_path TO pg17; + +\d+ alt_test; +\d alt_test_pt_1; +\d alt_test_pt_2 + +\c - - - :master_port +SET search_path TO pg17; + +-- Repeat testing of partitions with identity column on a citus local table + +CREATE TABLE local_partitioned_table ( + a bigint GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE lpt_1 PARTITION OF local_partitioned_table FOR VALUES FROM (1) TO (50); + +SELECT citus_add_local_table_to_metadata('local_partitioned_table'); + +-- Can create tables as partitions and attach tables as partitions to a citus local table: +CREATE TABLE lpt_2 PARTITION OF local_partitioned_table FOR VALUES FROM (50) TO (1000); + +CREATE TABLE lpt_3 (a bigint not null, c int); +ALTER TABLE local_partitioned_table ATTACH PARTITION lpt_3 FOR VALUES FROM (1000) TO (2000); + +-- The partitions have the same identity column as the parent table, on coordinator and worker nodes +\d+ local_partitioned_table; +\d lpt_1; +\d lpt_2; +\d lpt_3; + +\c - - - :worker_1_port +SET search_path TO pg17; + +\d+ local_partitioned_table; +\d lpt_1; +\d lpt_2; +\d lpt_3; + +\c - - - :master_port +SET search_path TO pg17; + +-- Test detaching a partition with an identity column from a citus local table +ALTER TABLE local_partitioned_table DETACH PARTITION lpt_3; + +\d+ local_partitioned_table; +\d lpt_3; + +\c - - - :worker_1_port +SET search_path TO pg17; + +\d+ local_partitioned_table; +\d lpt_3; + +\c - - - :master_port +SET search_path TO pg17; + +DROP TABLE partitioned_table; +DROP TABLE local_partitioned_table; +DROP TABLE lpt_3; +DROP TABLE pt_3; +DROP TABLE pt_4; +DROP TABLE alt_test; + +-- End of partition with identity columns testing + RESET citus.next_shard_id; RESET citus.shard_count; RESET citus.shard_replication_factor; From a0cd8bd37b854a69589b005eed4d38345b104c66 Mon Sep 17 00:00:00 2001 From: Teja Mupparti <44680808+tejeswarm@users.noreply.github.com> Date: Thu, 19 Dec 2024 03:02:24 -0800 Subject: [PATCH 083/155] PG17 Compatibility: Support MERGE features in Citus with clean exceptions (#7781) - Adapted `pgmerge.sql` tests from PostgreSQL community's `merge.sql` to Citus by converting tables into Citus local tables. - Identified two new PostgreSQL 17 MERGE features (`RETURNING` support and MERGE on updatable views) not yet supported by Citus. - Implemented changes to detect unsupported features and raise clean exceptions, ensuring pgmerge tests pass without diffs. - Addressed breaking changes caused by `MERGE ... WHEN NOT MATCHED BY SOURCE` restructuring, reducing diffs in pgmerge tests. - Segregated unsupported test cases into `merge_unsupported.sql` to maintain clarity and avoid large diffs in test files. - Prepared the Citus MERGE planner to handle new PostgreSQL changes, reducing remaining test discrepancies. All merge tests now pass cleanly, with unsupported cases clearly isolated. Relevant PG commits: c649fa24a https://github.com/postgres/postgres/commit/c649fa24a 0294df2f1 https://github.com/postgres/postgres/commit/0294df2f1 --------- Co-authored-by: naisila --- src/backend/distributed/commands/multi_copy.c | 29 +++++ .../distributed/planner/merge_planner.c | 86 +++++++++++++-- .../planner/multi_router_planner.c | 3 +- src/include/distributed/merge_planner.h | 1 + .../regress/expected/merge_unsupported.out | 101 ++++++++++++++++++ .../regress/expected/merge_unsupported_0.out | 100 +++++++++++++++++ .../regress/expected/merge_unsupported_1.out | 17 +++ src/test/regress/expected/pg15.out | 10 -- src/test/regress/expected/pgmerge.out | 37 ------- src/test/regress/multi_schedule | 2 +- src/test/regress/sql/merge_unsupported.sql | 89 +++++++++++++++ src/test/regress/sql/pg15.sql | 10 -- src/test/regress/sql/pgmerge.sql | 34 ------ 13 files changed, 415 insertions(+), 104 deletions(-) create mode 100644 src/test/regress/expected/merge_unsupported.out create mode 100644 src/test/regress/expected/merge_unsupported_0.out create mode 100644 src/test/regress/expected/merge_unsupported_1.out create mode 100644 src/test/regress/sql/merge_unsupported.sql diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index b9e905850..268010e7a 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -346,6 +346,7 @@ static LocalCopyStatus GetLocalCopyStatus(void); static bool ShardIntervalListHasLocalPlacements(List *shardIntervalList); static void LogLocalCopyToRelationExecution(uint64 shardId); static void LogLocalCopyToFileExecution(uint64 shardId); +static void ErrorIfMergeInCopy(CopyStmt *copyStatement); /* exports for SQL callable functions */ @@ -2828,6 +2829,32 @@ CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName) } +/* + * ErrorIfMergeInCopy Raises an exception if the MERGE is called in the COPY + * where Citus tables are involved, as we don't support this yet + * Relevant PG17 commit: c649fa24a + */ +static void +ErrorIfMergeInCopy(CopyStmt *copyStatement) +{ +#if PG_VERSION_NUM < 170000 + return; +#else + if (!copyStatement->relation && (IsA(copyStatement->query, MergeStmt))) + { + /* + * This path is currently not reachable because Merge in COPY can + * only work with a RETURNING clause, and a RETURNING check + * will error out sooner for Citus + */ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("MERGE with Citus tables " + "is not yet supported in COPY"))); + } +#endif +} + + /* * ProcessCopyStmt handles Citus specific concerns for COPY like supporting * COPYing from distributed tables and preventing unsupported actions. The @@ -2865,6 +2892,8 @@ ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletion *completionTag, const */ if (copyStatement->relation != NULL) { + ErrorIfMergeInCopy(copyStatement); + bool isFrom = copyStatement->is_from; /* consider using RangeVarGetRelidExtended to check perms before locking */ diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 8048002e0..5fee6e956 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -97,6 +97,8 @@ static DistributedPlan * CreateNonPushableMergePlan(Oid targetRelationId, uint64 plannerRestrictionContext, ParamListInfo boundParams); static char * MergeCommandResultIdPrefix(uint64 planId); +static void ErrorIfMergeHasReturningList(Query *query); +static Node * GetMergeJoinCondition(Query *mergeQuery); #endif @@ -156,8 +158,51 @@ CreateMergePlan(uint64 planId, Query *originalQuery, Query *query, } +/* + * GetMergeJoinTree constructs and returns the jointree for a MERGE query. + */ +FromExpr * +GetMergeJoinTree(Query *mergeQuery) +{ + FromExpr *mergeJointree = NULL; +#if PG_VERSION_NUM >= PG_VERSION_17 + + /* + * In Postgres 17, the query tree has a specific field for the merge condition. + * For deriving the WhereClauseList from the merge condition, we construct a dummy + * jointree with an empty fromlist. This works because the fromlist of a merge query + * join tree consists of range table references only, and range table references are + * disregarded by the WhereClauseList() walker. + * Relevant PG17 commit: 0294df2f1 + */ + mergeJointree = makeFromExpr(NIL, mergeQuery->mergeJoinCondition); +#else + mergeJointree = mergeQuery->jointree; +#endif + + return mergeJointree; +} + + #if PG_VERSION_NUM >= PG_VERSION_15 + +/* + * GetMergeJoinCondition returns the quals of the ON condition + */ +static Node * +GetMergeJoinCondition(Query *mergeQuery) +{ + Node *joinCondition = NULL; +#if PG_VERSION_NUM >= PG_VERSION_17 + joinCondition = (Node *) mergeQuery->mergeJoinCondition; +#else + joinCondition = (Node *) mergeQuery->jointree->quals; +#endif + return joinCondition; +} + + /* * CreateRouterMergePlan attempts to create a pushable plan for the given MERGE * SQL statement. If the planning fails, the ->planningError is set to a description @@ -562,7 +607,7 @@ MergeQualAndTargetListFunctionsSupported(Oid resultRelationId, Query *query, List *targetList, CmdType commandType) { uint32 targetRangeTableIndex = query->resultRelation; - FromExpr *joinTree = query->jointree; + FromExpr *joinTree = GetMergeJoinTree(query); Var *distributionColumn = NULL; if (IsCitusTable(resultRelationId) && HasDistributionKey(resultRelationId)) { @@ -722,8 +767,9 @@ ErrorIfRepartitionMergeNotSupported(Oid targetRelationId, Query *mergeQuery, /* * Sub-queries and CTEs are not allowed in actions and ON clause */ - if (FindNodeMatchingCheckFunction((Node *) mergeQuery->jointree->quals, - IsNodeSubquery)) + Node *joinCondition = GetMergeJoinCondition(mergeQuery); + + if (FindNodeMatchingCheckFunction(joinCondition, IsNodeSubquery)) { ereport(ERROR, (errmsg("Sub-queries and CTEs are not allowed in ON clause for MERGE " @@ -949,9 +995,26 @@ ConvertSourceRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, } +/* + * ErrorIfMergeHasReturningList raises an exception if the MERGE has + * a RETURNING clause, as we don't support this yet for Citus tables + * Relevant PG17 commit: c649fa24a + */ +static void +ErrorIfMergeHasReturningList(Query *query) +{ + if (query->returningList) + { + ereport(ERROR, (errmsg("MERGE with RETURNING is not yet supported " + "for Citus tables"))); + } +} + + /* * ErrorIfMergeNotSupported Checks for conditions that are not supported in either * the routable or repartition strategies. It checks for + * - MERGE with a RETURNING clause * - Supported table types and their combinations * - Check the target lists and quals of both the query and merge actions * - Supported CTEs @@ -959,6 +1022,7 @@ ConvertSourceRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, static void ErrorIfMergeNotSupported(Query *query, Oid targetRelationId, List *rangeTableList) { + ErrorIfMergeHasReturningList(query); ErrorIfMergeHasUnsupportedTables(targetRelationId, rangeTableList); ErrorIfMergeQueryQualAndTargetListNotSupported(targetRelationId, query); ErrorIfUnsupportedCTEs(query); @@ -1207,12 +1271,15 @@ ErrorIfMergeQueryQualAndTargetListNotSupported(Oid targetRelationId, Query *orig "supported in MERGE sql with distributed tables"))); } + Node *joinCondition = GetMergeJoinCondition(originalQuery); + DeferredErrorMessage *deferredError = - MergeQualAndTargetListFunctionsSupported(targetRelationId, - originalQuery, - originalQuery->jointree->quals, - originalQuery->targetList, - originalQuery->commandType); + MergeQualAndTargetListFunctionsSupported( + targetRelationId, + originalQuery, + joinCondition, + originalQuery->targetList, + originalQuery->commandType); if (deferredError) { @@ -1286,8 +1353,7 @@ static int SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, CitusTableCacheEntry *targetRelation) { - /* Get all the Join conditions from the ON clause */ - List *mergeJoinConditionList = WhereClauseList(mergeQuery->jointree); + List *mergeJoinConditionList = WhereClauseList(GetMergeJoinTree(mergeQuery)); Var *targetColumn = targetRelation->partitionColumn; Var *sourceRepartitionVar = NULL; bool foundTypeMismatch = false; diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index bb1f64488..72d1dddd1 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -598,8 +598,7 @@ TargetlistAndFunctionsSupported(Oid resultRelationId, FromExpr *joinTree, Node * } if (commandType == CMD_UPDATE && targetEntryPartitionColumn && - TargetEntryChangesValue(targetEntry, partitionColumn, - joinTree)) + TargetEntryChangesValue(targetEntry, partitionColumn, joinTree)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "modifying the partition value of rows is not " diff --git a/src/include/distributed/merge_planner.h b/src/include/distributed/merge_planner.h index b6636687a..53d451ea6 100644 --- a/src/include/distributed/merge_planner.h +++ b/src/include/distributed/merge_planner.h @@ -32,6 +32,7 @@ extern void NonPushableMergeCommandExplainScan(CustomScanState *node, List *ance struct ExplainState *es); extern Var * FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query); extern RangeTblEntry * ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk); +extern FromExpr * GetMergeJoinTree(Query *mergeQuery); #endif /* MERGE_PLANNER_H */ diff --git a/src/test/regress/expected/merge_unsupported.out b/src/test/regress/expected/merge_unsupported.out new file mode 100644 index 000000000..62f51a679 --- /dev/null +++ b/src/test/regress/expected/merge_unsupported.out @@ -0,0 +1,101 @@ +SHOW server_version \gset +SELECT CASE + WHEN substring(current_setting('server_version'), '\d+')::int >= 17 THEN '17+' + WHEN substring(current_setting('server_version'), '\d+')::int IN (15, 16) THEN '15_16' + WHEN substring(current_setting('server_version'), '\d+')::int = 14 THEN '14' + ELSE 'Unsupported version' + END AS version_category; + version_category +--------------------------------------------------------------------- + 17+ +(1 row) + +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif +-- +-- MERGE test from PG community (adapted to Citus by converting all tables to Citus local) +-- +DROP SCHEMA IF EXISTS pgmerge_schema CASCADE; +NOTICE: schema "pgmerge_schema" does not exist, skipping +CREATE SCHEMA pgmerge_schema; +SET search_path TO pgmerge_schema; +SET citus.use_citus_managed_tables to true; +CREATE TABLE target (tid integer, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE source (sid integer, delta integer) -- no index + WITH (autovacuum_enabled=off); +\set SHOW_CONTEXT errors +-- used in a CTE +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) SELECT * FROM foo; +ERROR: WITH query "foo" does not have a RETURNING clause +-- used in COPY +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) TO stdout; +ERROR: COPY query must have a RETURNING clause +-- used in a CTE with RETURNING +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) SELECT * FROM foo; +ERROR: MERGE with RETURNING is not yet supported for Citus tables +-- used in COPY with RETURNING +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) TO stdout; +ERROR: MERGE with RETURNING is not yet supported for Citus tables +-- unsupported relation types +-- view +CREATE VIEW tv AS SELECT count(tid) AS tid FROM target; +MERGE INTO tv t +USING source s +ON t.tid = s.sid +WHEN NOT MATCHED THEN + INSERT DEFAULT VALUES; +ERROR: cannot insert into view "tv" +DETAIL: Views that return aggregate functions are not automatically updatable. +HINT: To enable inserting into the view using MERGE, provide an INSTEAD OF INSERT trigger. +DROP VIEW tv; +CREATE TABLE sq_target (tid integer NOT NULL, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) + WITH (autovacuum_enabled=off); +SELECT citus_add_local_table_to_metadata('sq_target'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT citus_add_local_table_to_metadata('sq_source'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); +INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); +CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; +-- RETURNING +BEGIN; +INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); +MERGE INTO sq_target t +USING v +ON tid = sid +WHEN MATCHED AND tid > 2 THEN + UPDATE SET balance = t.balance + delta +WHEN NOT MATCHED THEN + INSERT (balance, tid) VALUES (balance + delta, sid) +WHEN MATCHED AND tid < 2 THEN + DELETE +RETURNING *; +ERROR: MERGE with RETURNING is not yet supported for Citus tables +ROLLBACK; diff --git a/src/test/regress/expected/merge_unsupported_0.out b/src/test/regress/expected/merge_unsupported_0.out new file mode 100644 index 000000000..b788c1670 --- /dev/null +++ b/src/test/regress/expected/merge_unsupported_0.out @@ -0,0 +1,100 @@ +SHOW server_version \gset +SELECT CASE + WHEN substring(current_setting('server_version'), '\d+')::int >= 17 THEN '17+' + WHEN substring(current_setting('server_version'), '\d+')::int IN (15, 16) THEN '15_16' + WHEN substring(current_setting('server_version'), '\d+')::int = 14 THEN '14' + ELSE 'Unsupported version' + END AS version_category; + version_category +--------------------------------------------------------------------- + 15_16 +(1 row) + +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif +-- +-- MERGE test from PG community (adapted to Citus by converting all tables to Citus local) +-- +DROP SCHEMA IF EXISTS pgmerge_schema CASCADE; +NOTICE: schema "pgmerge_schema" does not exist, skipping +CREATE SCHEMA pgmerge_schema; +SET search_path TO pgmerge_schema; +SET citus.use_citus_managed_tables to true; +CREATE TABLE target (tid integer, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE source (sid integer, delta integer) -- no index + WITH (autovacuum_enabled=off); +\set SHOW_CONTEXT errors +-- used in a CTE +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) SELECT * FROM foo; +ERROR: MERGE not supported in WITH query +-- used in COPY +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) TO stdout; +ERROR: MERGE not supported in COPY +-- used in a CTE with RETURNING +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) SELECT * FROM foo; +ERROR: syntax error at or near "RETURNING" +-- used in COPY with RETURNING +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) TO stdout; +ERROR: syntax error at or near "RETURNING" +-- unsupported relation types +-- view +CREATE VIEW tv AS SELECT count(tid) AS tid FROM target; +MERGE INTO tv t +USING source s +ON t.tid = s.sid +WHEN NOT MATCHED THEN + INSERT DEFAULT VALUES; +ERROR: cannot execute MERGE on relation "tv" +DETAIL: This operation is not supported for views. +DROP VIEW tv; +CREATE TABLE sq_target (tid integer NOT NULL, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) + WITH (autovacuum_enabled=off); +SELECT citus_add_local_table_to_metadata('sq_target'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT citus_add_local_table_to_metadata('sq_source'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); +INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); +CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; +-- RETURNING +BEGIN; +INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); +MERGE INTO sq_target t +USING v +ON tid = sid +WHEN MATCHED AND tid > 2 THEN + UPDATE SET balance = t.balance + delta +WHEN NOT MATCHED THEN + INSERT (balance, tid) VALUES (balance + delta, sid) +WHEN MATCHED AND tid < 2 THEN + DELETE +RETURNING *; +ERROR: syntax error at or near "RETURNING" +ROLLBACK; diff --git a/src/test/regress/expected/merge_unsupported_1.out b/src/test/regress/expected/merge_unsupported_1.out new file mode 100644 index 000000000..187c5d630 --- /dev/null +++ b/src/test/regress/expected/merge_unsupported_1.out @@ -0,0 +1,17 @@ +SHOW server_version \gset +SELECT CASE + WHEN substring(current_setting('server_version'), '\d+')::int >= 17 THEN '17+' + WHEN substring(current_setting('server_version'), '\d+')::int IN (15, 16) THEN '15_16' + WHEN substring(current_setting('server_version'), '\d+')::int = 14 THEN '14' + ELSE 'Unsupported version' + END AS version_category; + version_category +--------------------------------------------------------------------- + 14 +(1 row) + +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index c06142671..d945a07d6 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -437,16 +437,6 @@ MERGE INTO tbl1 USING targq ON (true) WHEN MATCHED THEN DELETE; ERROR: The required join operation is missing between the target's distribution column and any expression originating from the source. The issue may arise from a non-equi-join. DETAIL: Without a equi-join condition on the target's distribution column, the source rows cannot be efficiently redistributed, and the NOT-MATCHED condition cannot be evaluated unambiguously. This can result in incorrect or unexpected results when attempting to merge tables in a distributed setting -WITH foo AS ( - MERGE INTO tbl1 USING tbl2 ON (true) - WHEN MATCHED THEN DELETE -) SELECT * FROM foo; -ERROR: MERGE not supported in WITH query -COPY ( - MERGE INTO tbl1 USING tbl2 ON (true) - WHEN MATCHED THEN DELETE -) TO stdout; -ERROR: MERGE not supported in COPY MERGE INTO tbl1 t USING tbl2 ON (true) diff --git a/src/test/regress/expected/pgmerge.out b/src/test/regress/expected/pgmerge.out index a0f5d0c86..0c2f9b741 100644 --- a/src/test/regress/expected/pgmerge.out +++ b/src/test/regress/expected/pgmerge.out @@ -162,29 +162,7 @@ ON tid = tid WHEN MATCHED THEN DO NOTHING; ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. --- used in a CTE -WITH foo AS ( - MERGE INTO target USING source ON (true) - WHEN MATCHED THEN DELETE -) SELECT * FROM foo; -ERROR: MERGE not supported in WITH query --- used in COPY -COPY ( - MERGE INTO target USING source ON (true) - WHEN MATCHED THEN DELETE -) TO stdout; -ERROR: MERGE not supported in COPY -- unsupported relation types --- view -CREATE VIEW tv AS SELECT * FROM target; -MERGE INTO tv t -USING source s -ON t.tid = s.sid -WHEN NOT MATCHED THEN - INSERT DEFAULT VALUES; -ERROR: cannot execute MERGE on relation "tv" -DETAIL: This operation is not supported for views. -DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t @@ -1376,21 +1354,6 @@ WHEN NOT MATCHED THEN WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; --- RETURNING -BEGIN; -INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); -MERGE INTO sq_target t -USING v -ON tid = sid -WHEN MATCHED AND tid > 2 THEN - UPDATE SET balance = t.balance + delta -WHEN NOT MATCHED THEN - INSERT (balance, tid) VALUES (balance + delta, sid) -WHEN MATCHED AND tid < 2 THEN - DELETE -RETURNING *; -ERROR: syntax error at or near "RETURNING" -ROLLBACK; -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 32d1f9707..0fa54bb38 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -118,7 +118,7 @@ test: merge pgmerge test: merge_repartition2 test: merge_repartition1 merge_schema_sharding test: merge_partition_tables -test: merge_vcore +test: merge_vcore merge_unsupported # --------- # test that no tests leaked intermediate results. This should always be last diff --git a/src/test/regress/sql/merge_unsupported.sql b/src/test/regress/sql/merge_unsupported.sql new file mode 100644 index 000000000..ef95e01ea --- /dev/null +++ b/src/test/regress/sql/merge_unsupported.sql @@ -0,0 +1,89 @@ + + +SHOW server_version \gset +SELECT CASE + WHEN substring(current_setting('server_version'), '\d+')::int >= 17 THEN '17+' + WHEN substring(current_setting('server_version'), '\d+')::int IN (15, 16) THEN '15_16' + WHEN substring(current_setting('server_version'), '\d+')::int = 14 THEN '14' + ELSE 'Unsupported version' + END AS version_category; +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif + +-- +-- MERGE test from PG community (adapted to Citus by converting all tables to Citus local) +-- +DROP SCHEMA IF EXISTS pgmerge_schema CASCADE; +CREATE SCHEMA pgmerge_schema; +SET search_path TO pgmerge_schema; + +SET citus.use_citus_managed_tables to true; + +CREATE TABLE target (tid integer, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE source (sid integer, delta integer) -- no index + WITH (autovacuum_enabled=off); + +\set SHOW_CONTEXT errors +-- used in a CTE +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) SELECT * FROM foo; +-- used in COPY +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE +) TO stdout; +-- used in a CTE with RETURNING +WITH foo AS ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) SELECT * FROM foo; +-- used in COPY with RETURNING +COPY ( + MERGE INTO target USING source ON (true) + WHEN MATCHED THEN DELETE RETURNING target.* +) TO stdout; + +-- unsupported relation types +-- view +CREATE VIEW tv AS SELECT count(tid) AS tid FROM target; +MERGE INTO tv t +USING source s +ON t.tid = s.sid +WHEN NOT MATCHED THEN + INSERT DEFAULT VALUES; +DROP VIEW tv; + +CREATE TABLE sq_target (tid integer NOT NULL, balance integer) + WITH (autovacuum_enabled=off); +CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) + WITH (autovacuum_enabled=off); + +SELECT citus_add_local_table_to_metadata('sq_target'); +SELECT citus_add_local_table_to_metadata('sq_source'); + +INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); +INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); + +CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; + +-- RETURNING +BEGIN; +INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); +MERGE INTO sq_target t +USING v +ON tid = sid +WHEN MATCHED AND tid > 2 THEN + UPDATE SET balance = t.balance + delta +WHEN NOT MATCHED THEN + INSERT (balance, tid) VALUES (balance + delta, sid) +WHEN MATCHED AND tid < 2 THEN + DELETE +RETURNING *; +ROLLBACK; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 0d53bc9dd..276092dcd 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -287,16 +287,6 @@ WITH targq AS ( MERGE INTO tbl1 USING targq ON (true) WHEN MATCHED THEN DELETE; -WITH foo AS ( - MERGE INTO tbl1 USING tbl2 ON (true) - WHEN MATCHED THEN DELETE -) SELECT * FROM foo; - -COPY ( - MERGE INTO tbl1 USING tbl2 ON (true) - WHEN MATCHED THEN DELETE -) TO stdout; - MERGE INTO tbl1 t USING tbl2 ON (true) diff --git a/src/test/regress/sql/pgmerge.sql b/src/test/regress/sql/pgmerge.sql index e1f3c7aab..69a0210bc 100644 --- a/src/test/regress/sql/pgmerge.sql +++ b/src/test/regress/sql/pgmerge.sql @@ -116,27 +116,8 @@ MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; --- used in a CTE -WITH foo AS ( - MERGE INTO target USING source ON (true) - WHEN MATCHED THEN DELETE -) SELECT * FROM foo; --- used in COPY -COPY ( - MERGE INTO target USING source ON (true) - WHEN MATCHED THEN DELETE -) TO stdout; -- unsupported relation types --- view -CREATE VIEW tv AS SELECT * FROM target; -MERGE INTO tv t -USING source s -ON t.tid = s.sid -WHEN NOT MATCHED THEN - INSERT DEFAULT VALUES; -DROP VIEW tv; - -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t @@ -905,21 +886,6 @@ WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; --- RETURNING -BEGIN; -INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); -MERGE INTO sq_target t -USING v -ON tid = sid -WHEN MATCHED AND tid > 2 THEN - UPDATE SET balance = t.balance + delta -WHEN NOT MATCHED THEN - INSERT (balance, tid) VALUES (balance + delta, sid) -WHEN MATCHED AND tid < 2 THEN - DELETE -RETURNING *; -ROLLBACK; - -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); From ad9a3bfc1a67fb89ec1e7ec6a11deb6c0a5c3a99 Mon Sep 17 00:00:00 2001 From: Colm Date: Thu, 19 Dec 2024 18:17:57 +0000 Subject: [PATCH 084/155] PG17 compatibility: ensure get_progress() output is consistent (#7793) in regress test isolation_progress_monitoring, with an ORDER BY. The implementation of get_progress() uses a tuplestore to hold the step and progress values, and tuplestore does not provide any guarantee on the ordering of the tuples so ORDER BY ensures stable test output. Also make the output more user friendly by including the column names. Fixing occasional failures seen in isolation_progress_monitoring. ![Screenshot (86)](https://github.com/user-attachments/assets/a019639f-559f-408d-b8a8-8b7a44d8095d) --- .../isolation_progress_monitoring.out | 60 +++++++++---------- .../spec/isolation_progress_monitoring.spec | 4 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/test/regress/expected/isolation_progress_monitoring.out b/src/test/regress/expected/isolation_progress_monitoring.out index 6b8391e3f..eafb0ecb9 100644 --- a/src/test/regress/expected/isolation_progress_monitoring.out +++ b/src/test/regress/expected/isolation_progress_monitoring.out @@ -70,21 +70,21 @@ step s3-start-operation: SELECT sample_operation(3778, 30, 9); step show-progress: - SELECT show_progress(1337); - SELECT show_progress(3778); + SELECT step, progress FROM show_progress(1337) ORDER BY 1, 2; + SELECT step, progress FROM show_progress(3778) ORDER BY 1, 2; -show_progress +step|progress --------------------------------------------------------------------- -(0,0) -(1,0) -(0,0) -(1,0) + 0| 0 + 0| 0 + 1| 0 + 1| 0 (4 rows) -show_progress +step|progress --------------------------------------------------------------------- -(0,0) -(1,0) + 0| 0 + 1| 0 (2 rows) step release-locks-1: @@ -109,21 +109,21 @@ t (1 row) step show-progress: - SELECT show_progress(1337); - SELECT show_progress(3778); + SELECT step, progress FROM show_progress(1337) ORDER BY 1, 2; + SELECT step, progress FROM show_progress(3778) ORDER BY 1, 2; -show_progress +step|progress --------------------------------------------------------------------- -(0,-1) -(1,0) -(0,2) -(1,0) + 0| -1 + 0| 2 + 1| 0 + 1| 0 (4 rows) -show_progress +step|progress --------------------------------------------------------------------- -(0,9) -(1,0) + 0| 9 + 1| 0 (2 rows) step release-locks-2: @@ -148,21 +148,21 @@ t (1 row) step show-progress: - SELECT show_progress(1337); - SELECT show_progress(3778); + SELECT step, progress FROM show_progress(1337) ORDER BY 1, 2; + SELECT step, progress FROM show_progress(3778) ORDER BY 1, 2; -show_progress +step|progress --------------------------------------------------------------------- -(0,-1) -(1,-1) -(0,2) -(1,2) + 0| -1 + 0| 2 + 1| -1 + 1| 2 (4 rows) -show_progress +step|progress --------------------------------------------------------------------- -(0,9) -(1,9) + 0| 9 + 1| 9 (2 rows) step release-locks-3: diff --git a/src/test/regress/spec/isolation_progress_monitoring.spec b/src/test/regress/spec/isolation_progress_monitoring.spec index 225451ec8..0a88c6b00 100644 --- a/src/test/regress/spec/isolation_progress_monitoring.spec +++ b/src/test/regress/spec/isolation_progress_monitoring.spec @@ -126,8 +126,8 @@ session "monitor" step "show-progress" { - SELECT show_progress(1337); - SELECT show_progress(3778); + SELECT step, progress FROM show_progress(1337) ORDER BY 1, 2; + SELECT step, progress FROM show_progress(3778) ORDER BY 1, 2; } permutation "take-locks" "s1-start-operation" "s2-start-operation" "s3-start-operation" "show-progress" "release-locks-1" "show-progress" "release-locks-2" "show-progress" "release-locks-3" From acd7b1e690c2a73b9ca5394df57a272eebbfae02 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Thu, 19 Dec 2024 22:21:23 +0300 Subject: [PATCH 085/155] PG17 compatibility: Fix Test Failure in local_dist_join_mixed (#7731) PostgreSQL 16 adds an extra condition (id IS NOT NULL) to the subquery. This condition is likely used to ensure that no null values are processed in the subquery. Instead of using the condition id IS NOT NULL, PostgreSQL 17 generates the subplan with a trivial condition (WHERE true), indicating that it does not need to explicitly check for non-null values. PostgreSQL 17 likely includes optimizations to handle null checks more efficiently. The WHERE (id IS NOT NULL) condition that was present in PostgreSQL 16 may now be considered redundant by the planner, as it is implicitly handled by the query execution engine. https://github.com/postgres/postgres/commit/b262ad44 ```diff SELECT foo1.id FROM (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo9, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo8, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo7, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo6, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo5, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo4, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo3, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo2, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo10, (SELECT local.id, local.title FROM local, distributed WHERE local.id = distributed.id ) as foo1 WHERE foo1.id = foo9.id AND foo1.id = foo8.id AND foo1.id = foo7.id AND foo1.id = foo6.id AND foo1.id = foo5.id AND foo1.id = foo4.id AND foo1.id = foo3.id AND foo1.id = foo2.id AND foo1.id = foo10.id AND foo1.id = foo1.id ORDER BY 1; ... -DEBUG: generating subplan XXX_10 for subquery SELECT id FROM local_dist_join_mixed.local WHERE (id IS NOT NULL) +DEBUG: generating subplan XXX_10 for subquery SELECT id FROM local_dist_join_mixed.local WHERE true ... ``` --- src/test/regress/bin/normalize.sed | 4 ++++ src/test/regress/expected/local_dist_join_mixed.out | 7 +++---- src/test/regress/sql/local_dist_join_mixed.sql | 3 +-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 59bf6d97c..c110c0c49 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -331,3 +331,7 @@ s/\| CHECK ([a-zA-Z])(.*)/| CHECK \(\1\2\)/g # supported Postgres version. /DEBUG: drop auto-cascades to type [a-zA-Z_]*.pg_temp_[0-9]*/d + +# Normalize subquery condition in debug logs to standardize across PostgreSQL versions #7731 +# https://github.com/postgres/postgres/commit/b262ad44 +s/for subquery SELECT id FROM local_dist_join_mixed\.local WHERE \(id IS NOT NULL\)/for subquery SELECT id FROM local_dist_join_mixed.local WHERE true/g diff --git a/src/test/regress/expected/local_dist_join_mixed.out b/src/test/regress/expected/local_dist_join_mixed.out index b8f074c73..e3e25f7a0 100644 --- a/src/test/regress/expected/local_dist_join_mixed.out +++ b/src/test/regress/expected/local_dist_join_mixed.out @@ -1244,8 +1244,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT local.id, dis foo1.id = foo4.id AND foo1.id = foo3.id AND foo1.id = foo2.id AND - foo1.id = foo10.id AND - foo1.id = foo1.id + foo1.id = foo10.id ORDER BY 1; DEBUG: Wrapping relation "local" to a subquery DEBUG: generating subplan XXX_1 for subquery SELECT id FROM local_dist_join_mixed.local WHERE true @@ -1266,8 +1265,8 @@ DEBUG: generating subplan XXX_8 for subquery SELECT id FROM local_dist_join_mix DEBUG: Wrapping relation "local" to a subquery DEBUG: generating subplan XXX_9 for subquery SELECT id FROM local_dist_join_mixed.local WHERE true DEBUG: Wrapping relation "local" to a subquery -DEBUG: generating subplan XXX_10 for subquery SELECT id FROM local_dist_join_mixed.local WHERE (id IS NOT NULL) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT foo1.id FROM (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo9, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo8, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo7, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo6, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo5, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_6'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo4, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_7'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo3, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_8'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo2, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_9'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo10, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_10'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo1 WHERE ((foo1.id OPERATOR(pg_catalog.=) foo9.id) AND (foo1.id OPERATOR(pg_catalog.=) foo8.id) AND (foo1.id OPERATOR(pg_catalog.=) foo7.id) AND (foo1.id OPERATOR(pg_catalog.=) foo6.id) AND (foo1.id OPERATOR(pg_catalog.=) foo5.id) AND (foo1.id OPERATOR(pg_catalog.=) foo4.id) AND (foo1.id OPERATOR(pg_catalog.=) foo3.id) AND (foo1.id OPERATOR(pg_catalog.=) foo2.id) AND (foo1.id OPERATOR(pg_catalog.=) foo10.id) AND (foo1.id OPERATOR(pg_catalog.=) foo1.id)) ORDER BY foo1.id +DEBUG: generating subplan XXX_10 for subquery SELECT id FROM local_dist_join_mixed.local WHERE true +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT foo1.id FROM (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo9, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo8, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo7, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo6, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo5, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_6'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo4, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_7'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo3, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_8'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo2, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_9'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo10, (SELECT local.id, local.title FROM (SELECT NULL::integer AS "dummy-1", local_1.id, NULL::integer AS "dummy-3", NULL::text AS title, NULL::integer AS "dummy-5" FROM (SELECT intermediate_result.id FROM read_intermediate_result('XXX_10'::text, 'binary'::citus_copy_format) intermediate_result(id bigint)) local_1) local, local_dist_join_mixed.distributed WHERE (local.id OPERATOR(pg_catalog.=) distributed.id)) foo1 WHERE ((foo1.id OPERATOR(pg_catalog.=) foo9.id) AND (foo1.id OPERATOR(pg_catalog.=) foo8.id) AND (foo1.id OPERATOR(pg_catalog.=) foo7.id) AND (foo1.id OPERATOR(pg_catalog.=) foo6.id) AND (foo1.id OPERATOR(pg_catalog.=) foo5.id) AND (foo1.id OPERATOR(pg_catalog.=) foo4.id) AND (foo1.id OPERATOR(pg_catalog.=) foo3.id) AND (foo1.id OPERATOR(pg_catalog.=) foo2.id) AND (foo1.id OPERATOR(pg_catalog.=) foo10.id)) ORDER BY foo1.id id --------------------------------------------------------------------- 0 diff --git a/src/test/regress/sql/local_dist_join_mixed.sql b/src/test/regress/sql/local_dist_join_mixed.sql index c6eb53d4e..14b0c6076 100644 --- a/src/test/regress/sql/local_dist_join_mixed.sql +++ b/src/test/regress/sql/local_dist_join_mixed.sql @@ -340,8 +340,7 @@ SELECT id, name FROM local LEFT JOIN distributed USING (id) ORDER BY 1 LIMIT 1; foo1.id = foo4.id AND foo1.id = foo3.id AND foo1.id = foo2.id AND - foo1.id = foo10.id AND - foo1.id = foo1.id + foo1.id = foo10.id ORDER BY 1; SELECT From e3db3751495cf238488aad156609b39fd74793eb Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Thu, 19 Dec 2024 22:21:51 +0300 Subject: [PATCH 086/155] PG17 compatibility: Fix Test Failure in local_table_join (#7732) PostgreSQL 17 seems to have introduced improvements in how correlated subqueries are handled during plan generation. Instead of generating a trivial subplan with WHERE true, it now applies more specific filtering (WHERE (key = 5)), which makes the execution plan more efficient. https://github.com/postgres/postgres/commit/b262ad44 ``` diff -dU10 -w /__w/citus/citus/src/test/regress/expected/local_table_join.out /__w/citus/citus/src/test/regress/results/local_table_join.out --- /__w/citus/citus/src/test/regress/expected/local_table_join.out.modified 2024-11-05 09:53:50.423970699 +0000 +++ /__w/citus/citus/src/test/regress/results/local_table_join.out.modified 2024-11-05 09:53:50.463971296 +0000 @@ -1420,32 +1420,32 @@ ) as subq_1 ) as subq_2; DEBUG: Wrapping relation "custom_pg_type" to a subquery DEBUG: generating subplan 204_1 for subquery SELECT typdefault FROM local_table_join.custom_pg_type WHERE true ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins -- correlated sublinks are not yet supported because of #4470, unless we convert not-correlated table SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table using(key) WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key and key = 5); DEBUG: Wrapping relation "postgres_table" to a subquery -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM local_table_join.postgres_table WHERE true +DEBUG: generating subplan 206_1 for subquery SELECT key FROM local_table_join.postgres_table WHERE (key OPERATOR(pg_catalog.=) 5) ``` Co-authored-by: Naisila Puka <37271756+naisila@users.noreply.github.com> --- .../regress/expected/local_table_join.out | 8 +- src/test/regress/expected/pg17.out | 245 ++++++++++++++++++ src/test/regress/sql/local_table_join.sql | 4 +- src/test/regress/sql/pg17.sql | 36 +++ 4 files changed, 287 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/local_table_join.out b/src/test/regress/expected/local_table_join.out index 297959d41..a4a3f5aa6 100644 --- a/src/test/regress/expected/local_table_join.out +++ b/src/test/regress/expected/local_table_join.out @@ -1425,10 +1425,10 @@ ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins -- correlated sublinks are not yet supported because of #4470, unless we convert not-correlated table SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table using(key) -WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key and key = 5); +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key); DEBUG: Wrapping relation "postgres_table" to a subquery DEBUG: generating subplan XXX_1 for subquery SELECT key FROM local_table_join.postgres_table WHERE true -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (local_table_join.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM local_table_join.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (local_table_join.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM local_table_join.distributed_table WHERE (d1.key OPERATOR(pg_catalog.=) distributed_table.key))) count --------------------------------------------------------------------- 100 @@ -1436,10 +1436,10 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c set citus.local_table_join_policy to 'prefer-distributed'; SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table using(key) -WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key and key = 5); +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key); DEBUG: Wrapping relation "distributed_table" "d1" to a subquery DEBUG: generating subplan XXX_1 for subquery SELECT key FROM local_table_join.distributed_table d1 WHERE true -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT d1_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) d1_1) d1 JOIN local_table_join.postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM local_table_join.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT d1_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) d1_1) d1 JOIN local_table_join.postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM local_table_join.distributed_table WHERE (d1.key OPERATOR(pg_catalog.=) distributed_table.key))) ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins set citus.local_table_join_policy to 'auto'; diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 35b4aa326..3cd86b4f2 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -962,6 +962,251 @@ DROP SCHEMA pg17 CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to function fake_am_handler(internal) drop cascades to access method fake_am +-- Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. +-- Enable DEBUG-level logging to capture detailed execution plans +SET client_min_messages TO DEBUG1; +-- Create the tables +CREATE TABLE postgres_table (key int, value text, value_2 jsonb); +CREATE TABLE reference_table (key int, value text, value_2 jsonb); +SELECT create_reference_table('reference_table'); +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER TABLE pg17.reference_table OWNER TO postgres +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER TABLE pg17.reference_table OWNER TO postgres +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.reference_table') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.reference_table') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.reference_table'::regclass, 'n', NULL, 1390013, 't') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.reference_table'::regclass, 'n', NULL, 1390013, 't') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.reference_table'::regclass, 20240024, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.reference_table'::regclass, 20240024, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (20240024, 0, 0, 3841), (20240024, 0, 14, 3842), (20240024, 0, 22, 3843)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (20240024, 0, 0, 3841), (20240024, 0, 14, 3842), (20240024, 0, 22, 3843)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'reference_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'reference_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT lock_shard_resources(3, ARRAY[20240024]) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE distributed_table (key int, value text, value_2 jsonb); +SELECT create_distributed_table('distributed_table', 'key'); +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240025, 'pg17', 'CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240025, 'pg17', 'ALTER TABLE pg17.distributed_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER TABLE pg17.distributed_table OWNER TO postgres +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER TABLE pg17.distributed_table OWNER TO postgres +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.distributed_table') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.distributed_table') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.distributed_table'::regclass, 'h', 'key', 1400006, 's') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.distributed_table'::regclass, 'h', 'key', 1400006, 's') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.distributed_table'::regclass, 20240025, 't'::"char", '-2147483648', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.distributed_table'::regclass, 20240025, 't'::"char", '-2147483648', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (xxxxxx, xxxxxx, xxxxxx, xxxxxx)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (xxxxxx, xxxxxx, xxxxxx, xxxxxx)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'distributed_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'distributed_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Insert test data +INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +DEBUG: distributed INSERT ... SELECT can only select from distributed tables +DEBUG: Collecting INSERT ... SELECT results on coordinator +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT lock_shard_resources(3, ARRAY[20240024]) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: executing the copy locally for shard xxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COPY pg17.reference_table_20240024 (key, value, value_2) FROM STDIN WITH (format 'binary') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COPY pg17.reference_table_20240024 (key, value, value_2) FROM STDIN WITH (format 'binary') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +DEBUG: distributed INSERT ... SELECT can only select from distributed tables +DEBUG: Collecting INSERT ... SELECT results on coordinator +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COPY pg17.distributed_table_20240025 (key, value, value_2) FROM STDIN WITH (format 'binary') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- Set local table join policy to auto before running the tests +SET citus.local_table_join_policy TO 'auto'; +-- Correlated sublinks are supported in PostgreSQL 17 +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); +DEBUG: Wrapping relation "postgres_table" to a subquery +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.postgres_table WHERE (key OPERATOR(pg_catalog.=) 5) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (pg17.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COPY "21_1" FROM STDIN WITH (format result) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT count(*) AS count FROM (pg17.distributed_table_20240025 d1(key, value, value_2) JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table_20240025 distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + count +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); +DEBUG: Wrapping relation "postgres_table" to a subquery +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.postgres_table WHERE (key OPERATOR(pg_catalog.=) 5) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (pg17.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COPY "22_1" FROM STDIN WITH (format result) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT count(*) AS count FROM (pg17.distributed_table_20240025 d1(key, value, value_2) JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table_20240025 distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + count +--------------------------------------------------------------------- + 1 +(1 row) + +SET citus.local_table_join_policy TO 'prefer-distributed'; +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); +DEBUG: Wrapping relation "distributed_table" "d1" to a subquery +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.distributed_table d1 WHERE (key OPERATOR(pg_catalog.=) 5) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT d1_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) d1_1) d1 JOIN pg17.postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) +ERROR: direct joins between distributed and local tables are not supported +HINT: Use CTE's or subqueries to select from local tables and use them in joins +SET citus.local_table_join_policy TO 'auto'; +-- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. +RESET citus.log_remote_commands; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; +DROP SCHEMA pg17 CASCADE; +DEBUG: switching to sequential query execution mode +DETAIL: A command for a distributed schema is run. To make sure subsequent commands see the schema correctly we need to make sure to use only one connection for all future commands +NOTICE: drop cascades to 7 other objects +DETAIL: drop cascades to function fake_am_handler(internal) +drop cascades to access method fake_am drop cascades to table dist_test +drop cascades to table postgres_table +drop cascades to table reference_table +drop cascades to table reference_table_20240024 +drop cascades to table distributed_table DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; diff --git a/src/test/regress/sql/local_table_join.sql b/src/test/regress/sql/local_table_join.sql index 393b15378..1ff29fbd2 100644 --- a/src/test/regress/sql/local_table_join.sql +++ b/src/test/regress/sql/local_table_join.sql @@ -397,11 +397,11 @@ select typdefault from ( -- correlated sublinks are not yet supported because of #4470, unless we convert not-correlated table SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table using(key) -WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key and key = 5); +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key); set citus.local_table_join_policy to 'prefer-distributed'; SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table using(key) -WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key and key = 5); +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key); set citus.local_table_join_policy to 'auto'; -- Some more subqueries diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 4730426eb..35d0e199c 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -468,6 +468,42 @@ RESET citus.next_shard_id; RESET citus.shard_count; RESET citus.shard_replication_factor; +DROP SCHEMA pg17 CASCADE; + +-- Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. +-- Enable DEBUG-level logging to capture detailed execution plans +RESET citus.log_remote_commands; +SET client_min_messages TO DEBUG1; +-- Create the tables +CREATE TABLE postgres_table (key int, value text, value_2 jsonb); +CREATE TABLE reference_table (key int, value text, value_2 jsonb); +SELECT create_reference_table('reference_table'); +CREATE TABLE distributed_table (key int, value text, value_2 jsonb); +SELECT create_distributed_table('distributed_table', 'key'); +-- Insert test data +INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +-- Set local table join policy to auto before running the tests +SET citus.local_table_join_policy TO 'auto'; +-- Correlated sublinks are supported in PostgreSQL 17 +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); + +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); + +SET citus.local_table_join_policy TO 'prefer-distributed'; +SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) +WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); +SET citus.local_table_join_policy TO 'auto'; +-- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. + +RESET citus.log_remote_commands; +RESET citus.next_shard_id; +RESET citus.shard_count; +RESET citus.shard_replication_factor; + DROP SCHEMA pg17 CASCADE; DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; From e4d48dc5da057052ceeffe042eaa6bc42b6dc62f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:47:36 +0300 Subject: [PATCH 087/155] Remove redundant normalize (#7794) Redundant from this commit https://github.com/citusdata/citus/commit/acd7b1e690c2a73b9ca5394df57a272eebbfae02 --- src/test/regress/bin/normalize.sed | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index c110c0c49..59bf6d97c 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -331,7 +331,3 @@ s/\| CHECK ([a-zA-Z])(.*)/| CHECK \(\1\2\)/g # supported Postgres version. /DEBUG: drop auto-cascades to type [a-zA-Z_]*.pg_temp_[0-9]*/d - -# Normalize subquery condition in debug logs to standardize across PostgreSQL versions #7731 -# https://github.com/postgres/postgres/commit/b262ad44 -s/for subquery SELECT id FROM local_dist_join_mixed\.local WHERE \(id IS NOT NULL\)/for subquery SELECT id FROM local_dist_join_mixed.local WHERE true/g From 61d11ec2d4c9a2534d60d2aa167be7b7e54c8c4d Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:59:09 +0300 Subject: [PATCH 088/155] PG17 Compatibility - Fix HideCitusDependentObjects function (#7796) There is a crash when running vanilla tests because of the `citus.hide_citus_dependent_objects` GUC. We turn on this GUC only for the pg vanilla tests. This GUC runs the following function `HideCitusDependentObjectsOnQueriesOfPgMetaTables`. This function doesn't take into account the new `mergeJoinCondition`. I rewrote the function such that it checks for merge join conditions as well. Relevant PG commit: https://github.com/postgres/postgres/commit/0294df2f1 The crash could be reproduced locally like the following: ```SQL SET citus.hide_citus_dependent_objects TO on; CREATE OR REPLACE FUNCTION pg_catalog.is_citus_depended_object(oid,oid) RETURNS bool LANGUAGE C AS 'citus', $$is_citus_depended_object$$; -- try a system catalog MERGE INTO pg_class c USING (SELECT 'pg_depend'::regclass AS oid) AS j ON j.oid = c.oid WHEN MATCHED THEN UPDATE SET reltuples = reltuples + 1 RETURNING j.oid; CREATE VIEW classv AS SELECT * FROM pg_class; MERGE INTO classv c USING pg_namespace n ON n.oid = c.relnamespace WHEN MATCHED AND c.oid = 'pg_depend'::regclass THEN UPDATE SET reltuples = reltuples - 1 RETURNING c.oid; -- crash happens here ``` --- .../distributed/utils/citus_depended_object.c | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/utils/citus_depended_object.c b/src/backend/distributed/utils/citus_depended_object.c index 3babf76f0..bc14490b5 100644 --- a/src/backend/distributed/utils/citus_depended_object.c +++ b/src/backend/distributed/utils/citus_depended_object.c @@ -243,12 +243,24 @@ HideCitusDependentObjectsOnQueriesOfPgMetaTables(Node *node, void *context) if (OidIsValid(metaTableOid)) { + bool mergeJoinCondition = false; +#if PG_VERSION_NUM >= PG_VERSION_17 + + /* + * In Postgres 17, the query tree has a specific field for the merge condition. + * So we shouldn't modify the jointree, but rather the mergeJoinCondition here + * Relevant PG17 commit: 0294df2f1 + */ + mergeJoinCondition = query->mergeJoinCondition; +#endif + /* * We found a valid pg meta class in query, * so we assert below conditions. */ - Assert(query->jointree != NULL); - Assert(query->jointree->fromlist != NULL); + Assert(mergeJoinCondition || + (query->jointree != NULL && + query->jointree->fromlist != NULL)); Node *citusDependentObjExpr = CreateCitusDependentObjectExpr(varno, metaTableOid); @@ -257,8 +269,18 @@ HideCitusDependentObjectsOnQueriesOfPgMetaTables(Node *node, void *context) * We do not use security quals because a postgres vanilla test fails * with a change of order for its result. */ - query->jointree->quals = make_and_qual( - query->jointree->quals, citusDependentObjExpr); + if (!mergeJoinCondition) + { + query->jointree->quals = make_and_qual( + query->jointree->quals, citusDependentObjExpr); + } + else + { +#if PG_VERSION_NUM >= PG_VERSION_17 + query->mergeJoinCondition = make_and_qual( + query->mergeJoinCondition, citusDependentObjExpr); +#endif + } } MemoryContextSwitchTo(originalContext); From f123632f85457535ca6219c607d6ebefaa613449 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Fri, 20 Dec 2024 20:13:48 +0300 Subject: [PATCH 089/155] Fix pg17 test (#7797) Broken from this commit https://github.com/citusdata/citus/commit/e3db3751495cf238488aad156609b39fd74793eb https://github.com/citusdata/citus/actions/runs/12429202397/attempts/1#summary-34702334056 --- src/test/regress/expected/pg17.out | 204 ++--------------------------- src/test/regress/sql/pg17.sql | 29 ++-- 2 files changed, 20 insertions(+), 213 deletions(-) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 3cd86b4f2..d9d05196a 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -955,92 +955,12 @@ DROP TABLE pt_3; DROP TABLE pt_4; DROP TABLE alt_test; -- End of partition with identity columns testing -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; -DROP SCHEMA pg17 CASCADE; -NOTICE: drop cascades to 3 other objects -DETAIL: drop cascades to function fake_am_handler(internal) -drop cascades to access method fake_am -- Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. -- Enable DEBUG-level logging to capture detailed execution plans -SET client_min_messages TO DEBUG1; -- Create the tables CREATE TABLE postgres_table (key int, value text, value_2 jsonb); CREATE TABLE reference_table (key int, value text, value_2 jsonb); SELECT create_reference_table('reference_table'); -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240024, 'pg17', 'ALTER TABLE pg17.reference_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE TABLE pg17.reference_table (key integer, value text, value_2 jsonb) USING heap -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing ALTER TABLE pg17.reference_table OWNER TO postgres -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing ALTER TABLE pg17.reference_table OWNER TO postgres -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.reference_table') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.reference_table') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.reference_table'::regclass, 'n', NULL, 1390013, 't') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.reference_table'::regclass, 'n', NULL, 1390013, 't') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.reference_table'::regclass, 20240024, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.reference_table'::regclass, 20240024, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (20240024, 0, 0, 3841), (20240024, 0, 14, 3842), (20240024, 0, 22, 3843)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (20240024, 0, 0, 3841), (20240024, 0, 14, 3842), (20240024, 0, 22, 3843)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'reference_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'reference_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT lock_shard_resources(3, ARRAY[20240024]) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx create_reference_table --------------------------------------------------------------------- @@ -1048,118 +968,24 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx CREATE TABLE distributed_table (key int, value text, value_2 jsonb); SELECT create_distributed_table('distributed_table', 'key'); -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240025, 'pg17', 'CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (20240025, 'pg17', 'ALTER TABLE pg17.distributed_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE TABLE pg17.distributed_table (key integer, value text, value_2 jsonb) USING heap -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing ALTER TABLE pg17.distributed_table OWNER TO postgres -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing ALTER TABLE pg17.distributed_table OWNER TO postgres -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.distributed_table') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_create_truncate_trigger('pg17.distributed_table') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.distributed_table'::regclass, 'h', 'key', 1400006, 's') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT citus_internal_add_partition_metadata ('pg17.distributed_table'::regclass, 'h', 'key', 1400006, 's') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.distributed_table'::regclass, 20240025, 't'::"char", '-2147483648', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('pg17.distributed_table'::regclass, 20240025, 't'::"char", '-2147483648', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (xxxxxx, xxxxxx, xxxxxx, xxxxxx)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (xxxxxx, xxxxxx, xxxxxx, xxxxxx)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'distributed_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['pg17', 'distributed_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx create_distributed_table --------------------------------------------------------------------- (1 row) -- Insert test data -INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; -INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT lock_shard_resources(3, ARRAY[20240024]) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: executing the copy locally for shard xxxxx -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COPY pg17.reference_table_20240024 (key, value, value_2) FROM STDIN WITH (format 'binary') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COPY pg17.reference_table_20240024 (key, value, value_2) FROM STDIN WITH (format 'binary') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COPY pg17.distributed_table_20240025 (key, value, value_2) FROM STDIN WITH (format 'binary') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_xx_xx_xx_xx' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; +INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; +INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; -- Set local table join policy to auto before running the tests SET citus.local_table_join_policy TO 'auto'; +SET client_min_messages TO DEBUG1; -- Correlated sublinks are supported in PostgreSQL 17 SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); DEBUG: Wrapping relation "postgres_table" to a subquery DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.postgres_table WHERE (key OPERATOR(pg_catalog.=) 5) DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (pg17.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COPY "21_1" FROM STDIN WITH (format result) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT count(*) AS count FROM (pg17.distributed_table_20240025 d1(key, value, value_2) JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table_20240025 distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 1 @@ -1170,14 +996,6 @@ WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = DEBUG: Wrapping relation "postgres_table" to a subquery DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.postgres_table WHERE (key OPERATOR(pg_catalog.=) 5) DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (pg17.distributed_table d1 JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COPY "22_1" FROM STDIN WITH (format result) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT count(*) AS count FROM (pg17.distributed_table_20240025 d1(key, value, value_2) JOIN (SELECT postgres_table_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) postgres_table_1) postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table_20240025 distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 1 @@ -1191,22 +1009,16 @@ DEBUG: generating subplan XXX_1 for subquery SELECT key FROM pg17.distributed_t DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT d1_1.key, NULL::text AS value, NULL::jsonb AS value_2 FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) d1_1) d1 JOIN pg17.postgres_table USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table.key FROM pg17.distributed_table WHERE ((d1.key OPERATOR(pg_catalog.=) distributed_table.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 5)))) ERROR: direct joins between distributed and local tables are not supported HINT: Use CTE's or subqueries to select from local tables and use them in joins -SET citus.local_table_join_policy TO 'auto'; +RESET citus.local_table_join_policy; +RESET client_min_messages; +DROP TABLE reference_table; -- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. -RESET citus.log_remote_commands; -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; DROP SCHEMA pg17 CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: A command for a distributed schema is run. To make sure subsequent commands see the schema correctly we need to make sure to use only one connection for all future commands -NOTICE: drop cascades to 7 other objects +NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to function fake_am_handler(internal) drop cascades to access method fake_am drop cascades to table dist_test drop cascades to table postgres_table -drop cascades to table reference_table -drop cascades to table reference_table_20240024 drop cascades to table distributed_table DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 35d0e199c..5326ede72 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -464,28 +464,25 @@ DROP TABLE alt_test; -- End of partition with identity columns testing -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; - -DROP SCHEMA pg17 CASCADE; - -- Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. -- Enable DEBUG-level logging to capture detailed execution plans -RESET citus.log_remote_commands; -SET client_min_messages TO DEBUG1; + -- Create the tables CREATE TABLE postgres_table (key int, value text, value_2 jsonb); CREATE TABLE reference_table (key int, value text, value_2 jsonb); SELECT create_reference_table('reference_table'); CREATE TABLE distributed_table (key int, value text, value_2 jsonb); SELECT create_distributed_table('distributed_table', 'key'); + -- Insert test data -INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; -INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; -INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 100) i; +INSERT INTO postgres_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; +INSERT INTO reference_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; +INSERT INTO distributed_table SELECT i, i::varchar(256), '{}'::jsonb FROM generate_series(1, 10) i; + -- Set local table join policy to auto before running the tests SET citus.local_table_join_policy TO 'auto'; +SET client_min_messages TO DEBUG1; + -- Correlated sublinks are supported in PostgreSQL 17 SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); @@ -496,13 +493,11 @@ WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = SET citus.local_table_join_policy TO 'prefer-distributed'; SELECT COUNT(*) FROM distributed_table d1 JOIN postgres_table USING (key) WHERE d1.key IN (SELECT key FROM distributed_table WHERE d1.key = key AND key = 5); -SET citus.local_table_join_policy TO 'auto'; --- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. -RESET citus.log_remote_commands; -RESET citus.next_shard_id; -RESET citus.shard_count; -RESET citus.shard_replication_factor; +RESET citus.local_table_join_policy; +RESET client_min_messages; +DROP TABLE reference_table; +-- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. DROP SCHEMA pg17 CASCADE; DROP ROLE regress_maintain; From 2d843cb382ddd033008e266c8da1f9d490bfa6e8 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Mon, 23 Dec 2024 13:30:54 +0300 Subject: [PATCH 090/155] PG17 compatibility: Fix Isolation Test Failure in isolation_multiuser_locking (#7714) This PR enhances `isolation_multiuser_locking.spec` test compatibility across multiple PostgreSQL versions by handling differences in error messages and behavior. Key updates include: - **Error Message Handling:** Adjustments to manage version-specific error messages, ensuring consistent test results. - Modified to address variations in locking behavior across PostgreSQL versions, ensuring test stability in multiuser scenarios. - **REINDEX Behavior Adjustment**: This PR accounts for a behavioral change introduced in PostgreSQL by commit ecb0fd337, which alters how REINDEX interacts with system catalogs. https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337 --------- Co-authored-by: Mehmet YILMAZ --- .../expected/isolation_multiuser_locking.out | 231 ++++++----- .../isolation_multiuser_locking_0.out | 359 ++++++++++++++++++ .../spec/isolation_multiuser_locking.spec | 12 + 3 files changed, 486 insertions(+), 116 deletions(-) create mode 100644 src/test/regress/expected/isolation_multiuser_locking_0.out diff --git a/src/test/regress/expected/isolation_multiuser_locking.out b/src/test/regress/expected/isolation_multiuser_locking.out index 943d579a4..feb5d0189 100644 --- a/src/test/regress/expected/isolation_multiuser_locking.out +++ b/src/test/regress/expected/isolation_multiuser_locking.out @@ -7,31 +7,31 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s2-reindex: - REINDEX TABLE test_table; + REINDEX TABLE test_table; -ERROR: must be owner of table test_table +ERROR: permission denied for table test_table step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s2-commit: - COMMIT; + COMMIT; step s1-commit: - COMMIT; + COMMIT; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s2-reindex s1-insert s2-insert s2-commit s1-commit @@ -41,39 +41,38 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s2-reindex: - REINDEX TABLE test_table; + REINDEX TABLE test_table; -ERROR: must be owner of table test_table step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; + +step s2-insert: + UPDATE test_table SET column2 = 2; -step s2-insert: - UPDATE test_table SET column2 = 2; - -ERROR: current transaction is aborted, commands ignored until end of transaction block step s2-commit: - COMMIT; + COMMIT; +step s1-insert: <... completed> step s1-commit: - COMMIT; + COMMIT; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-reindex s2-insert s1-insert s1-commit s2-commit @@ -83,38 +82,38 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s1-reindex: - REINDEX TABLE test_table; + REINDEX TABLE test_table; step s2-insert: - UPDATE test_table SET column2 = 2; + UPDATE test_table SET column2 = 2; step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s1-commit: - COMMIT; + COMMIT; step s2-insert: <... completed> step s2-commit: - COMMIT; + COMMIT; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-begin s2-begin s2-index s1-insert s2-commit s1-commit s2-drop-index @@ -124,34 +123,34 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s2-index: - CREATE INDEX test_index ON test_table(column1); + CREATE INDEX test_index ON test_table(column1); ERROR: must be owner of table test_table step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s2-commit: - COMMIT; + COMMIT; step s1-commit: - COMMIT; + COMMIT; step s2-drop-index: - DROP INDEX IF EXISTS test_index; + DROP INDEX IF EXISTS test_index; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s2-insert s1-index s2-insert s2-commit s1-commit s1-drop-index @@ -161,41 +160,41 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s2-insert: - UPDATE test_table SET column2 = 2; + UPDATE test_table SET column2 = 2; step s1-index: - CREATE INDEX test_index ON test_table(column1); + CREATE INDEX test_index ON test_table(column1); step s2-insert: - UPDATE test_table SET column2 = 2; + UPDATE test_table SET column2 = 2; step s2-commit: - COMMIT; + COMMIT; step s1-index: <... completed> step s1-commit: - COMMIT; + COMMIT; step s1-drop-index: - DROP INDEX IF EXISTS test_index; + DROP INDEX IF EXISTS test_index; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-index s2-index s1-insert s1-commit s2-commit s1-drop-index s2-drop-index @@ -205,44 +204,44 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s1-index: - CREATE INDEX test_index ON test_table(column1); + CREATE INDEX test_index ON test_table(column1); step s2-index: - CREATE INDEX test_index ON test_table(column1); + CREATE INDEX test_index ON test_table(column1); ERROR: must be owner of table test_table step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s1-commit: - COMMIT; + COMMIT; step s2-commit: - COMMIT; + COMMIT; step s1-drop-index: - DROP INDEX IF EXISTS test_index; + DROP INDEX IF EXISTS test_index; step s2-drop-index: - DROP INDEX IF EXISTS test_index; + DROP INDEX IF EXISTS test_index; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-begin s2-begin s2-truncate s1-insert s2-commit s1-commit @@ -252,31 +251,31 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s2-truncate: - TRUNCATE test_table; + TRUNCATE test_table; ERROR: permission denied for table test_table step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s2-commit: - COMMIT; + COMMIT; step s1-commit: - COMMIT; + COMMIT; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-truncate s2-insert s1-insert s1-commit s2-commit @@ -286,38 +285,38 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s1-truncate: - TRUNCATE test_table; + TRUNCATE test_table; step s2-insert: - UPDATE test_table SET column2 = 2; + UPDATE test_table SET column2 = 2; step s1-insert: - UPDATE test_table SET column2 = 1; + UPDATE test_table SET column2 = 1; step s1-commit: - COMMIT; + COMMIT; step s2-insert: <... completed> step s2-commit: - COMMIT; + COMMIT; starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-truncate s2-truncate s1-commit s2-commit @@ -327,33 +326,33 @@ create_distributed_table (1 row) step s1-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s2-no-connection-cache: - SET citus.max_cached_conns_per_worker to 0; + SET citus.max_cached_conns_per_worker to 0; step s1-grant: - SET ROLE test_user_1; - GRANT ALL ON test_table TO test_user_2; + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; step s1-begin: - BEGIN; - SET ROLE test_user_1; + BEGIN; + SET ROLE test_user_1; step s2-begin: - BEGIN; - SET ROLE test_user_2; + BEGIN; + SET ROLE test_user_2; step s1-truncate: - TRUNCATE test_table; + TRUNCATE test_table; step s2-truncate: - TRUNCATE test_table; + TRUNCATE test_table; step s1-commit: - COMMIT; + COMMIT; step s2-truncate: <... completed> step s2-commit: - COMMIT; + COMMIT; diff --git a/src/test/regress/expected/isolation_multiuser_locking_0.out b/src/test/regress/expected/isolation_multiuser_locking_0.out new file mode 100644 index 000000000..af1633161 --- /dev/null +++ b/src/test/regress/expected/isolation_multiuser_locking_0.out @@ -0,0 +1,359 @@ +Parsed test spec with 2 sessions + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-begin s2-begin s2-reindex s1-insert s2-commit s1-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s2-reindex: + REINDEX TABLE test_table; + +ERROR: must be owner of table test_table +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s2-commit: + COMMIT; + +step s1-commit: + COMMIT; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s2-reindex s1-insert s2-insert s2-commit s1-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s2-reindex: + REINDEX TABLE test_table; + +ERROR: must be owner of table test_table +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s2-insert: + UPDATE test_table SET column2 = 2; + +ERROR: current transaction is aborted, commands ignored until end of transaction block +step s2-commit: + COMMIT; + +step s1-commit: + COMMIT; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-reindex s2-insert s1-insert s1-commit s2-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s1-reindex: + REINDEX TABLE test_table; + +step s2-insert: + UPDATE test_table SET column2 = 2; + +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s1-commit: + COMMIT; + +step s2-insert: <... completed> +step s2-commit: + COMMIT; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-begin s2-begin s2-index s1-insert s2-commit s1-commit s2-drop-index +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s2-index: + CREATE INDEX test_index ON test_table(column1); + +ERROR: must be owner of table test_table +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s2-commit: + COMMIT; + +step s1-commit: + COMMIT; + +step s2-drop-index: + DROP INDEX IF EXISTS test_index; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s2-insert s1-index s2-insert s2-commit s1-commit s1-drop-index +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s2-insert: + UPDATE test_table SET column2 = 2; + +step s1-index: + CREATE INDEX test_index ON test_table(column1); + +step s2-insert: + UPDATE test_table SET column2 = 2; + +step s2-commit: + COMMIT; + +step s1-index: <... completed> +step s1-commit: + COMMIT; + +step s1-drop-index: + DROP INDEX IF EXISTS test_index; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-index s2-index s1-insert s1-commit s2-commit s1-drop-index s2-drop-index +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s1-index: + CREATE INDEX test_index ON test_table(column1); + +step s2-index: + CREATE INDEX test_index ON test_table(column1); + +ERROR: must be owner of table test_table +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s1-commit: + COMMIT; + +step s2-commit: + COMMIT; + +step s1-drop-index: + DROP INDEX IF EXISTS test_index; + +step s2-drop-index: + DROP INDEX IF EXISTS test_index; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-begin s2-begin s2-truncate s1-insert s2-commit s1-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s2-truncate: + TRUNCATE test_table; + +ERROR: permission denied for table test_table +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s2-commit: + COMMIT; + +step s1-commit: + COMMIT; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-truncate s2-insert s1-insert s1-commit s2-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s1-truncate: + TRUNCATE test_table; + +step s2-insert: + UPDATE test_table SET column2 = 2; + +step s1-insert: + UPDATE test_table SET column2 = 1; + +step s1-commit: + COMMIT; + +step s2-insert: <... completed> +step s2-commit: + COMMIT; + + +starting permutation: s1-no-connection-cache s2-no-connection-cache s1-grant s1-begin s2-begin s1-truncate s2-truncate s1-commit s2-commit +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s1-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s2-no-connection-cache: + SET citus.max_cached_conns_per_worker to 0; + +step s1-grant: + SET ROLE test_user_1; + GRANT ALL ON test_table TO test_user_2; + +step s1-begin: + BEGIN; + SET ROLE test_user_1; + +step s2-begin: + BEGIN; + SET ROLE test_user_2; + +step s1-truncate: + TRUNCATE test_table; + +step s2-truncate: + TRUNCATE test_table; + +step s1-commit: + COMMIT; + +step s2-truncate: <... completed> +step s2-commit: + COMMIT; + \ No newline at end of file diff --git a/src/test/regress/spec/isolation_multiuser_locking.spec b/src/test/regress/spec/isolation_multiuser_locking.spec index 2d5c1cc03..e892cb636 100644 --- a/src/test/regress/spec/isolation_multiuser_locking.spec +++ b/src/test/regress/spec/isolation_multiuser_locking.spec @@ -1,3 +1,15 @@ +// Two alternative test outputs: +// isolation_multiuser_locking_0.out for PG16 and before +// isolation_multiuser_locking.out for PG17 +// +// REINDEX TABLE now requires table ownership (PostgreSQL commit ecb0fd337). +// UPDATE statements include to reflect new lock waiting behavior. +// Previous behavior: Transactions failed with "current transaction is aborted". +// New behavior: Transactions wait for locks, ensuring proper isolation. +// <... completed> tracks transaction states for clarity with locking changes. +// Reference: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=ecb0fd337 + + setup { SET citus.max_cached_conns_per_worker to 0; From aa8ed4c60b2606f19f29cf9798cf190ac22c85cc Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:15:15 +0300 Subject: [PATCH 091/155] Bump postgres versions in CI and dev: 14.14, 15.9, 16.5 (#7779) Upgrade postgres versions to: - 14.14 - 15.9 - 16.5 Depends on https://github.com/citusdata/the-process/pull/163 We had some errors with the latest minors, so this is a 2-level bump for now. --- .devcontainer/Dockerfile | 8 ++++---- .github/workflows/build_and_test.yml | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 33bba98d5..7dc75abd4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -68,7 +68,7 @@ USER citus # build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions FROM base AS pg14 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.12 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.14 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -80,7 +80,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg15 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.7 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.9 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -92,7 +92,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg16 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.3 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.5 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -211,7 +211,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 16.3 +RUN pgenv switch 16.5 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 54721c0e0..0e077e82c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,14 +26,14 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.18" - sql_snapshot_pg_version: "16.3" - image_suffix: "-v13fd57c" - pg14_version: '{ "major": "14", "full": "14.12" }' - pg15_version: '{ "major": "15", "full": "15.7" }' - pg16_version: '{ "major": "16", "full": "16.3" }' - upgrade_pg_versions: "14.12-15.7-16.3" + sql_snapshot_pg_version: "16.5" + image_suffix: "-v1d9d7d7" + pg14_version: '{ "major": "14", "full": "14.14" }' + pg15_version: '{ "major": "15", "full": "15.9" }' + pg16_version: '{ "major": "16", "full": "16.5" }' + upgrade_pg_versions: "14.14-15.9-16.5" steps: - # Since GHA jobs needs at least one step we use a noop step here. + # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters run: echo 'noop' check-sql-snapshots: From 351b1ca63c8526e0d2892eb4fbddace02e48c60a Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Mon, 23 Dec 2024 15:24:46 +0300 Subject: [PATCH 092/155] PG17 compatibility: Fix Test Failure in multi_alter_table_add_const (#7733) In earlier versions of PostgreSQL, exclusion constraints were not allowed on partitioned tables. This is why the error in your regression test (ERROR: exclusion constraints are not supported on partitioned tables) was raised in PostgreSQL 16. In PostgreSQL 17, exclusion constraints are now allowed on partitioned tables, which is why the error no longer appears when you attempt to add an exclusion constraint. The constraint exclusion mechanism, described in the documentation, relies on CHECK constraints to decide which partitions or child tables need to be queried. [CHECK constraints](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-CONSTRAINT-EXCLUSION) ```diff -- Check "ADD EXCLUDE" errors out for partitioned table since the postgres does not allow it ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD EXCLUDE(partition_col WITH =); -ERROR: exclusion constraints are not supported on partitioned tables -- Check "ADD CHECK" SET client_min_messages TO DEBUG1; ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD CHECK (dist_col > 0); DEBUG: the constraint name on the shards of the partition is too long, switching to sequential and local execution mode to prevent self deadlocks: longlonglonglonglonglonglonglonglonglonglonglo_537570f5_5_check DEBUG: verifying table "longlonglonglonglonglonglonglonglonglonglonglonglonglonglongabc" DEBUG: verifying table "p1" RESET client_min_messages; SELECT con.conname FROM pg_catalog.pg_constraint con INNER JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace WHERE rel.relname = 'citus_local_partitioned_table'; conname -------------------------------------------------- + citus_local_partitioned_table_partition_col_excl citus_local_partitioned_table_check -(1 row) +(2 rows) ``` --- ...ter_table_add_constraints_without_name.out | 3 - src/test/regress/expected/pg17.out | 104 ++++++++++++++++++ ...ter_table_add_constraints_without_name.sql | 3 - src/test/regress/sql/pg17.sql | 75 +++++++++++++ 4 files changed, 179 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out b/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out index 0b048946c..0c268264d 100644 --- a/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out +++ b/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out @@ -1119,9 +1119,6 @@ SELECT con.conname \c - - :master_host :master_port ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table DROP CONSTRAINT citus_local_partitioned_table_partition_col_key; --- Check "ADD EXCLUDE" errors out for partitioned table since the postgres does not allow it -ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD EXCLUDE(partition_col WITH =); -ERROR: exclusion constraints are not supported on partitioned tables -- Check "ADD CHECK" SET client_min_messages TO DEBUG1; ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD CHECK (dist_col > 0); diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index d9d05196a..fbe8ebbe1 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1013,6 +1013,110 @@ RESET citus.local_table_join_policy; RESET client_min_messages; DROP TABLE reference_table; -- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. +-- Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment +-- Step 1: Create a distributed partitioned table +\c - - :master_host :master_port +SET search_path TO pg17; +CREATE TABLE distributed_partitioned_table ( + id serial NOT NULL, + partition_col int NOT NULL, + PRIMARY KEY (id, partition_col) +) PARTITION BY RANGE (partition_col); +-- Add partitions to the distributed partitioned table +CREATE TABLE distributed_partitioned_table_p1 PARTITION OF distributed_partitioned_table +FOR VALUES FROM (1) TO (100); +CREATE TABLE distributed_partitioned_table_p2 PARTITION OF distributed_partitioned_table +FOR VALUES FROM (100) TO (200); +-- Distribute the table +SELECT create_distributed_table('distributed_partitioned_table', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 2: Create a partitioned Citus local table +CREATE TABLE local_partitioned_table ( + id serial NOT NULL, + partition_col int NOT NULL, + PRIMARY KEY (id, partition_col) +) PARTITION BY RANGE (partition_col); +-- Add partitions to the local partitioned table +CREATE TABLE local_partitioned_table_p1 PARTITION OF local_partitioned_table +FOR VALUES FROM (1) TO (100); +CREATE TABLE local_partitioned_table_p2 PARTITION OF local_partitioned_table +FOR VALUES FROM (100) TO (200); +SELECT citus_add_local_table_to_metadata('local_partitioned_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- Verify the Citus tables +SELECT table_name, citus_table_type FROM pg_catalog.citus_tables +WHERE table_name::regclass::text LIKE '%_partitioned_table' ORDER BY 1; +ERROR: relation "pg_catalog.citus_tables" does not exist +-- Step 3: Add an exclusion constraint with a name to the distributed partitioned table +ALTER TABLE distributed_partitioned_table ADD CONSTRAINT dist_exclude_named EXCLUDE USING btree (id WITH =, partition_col WITH =); +-- Step 4: Verify propagation of exclusion constraint to worker nodes +\c - - :public_worker_1_host :worker_1_port +SET search_path TO pg17; +SELECT conname FROM pg_constraint WHERE conrelid = 'pg17.distributed_partitioned_table'::regclass AND conname = 'dist_exclude_named'; + conname +--------------------------------------------------------------------- + dist_exclude_named +(1 row) + +-- Step 5: Add an exclusion constraint with a name to the Citus local partitioned table +\c - - :master_host :master_port +SET search_path TO pg17; +ALTER TABLE local_partitioned_table ADD CONSTRAINT local_exclude_named EXCLUDE USING btree (partition_col WITH =); +-- Step 6: Verify the exclusion constraint on the local partitioned table +SELECT conname, contype FROM pg_constraint WHERE conname = 'local_exclude_named' AND contype = 'x'; + conname | contype +--------------------------------------------------------------------- + local_exclude_named | x +(1 row) + +-- Step 7: Add exclusion constraints without names to both tables +ALTER TABLE distributed_partitioned_table ADD EXCLUDE USING btree (id WITH =, partition_col WITH =); +ALTER TABLE local_partitioned_table ADD EXCLUDE USING btree (partition_col WITH =); +-- Step 8: Verify the unnamed exclusion constraints were added +SELECT conname, contype FROM pg_constraint WHERE conrelid = 'local_partitioned_table'::regclass AND contype = 'x'; + conname | contype +--------------------------------------------------------------------- + local_exclude_named | x + local_partitioned_table_partition_col_excl | x +(2 rows) + +\c - - :public_worker_1_host :worker_1_port +SET search_path TO pg17; +SELECT conname, contype FROM pg_constraint WHERE conrelid = 'pg17.distributed_partitioned_table'::regclass AND contype = 'x'; + conname | contype +--------------------------------------------------------------------- + dist_exclude_named | x + distributed_partitioned_table_id_partition_col_excl | x +(2 rows) + +-- Step 9: Drop the exclusion constraints from both tables +\c - - :master_host :master_port +SET search_path TO pg17; +ALTER TABLE distributed_partitioned_table DROP CONSTRAINT dist_exclude_named; +ALTER TABLE local_partitioned_table DROP CONSTRAINT local_exclude_named; +-- Step 10: Verify the constraints were dropped +SELECT * FROM pg_constraint WHERE conname = 'dist_exclude_named' AND contype = 'x'; + oid | conname | connamespace | contype | condeferrable | condeferred | convalidated | conrelid | contypid | conindid | conparentid | confrelid | confupdtype | confdeltype | confmatchtype | conislocal | coninhcount | connoinherit | conkey | confkey | conpfeqop | conppeqop | conffeqop | confdelsetcols | conexclop | conbin +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM pg_constraint WHERE conname = 'local_exclude_named' AND contype = 'x'; + oid | conname | connamespace | contype | condeferrable | condeferred | convalidated | conrelid | contypid | conindid | conparentid | confrelid | confupdtype | confdeltype | confmatchtype | conislocal | coninhcount | connoinherit | conkey | confkey | conpfeqop | conppeqop | conffeqop | confdelsetcols | conexclop | conbin +--------------------------------------------------------------------- +(0 rows) + +-- Step 11: Clean up - Drop the tables +DROP TABLE distributed_partitioned_table CASCADE; +DROP TABLE local_partitioned_table CASCADE; +-- End of Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment DROP SCHEMA pg17 CASCADE; NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to function fake_am_handler(internal) diff --git a/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql b/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql index 700e37f6e..f5fd653f5 100644 --- a/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql +++ b/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql @@ -785,9 +785,6 @@ SELECT con.conname \c - - :master_host :master_port ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table DROP CONSTRAINT citus_local_partitioned_table_partition_col_key; --- Check "ADD EXCLUDE" errors out for partitioned table since the postgres does not allow it -ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD EXCLUDE(partition_col WITH =); - -- Check "ADD CHECK" SET client_min_messages TO DEBUG1; ALTER TABLE AT_AddConstNoName.citus_local_partitioned_table ADD CHECK (dist_col > 0); diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 5326ede72..f0c6d5b65 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -499,6 +499,81 @@ RESET client_min_messages; DROP TABLE reference_table; -- End for Correlated sublinks are now supported as of PostgreSQL 17, resolving issue #4470. +-- Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment +-- Step 1: Create a distributed partitioned table +\c - - :master_host :master_port +SET search_path TO pg17; +CREATE TABLE distributed_partitioned_table ( + id serial NOT NULL, + partition_col int NOT NULL, + PRIMARY KEY (id, partition_col) +) PARTITION BY RANGE (partition_col); +-- Add partitions to the distributed partitioned table +CREATE TABLE distributed_partitioned_table_p1 PARTITION OF distributed_partitioned_table +FOR VALUES FROM (1) TO (100); +CREATE TABLE distributed_partitioned_table_p2 PARTITION OF distributed_partitioned_table +FOR VALUES FROM (100) TO (200); +-- Distribute the table +SELECT create_distributed_table('distributed_partitioned_table', 'id'); + +-- Step 2: Create a partitioned Citus local table +CREATE TABLE local_partitioned_table ( + id serial NOT NULL, + partition_col int NOT NULL, + PRIMARY KEY (id, partition_col) +) PARTITION BY RANGE (partition_col); +-- Add partitions to the local partitioned table +CREATE TABLE local_partitioned_table_p1 PARTITION OF local_partitioned_table +FOR VALUES FROM (1) TO (100); +CREATE TABLE local_partitioned_table_p2 PARTITION OF local_partitioned_table +FOR VALUES FROM (100) TO (200); +SELECT citus_add_local_table_to_metadata('local_partitioned_table'); + +-- Verify the Citus tables +SELECT table_name, citus_table_type FROM pg_catalog.citus_tables +WHERE table_name::regclass::text LIKE '%_partitioned_table' ORDER BY 1; + +-- Step 3: Add an exclusion constraint with a name to the distributed partitioned table +ALTER TABLE distributed_partitioned_table ADD CONSTRAINT dist_exclude_named EXCLUDE USING btree (id WITH =, partition_col WITH =); + +-- Step 4: Verify propagation of exclusion constraint to worker nodes +\c - - :public_worker_1_host :worker_1_port +SET search_path TO pg17; +SELECT conname FROM pg_constraint WHERE conrelid = 'pg17.distributed_partitioned_table'::regclass AND conname = 'dist_exclude_named'; + +-- Step 5: Add an exclusion constraint with a name to the Citus local partitioned table +\c - - :master_host :master_port +SET search_path TO pg17; +ALTER TABLE local_partitioned_table ADD CONSTRAINT local_exclude_named EXCLUDE USING btree (partition_col WITH =); + +-- Step 6: Verify the exclusion constraint on the local partitioned table +SELECT conname, contype FROM pg_constraint WHERE conname = 'local_exclude_named' AND contype = 'x'; + +-- Step 7: Add exclusion constraints without names to both tables +ALTER TABLE distributed_partitioned_table ADD EXCLUDE USING btree (id WITH =, partition_col WITH =); +ALTER TABLE local_partitioned_table ADD EXCLUDE USING btree (partition_col WITH =); + +-- Step 8: Verify the unnamed exclusion constraints were added +SELECT conname, contype FROM pg_constraint WHERE conrelid = 'local_partitioned_table'::regclass AND contype = 'x'; +\c - - :public_worker_1_host :worker_1_port +SET search_path TO pg17; +SELECT conname, contype FROM pg_constraint WHERE conrelid = 'pg17.distributed_partitioned_table'::regclass AND contype = 'x'; + +-- Step 9: Drop the exclusion constraints from both tables +\c - - :master_host :master_port +SET search_path TO pg17; +ALTER TABLE distributed_partitioned_table DROP CONSTRAINT dist_exclude_named; +ALTER TABLE local_partitioned_table DROP CONSTRAINT local_exclude_named; + +-- Step 10: Verify the constraints were dropped +SELECT * FROM pg_constraint WHERE conname = 'dist_exclude_named' AND contype = 'x'; +SELECT * FROM pg_constraint WHERE conname = 'local_exclude_named' AND contype = 'x'; + +-- Step 11: Clean up - Drop the tables +DROP TABLE distributed_partitioned_table CASCADE; +DROP TABLE local_partitioned_table CASCADE; +-- End of Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment + DROP SCHEMA pg17 CASCADE; DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; From 743f0ebb75d09e7d9d961221352d3157ab391a92 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 24 Dec 2024 11:40:59 +0300 Subject: [PATCH 093/155] Bump Citus version into 13.0.0 (#7792) We are using `release-13.0` branch for both development and release, to deliver PG17 support in Citus. Afterwards, we will (probably) merge this branch into main. Some potential changes for main branch, after we are done working on release-13.0: - Merge changes from `release-13.0` to `main` - Figure out what changes were there on 12.2, move them to 13.1 version. In a nutshell: rename `12.1--12.2` to `13.0--13.1` and fix issues. - Set version to 13.1devel --- configure | 18 +++++++-------- configure.ac | 2 +- src/backend/distributed/citus.control | 2 +- .../distributed/sql/citus--12.1-1--13.0-1.sql | 3 +++ .../sql/downgrades/citus--13.0-1--12.1-1.sql | 2 ++ src/test/regress/citus_tests/common.py | 2 +- src/test/regress/citus_tests/config.py | 2 +- src/test/regress/expected/multi_extension.out | 22 ++++++++++++++++--- src/test/regress/sql/multi_extension.sql | 9 ++++++++ 9 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 src/backend/distributed/sql/citus--12.1-1--13.0-1.sql create mode 100644 src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql diff --git a/configure b/configure index 468c8be10..4bda6f37f 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 12.1.5. +# Generated by GNU Autoconf 2.69 for Citus 13.0.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='12.1.5' -PACKAGE_STRING='Citus 12.1.5' +PACKAGE_VERSION='13.0.0' +PACKAGE_STRING='Citus 13.0.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 12.1.5 to adapt to many kinds of systems. +\`configure' configures Citus 13.0.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 12.1.5:";; + short | recursive ) echo "Configuration of Citus 13.0.0:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 12.1.5 +Citus configure 13.0.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 12.1.5, which was +It was created by Citus $as_me 13.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 12.1.5, which was +This file was extended by Citus $as_me 13.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 12.1.5 +Citus config.status 13.0.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e3063ee70..6ecb13760 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [12.1.5]) +AC_INIT([Citus], [13.0.0]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control index 831955ad8..9c0202632 100644 --- a/src/backend/distributed/citus.control +++ b/src/backend/distributed/citus.control @@ -1,6 +1,6 @@ # Citus extension comment = 'Citus distributed database' -default_version = '12.1-1' +default_version = '13.0-1' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog diff --git a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql new file mode 100644 index 000000000..3a342a0fe --- /dev/null +++ b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql @@ -0,0 +1,3 @@ +-- citus--12.1-1--13.0-1.sql + +-- bump version to 13.0-1 diff --git a/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql b/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql new file mode 100644 index 000000000..006349990 --- /dev/null +++ b/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql @@ -0,0 +1,2 @@ +-- citus--13.0-1--12.1-1 +-- this is an empty downgrade path since citus--12.1-1--13.0-1.sql is empty diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 4d04d268c..05fd43fab 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -92,7 +92,7 @@ PG_MAJOR_VERSION = get_pg_major_version() OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 14: "10.2.0", 15: "11.1.5", - 16: "12.1devel", + 16: "12.1.5", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION] diff --git a/src/test/regress/citus_tests/config.py b/src/test/regress/citus_tests/config.py index 97f8a421f..d5ab5a59a 100644 --- a/src/test/regress/citus_tests/config.py +++ b/src/test/regress/citus_tests/config.py @@ -43,7 +43,7 @@ CITUS_ARBITRARY_TEST_DIR = "./tmp_citus_test" MASTER = "master" # This should be updated when citus version changes -MASTER_VERSION = "12.1" +MASTER_VERSION = "13.0" HOME = expanduser("~") diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 627b0b72f..2f27722f7 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1430,12 +1430,28 @@ SELECT * FROM multi_extension.print_extension_changes(); | function citus_schema_move(regnamespace,text,integer,citus.shard_transfer_mode) void (5 rows) +-- Test downgrade to 12.1-1 from 13.0-1 +ALTER EXTENSION citus UPDATE TO '13.0-1'; +ALTER EXTENSION citus UPDATE TO '12.1-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 13.0-1 +ALTER EXTENSION citus UPDATE TO '13.0-1'; +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version SHOW citus.version; citus.version --------------------------------------------------------------------- - 12.1.5 + 13.0.0 (1 row) -- ensure no unexpected objects were created outside pg_catalog @@ -1470,7 +1486,7 @@ DROP EXTENSION citus; DROP EXTENSION citus_columnar; CREATE EXTENSION citus VERSION '8.0-1'; ERROR: specified version incompatible with loaded Citus library -DETAIL: Loaded library requires 12.1, but 8.0-1 was specified. +DETAIL: Loaded library requires 13.0, but 8.0-1 was specified. HINT: If a newer library is present, restart the database and try the command again. -- Test non-distributed queries work even in version mismatch SET citus.enable_version_checks TO 'false'; @@ -1515,7 +1531,7 @@ ORDER BY 1; -- We should not distribute table in version mistmatch SELECT create_distributed_table('version_mismatch_table', 'column1'); ERROR: loaded Citus library version differs from installed extension version -DETAIL: Loaded library requires 12.1, but the installed extension version is 8.1-1. +DETAIL: Loaded library requires 13.0, but the installed extension version is 8.1-1. HINT: Run ALTER EXTENSION citus UPDATE and try again. -- This function will cause fail in next ALTER EXTENSION CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass) diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index 4e917ef36..e0c70fe28 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -658,6 +658,15 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '12.1-1'; SELECT * FROM multi_extension.print_extension_changes(); +-- Test downgrade to 12.1-1 from 13.0-1 +ALTER EXTENSION citus UPDATE TO '13.0-1'; +ALTER EXTENSION citus UPDATE TO '12.1-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); +-- Snapshot of state at 13.0-1 +ALTER EXTENSION citus UPDATE TO '13.0-1'; +SELECT * FROM multi_extension.print_extension_changes(); + DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version From 6fed917a530e49d9aff9558c263b76df9690aec7 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 24 Dec 2024 17:56:51 +0300 Subject: [PATCH 094/155] Adds PG17.1 support - Regression tests sanity (#7661) This is the final commit that adds PG17 compatibility with Citus's current capabilities. You can use Citus community, release-13.0 branch, with PG17.1. --------- Specifically, this commit: - Enables PG17 in the configure script. - Adds PG17 tests to CI using test images that have 17.1 - Fixes an upgrade test: see below for details In `citus_prepare_upgrade()`, don't drop any_value when upgrading from PG16+, because PG16+ has its own any_value function. Attempting to do so results in the error seen in [pg16-pg17 upgrade](https://github.com/citusdata/citus/actions/runs/11768444117/job/32778340003?pr=7661): ``` ERROR: cannot drop function any_value(anyelement) because it is required by the database system CONTEXT: SQL statement "DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement)" ``` When 16 becomes the minimum supported Postgres version, the drop statements can be removed. --------- Several PG17 Compatibility commits have been merged before this final one. All these subtasks are done https://github.com/citusdata/citus/issues/7653 See the list below: Compilation PR: https://github.com/citusdata/citus/pull/7699 Ruleutils PR: https://github.com/citusdata/citus/pull/7725 Sister PR for tests: https://github.com/citusdata/the-process/pull/159 Helpful smaller PRs: - https://github.com/citusdata/citus/pull/7714 - https://github.com/citusdata/citus/pull/7726 - https://github.com/citusdata/citus/pull/7731 - https://github.com/citusdata/citus/pull/7732 - https://github.com/citusdata/citus/pull/7733 - https://github.com/citusdata/citus/pull/7738 - https://github.com/citusdata/citus/pull/7745 - https://github.com/citusdata/citus/pull/7747 - https://github.com/citusdata/citus/pull/7748 - https://github.com/citusdata/citus/pull/7749 - https://github.com/citusdata/citus/pull/7752 - https://github.com/citusdata/citus/pull/7755 - https://github.com/citusdata/citus/pull/7757 - https://github.com/citusdata/citus/pull/7759 - https://github.com/citusdata/citus/pull/7760 - https://github.com/citusdata/citus/pull/7761 - https://github.com/citusdata/citus/pull/7762 - https://github.com/citusdata/citus/pull/7765 - https://github.com/citusdata/citus/pull/7766 - https://github.com/citusdata/citus/pull/7768 - https://github.com/citusdata/citus/pull/7769 - https://github.com/citusdata/citus/pull/7771 - https://github.com/citusdata/citus/pull/7774 - https://github.com/citusdata/citus/pull/7776 - https://github.com/citusdata/citus/pull/7780 - https://github.com/citusdata/citus/pull/7781 - https://github.com/citusdata/citus/pull/7785 - https://github.com/citusdata/citus/pull/7788 - https://github.com/citusdata/citus/pull/7793 - https://github.com/citusdata/citus/pull/7796 --------- Co-authored-by: Colm --- .devcontainer/Dockerfile | 14 ++- .github/workflows/build_and_test.yml | 40 ++++++- configure | 2 +- configure.ac | 2 +- .../distributed/sql/citus--12.1-1--13.0-1.sql | 1 + .../udfs/citus_prepare_pg_upgrade/13.0-1.sql | 100 ++++++++++++++++++ .../udfs/citus_prepare_pg_upgrade/latest.sql | 14 +-- src/test/regress/citus_tests/common.py | 1 + 8 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7dc75abd4..23a94f1d6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -103,6 +103,18 @@ RUN mkdir .pgenv-staging/ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf +FROM base AS pg17 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 17.1 +RUN rm .pgenv/src/*.tar* +RUN make -C .pgenv/src/postgresql-*/ clean +RUN make -C .pgenv/src/postgresql-*/src/include install + +# create a staging directory with all files we want to copy from our pgenv build +# we will copy the contents of the staged folder into the final image at once +RUN mkdir .pgenv-staging/ +RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ +RUN rm .pgenv-staging/config/default.conf + FROM base AS uncrustify-builder RUN sudo apt update && sudo apt install -y cmake tree @@ -211,7 +223,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 16.5 +RUN pgenv switch 17.1 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0e077e82c..50f25fe3c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,12 +26,13 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.18" - sql_snapshot_pg_version: "16.5" - image_suffix: "-v1d9d7d7" + sql_snapshot_pg_version: "17.1" + image_suffix: "-v84c0cf8" pg14_version: '{ "major": "14", "full": "14.14" }' pg15_version: '{ "major": "15", "full": "15.9" }' pg16_version: '{ "major": "16", "full": "16.5" }' - upgrade_pg_versions: "14.14-15.9-16.5" + pg17_version: '{ "major": "17", "full": "17.1" }' + upgrade_pg_versions: "14.14-15.9-16.5-17.1" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters @@ -108,6 +109,7 @@ jobs: - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} + - ${{ needs.params.outputs.pg17_version }} runs-on: ubuntu-20.04 container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" @@ -139,6 +141,7 @@ jobs: - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} + - ${{ needs.params.outputs.pg17_version }} make: - check-split - check-multi @@ -168,6 +171,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-failure + pg_version: ${{ needs.params.outputs.pg17_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress @@ -180,6 +187,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-enterprise-failure + pg_version: ${{ needs.params.outputs.pg17_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress @@ -192,6 +203,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-pytest + pg_version: ${{ needs.params.outputs.pg17_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: installcheck suite: cdc image_name: ${{ needs.params.outputs.test_image_name }} @@ -200,6 +215,10 @@ jobs: suite: cdc image_name: ${{ needs.params.outputs.test_image_name }} pg_version: ${{ needs.params.outputs.pg16_version }} + - make: installcheck + suite: cdc + image_name: ${{ needs.params.outputs.test_image_name }} + pg_version: ${{ needs.params.outputs.pg17_version }} - make: check-query-generator pg_version: ${{ needs.params.outputs.pg14_version }} suite: regress @@ -212,6 +231,10 @@ jobs: pg_version: ${{ needs.params.outputs.pg16_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} + - make: check-query-generator + pg_version: ${{ needs.params.outputs.pg17_version }} + suite: regress + image_name: ${{ needs.params.outputs.fail_test_image_name }} runs-on: ubuntu-20.04 container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" @@ -255,6 +278,7 @@ jobs: - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} + - ${{ needs.params.outputs.pg17_version }} parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs steps: - uses: actions/checkout@v3.5.0 @@ -303,6 +327,12 @@ jobs: new_pg_major: 16 - old_pg_major: 14 new_pg_major: 16 + - old_pg_major: 16 + new_pg_major: 17 + - old_pg_major: 15 + new_pg_major: 17 + - old_pg_major: 14 + new_pg_major: 17 env: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} @@ -386,7 +416,7 @@ jobs: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} runs-on: ubuntu-20.04 container: - image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }} + image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg17_version).full }}${{ needs.params.outputs.image_suffix }} needs: - params - test-citus @@ -481,7 +511,7 @@ jobs: name: Test flakyness runs-on: ubuntu-20.04 container: - image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg16_version }}${{ needs.params.outputs.image_suffix }} + image: ${{ needs.params.outputs.fail_test_image_name }}:${{ needs.params.outputs.pg17_version }}${{ needs.params.outputs.image_suffix }} options: --user root env: runs: 8 diff --git a/configure b/configure index 4bda6f37f..5240df4db 100755 --- a/configure +++ b/configure @@ -2588,7 +2588,7 @@ fi if test "$with_pg_version_check" = no; then { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;} -elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16'; then +elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.ac b/configure.ac index 6ecb13760..c7fde02de 100644 --- a/configure.ac +++ b/configure.ac @@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check) if test "$with_pg_version_check" = no; then AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)]) -elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16'; then +elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) diff --git a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql index 3a342a0fe..216171664 100644 --- a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql @@ -1,3 +1,4 @@ -- citus--12.1-1--13.0-1.sql -- bump version to 13.0-1 +#include "udfs/citus_prepare_pg_upgrade/13.0-1.sql" diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql new file mode 100644 index 000000000..4f07ce5c4 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql @@ -0,0 +1,100 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $cppu$ +BEGIN + + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + -- + -- We are dropping the aggregates because postgres 14 changed + -- array_cat type from anyarray to anycompatiblearray. When + -- upgrading to pg14, specifically when running pg_restore on + -- array_cat_agg we would get an error. So we drop the aggregate + -- and create the right one on citus_finish_pg_upgrade. + + DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); + DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); + + -- We should drop any_value because PG16+ has its own any_value function + -- We can remove this part when we drop support for PG16 + IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); + DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + END IF; + + -- + -- Drop existing backup tables + -- + DROP TABLE IF EXISTS public.pg_dist_partition; + DROP TABLE IF EXISTS public.pg_dist_shard; + DROP TABLE IF EXISTS public.pg_dist_placement; + DROP TABLE IF EXISTS public.pg_dist_node_metadata; + DROP TABLE IF EXISTS public.pg_dist_node; + DROP TABLE IF EXISTS public.pg_dist_local_group; + DROP TABLE IF EXISTS public.pg_dist_transaction; + DROP TABLE IF EXISTS public.pg_dist_colocation; + DROP TABLE IF EXISTS public.pg_dist_authinfo; + DROP TABLE IF EXISTS public.pg_dist_poolinfo; + DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; + DROP TABLE IF EXISTS public.pg_dist_object; + DROP TABLE IF EXISTS public.pg_dist_cleanup; + DROP TABLE IF EXISTS public.pg_dist_schema; + DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; + + -- + -- backup citus catalog tables + -- + CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; + CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; + CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; + CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; + CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; + CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; + CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; + CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; + CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; + -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade + CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; + -- enterprise catalog tables + CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; + CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; + -- sequences + CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; + CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT + name, + default_strategy, + shard_cost_function::regprocedure::text, + node_capacity_function::regprocedure::text, + shard_allowed_on_node_function::regprocedure::text, + default_threshold, + minimum_threshold, + improvement_threshold + FROM pg_catalog.pg_dist_rebalance_strategy; + + -- store upgrade stable identifiers on pg_dist_object catalog + CREATE TABLE public.pg_dist_object AS SELECT + address.type, + address.object_names, + address.object_args, + objects.distribution_argument_index, + objects.colocationid + FROM pg_catalog.pg_dist_object objects, + pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we will need to regenerate the partkeys because they will include varnullingrels as well. + -- so we save the partkeys as column names here + CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS + SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name + FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; +END; +$cppu$; + +COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() + IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql index b4bc653f2..4f07ce5c4 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql @@ -18,13 +18,15 @@ BEGIN DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); - -- We should drop any_value because PG16 has its own any_value function + -- We should drop any_value because PG16+ has its own any_value function -- We can remove this part when we drop support for PG16 - DELETE FROM pg_depend WHERE - objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND - refobjid IN (select oid from pg_extension where extname = 'citus'); - DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); - DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); + DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); + END IF; -- -- Drop existing backup tables diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 05fd43fab..3459af33d 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -93,6 +93,7 @@ OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 14: "10.2.0", 15: "11.1.5", 16: "12.1.5", + 17: "13.0.0", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION] From af88c37b56ea2dcfedfb14e1cff3312a4cace369 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 25 Dec 2024 16:58:51 +0300 Subject: [PATCH 095/155] PG17: ALTER INDEX ALTER COLUMN SET STATISTICS DEFAULT (#7808) DESCRIPTION: Propagates ALTER INDEX ALTER COLUMN SET STATISTICS DEFAULT We automatically support this. Adding tests only. We currently don't support ALTER TABLE ALTER COLUMN SET STATISTICS Relevant PG commit: https://github.com/postgres/postgres/commit/4f622503d --- src/test/regress/expected/pg17.out | 108 +++++++++++++++++++++++++++-- src/test/regress/sql/pg17.sql | 35 ++++++++++ 2 files changed, 137 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index fbe8ebbe1..ca2f7643f 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1117,12 +1117,108 @@ SELECT * FROM pg_constraint WHERE conname = 'local_exclude_named' AND contype = DROP TABLE distributed_partitioned_table CASCADE; DROP TABLE local_partitioned_table CASCADE; -- End of Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment +-- Propagate SET STATISTICS DEFAULT +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/4f622503d +SET citus.next_shard_id TO 25122024; +CREATE TABLE tbl (c1 int, c2 int); +SELECT citus_add_local_table_to_metadata('tbl'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +CREATE INDEX tbl_idx ON tbl (c1, (c1+0)) INCLUDE (c2); +-- Citus currently doesn't support ALTER TABLE ALTER COLUMN SET STATISTICS anyway +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS 100; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS DEFAULT; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS -1; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +-- Citus propagates ALTER INDEX ALTER COLUMN SET STATISTICS DEFAULT to the nodes and shards +SET citus.log_remote_commands TO true; +SET citus.grep_remote_commands = '%STATISTICS%'; +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS 1000; +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS 1000; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS 1000; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (25122024, 'pg17', 'ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS 1000;') +\d+ tbl_idx + Index "pg17.tbl_idx" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | 1000 + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl" + +\d+ tbl_idx_25122024 + Index "pg17.tbl_idx_25122024" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | 1000 + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl_25122024" + +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS DEFAULT; +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS DEFAULT; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS DEFAULT; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (25122024, 'pg17', 'ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS DEFAULT;') +\d+ tbl_idx + Index "pg17.tbl_idx" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl" + +\d+ tbl_idx_25122024 + Index "pg17.tbl_idx_25122024" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl_25122024" + +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1; +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (25122024, 'pg17', 'ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1;') +\d+ tbl_idx + Index "pg17.tbl_idx" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl" + +\d+ tbl_idx_25122024 + Index "pg17.tbl_idx_25122024" + Column | Type | Key? | Definition | Storage | Stats target +--------------------------------------------------------------------- + c1 | integer | yes | c1 | plain | + expr | integer | yes | (c1 + 0) | plain | + c2 | integer | no | c2 | plain | +btree, for table "pg17.tbl_25122024" + +-- End of testing SET STATISTICS DEFAULT +\set VERBOSITY terse +SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; -NOTICE: drop cascades to 5 other objects -DETAIL: drop cascades to function fake_am_handler(internal) -drop cascades to access method fake_am -drop cascades to table dist_test -drop cascades to table postgres_table -drop cascades to table distributed_table +\set VERBOSITY default +RESET client_min_messages; DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index f0c6d5b65..aad8445ea 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -574,6 +574,41 @@ DROP TABLE distributed_partitioned_table CASCADE; DROP TABLE local_partitioned_table CASCADE; -- End of Test for exclusion constraints on partitioned and distributed partitioned tables in Citus environment +-- Propagate SET STATISTICS DEFAULT +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/4f622503d +SET citus.next_shard_id TO 25122024; + +CREATE TABLE tbl (c1 int, c2 int); +SELECT citus_add_local_table_to_metadata('tbl'); +CREATE INDEX tbl_idx ON tbl (c1, (c1+0)) INCLUDE (c2); + +-- Citus currently doesn't support ALTER TABLE ALTER COLUMN SET STATISTICS anyway +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS 100; +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS DEFAULT; +ALTER TABLE tbl ALTER COLUMN 1 SET STATISTICS -1; + +-- Citus propagates ALTER INDEX ALTER COLUMN SET STATISTICS DEFAULT to the nodes and shards +SET citus.log_remote_commands TO true; +SET citus.grep_remote_commands = '%STATISTICS%'; + +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS 1000; +\d+ tbl_idx +\d+ tbl_idx_25122024 +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS DEFAULT; +\d+ tbl_idx +\d+ tbl_idx_25122024 +ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1; +\d+ tbl_idx +\d+ tbl_idx_25122024 + +-- End of testing SET STATISTICS DEFAULT + +\set VERBOSITY terse +SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; +\set VERBOSITY default +RESET client_min_messages; + DROP ROLE regress_maintain; DROP ROLE regress_no_maintain; From d682bf06f722a30604fc37a671080fd4209f4a54 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 26 Dec 2024 15:29:44 +0300 Subject: [PATCH 096/155] Error for COPY FROM ... on_error, log_verbosity with Citus tables (#7811) PG17 added the new ON_ERROR option for COPY FROM. When this option is specified, COPY skips soft errors and continues copying. Relevant PG commits: -- https://github.com/postgres/postgres/commit/9e2d87011 -- https://github.com/postgres/postgres/commit/b725b7eec I tried it locally with Citus tables. Without further implementation, it doesn't work correctly. Therefore, we error out for now, and add it to future work. PG17 also added log_verbosity option, which controls the amount of messages emitted during processing. This is currently used in COPY FROM when ON_ERROR option is set to ignore. Therefore, we error out for this option as well. Relevant PG17 commit: https://github.com/postgres/postgres/commit/f5a227895 --- src/backend/distributed/commands/multi_copy.c | 41 +++++++++++++++++++ src/test/regress/expected/pg17.out | 23 +++++++++++ src/test/regress/sql/pg17.sql | 17 ++++++++ 3 files changed, 81 insertions(+) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 268010e7a..3d79e0cbc 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -301,6 +301,7 @@ static SelectStmt * CitusCopySelect(CopyStmt *copyStatement); static void CitusCopyTo(CopyStmt *copyStatement, QueryCompletion *completionTag); static int64 ForwardCopyDataFromConnection(CopyOutState copyOutState, MultiConnection *connection); +static void ErrorIfCopyHasOnErrorLogVerbosity(CopyStmt *copyStatement); /* Private functions copied and adapted from copy.c in PostgreSQL */ static void SendCopyBegin(CopyOutState cstate); @@ -2829,6 +2830,44 @@ CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName) } +/* + * ErrorIfCopyHasOnErrorLogVerbosity errors out if the COPY statement + * has on_error option or log_verbosity option specified + */ +static void +ErrorIfCopyHasOnErrorLogVerbosity(CopyStmt *copyStatement) +{ +#if PG_VERSION_NUM >= PG_VERSION_17 + bool log_verbosity = false; + foreach_ptr(DefElem, option, copyStatement->options) + { + if (strcmp(option->defname, "on_error") == 0) + { + ereport(ERROR, (errmsg( + "Citus does not support COPY FROM with ON_ERROR option."))); + } + else if (strcmp(option->defname, "log_verbosity") == 0) + { + log_verbosity = true; + } + } + + /* + * Given that log_verbosity is currently used in COPY FROM + * when ON_ERROR option is set to ignore, it makes more + * sense to error out for ON_ERROR option first. For this reason, + * we don't error out in the previous loop directly. + * Relevant PG17 commit: https://github.com/postgres/postgres/commit/f5a227895 + */ + if (log_verbosity) + { + ereport(ERROR, (errmsg( + "Citus does not support COPY FROM with LOG_VERBOSITY option."))); + } +#endif +} + + /* * ErrorIfMergeInCopy Raises an exception if the MERGE is called in the COPY * where Citus tables are involved, as we don't support this yet @@ -2931,6 +2970,8 @@ ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletion *completionTag, const "Citus does not support COPY FROM with WHERE"))); } + ErrorIfCopyHasOnErrorLogVerbosity(copyStatement); + /* check permissions, we're bypassing postgres' normal checks */ CheckCopyPermissions(copyStatement); CitusCopyFrom(copyStatement, completionTag); diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index ca2f7643f..99ed0c3e2 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1215,6 +1215,29 @@ btree, for table "pg17.tbl" btree, for table "pg17.tbl_25122024" -- End of testing SET STATISTICS DEFAULT +-- COPY ON_ERROR option +-- Error out for Citus tables because we don't support it yet +-- Relevant PG17 commits: +-- https://github.com/postgres/postgres/commit/9e2d87011 +-- https://github.com/postgres/postgres/commit/b725b7eec +CREATE TABLE check_ign_err (n int, m int[], k int); +SELECT create_distributed_table('check_ign_err', 'n'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +COPY check_ign_err FROM STDIN WITH (on_error stop); +ERROR: Citus does not support COPY FROM with ON_ERROR option. +COPY check_ign_err FROM STDIN WITH (ON_ERROR ignore); +ERROR: Citus does not support COPY FROM with ON_ERROR option. +COPY check_ign_err FROM STDIN WITH (on_error ignore, log_verbosity verbose); +ERROR: Citus does not support COPY FROM with ON_ERROR option. +COPY check_ign_err FROM STDIN WITH (log_verbosity verbose, on_error ignore); +ERROR: Citus does not support COPY FROM with ON_ERROR option. +COPY check_ign_err FROM STDIN WITH (log_verbosity verbose); +ERROR: Citus does not support COPY FROM with LOG_VERBOSITY option. +-- End of Test for COPY ON_ERROR option \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index aad8445ea..2018d8b3b 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -604,6 +604,23 @@ ALTER INDEX tbl_idx ALTER COLUMN 2 SET STATISTICS -1; -- End of testing SET STATISTICS DEFAULT +-- COPY ON_ERROR option +-- Error out for Citus tables because we don't support it yet +-- Relevant PG17 commits: +-- https://github.com/postgres/postgres/commit/9e2d87011 +-- https://github.com/postgres/postgres/commit/b725b7eec + +CREATE TABLE check_ign_err (n int, m int[], k int); +SELECT create_distributed_table('check_ign_err', 'n'); + +COPY check_ign_err FROM STDIN WITH (on_error stop); +COPY check_ign_err FROM STDIN WITH (ON_ERROR ignore); +COPY check_ign_err FROM STDIN WITH (on_error ignore, log_verbosity verbose); +COPY check_ign_err FROM STDIN WITH (log_verbosity verbose, on_error ignore); +COPY check_ign_err FROM STDIN WITH (log_verbosity verbose); + +-- End of Test for COPY ON_ERROR option + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From b4cc7219ae4739bb591dedd7cf5323edead43f13 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:52:42 +0300 Subject: [PATCH 097/155] Add tests for FORCE_NULL * and FORCE_NOT_NULL * options for COPY FROM (#7812) These options already existed in PG17, and we support them and have tests for them in `multi_copy.sql`. In PG17, their capability was extended to specify ALL columns at once using *. Citus performs the COPY correctly, as is validated by the added tests in this PR. Relevant PG commit: https://github.com/postgres/postgres/commit/f6d4c9cf1 Copy-pasting from Postgres documentation what these options do, such that the reviewer may better understand the tests added: `FORCE_NOT_NULL`: Do not match the specified columns' values against the null string. In the default case where the null string is empty, this means that empty values will be read as zero-length strings rather than nulls, even when they are not quoted. If * is specified, the option will be applied to all columns. This option is allowed only in `COPY FROM`, and only when using `CSV` format. `FORCE_NULL`: Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to `NULL`. In the default case where the null string is empty, this converts a quoted empty string into `NULL`. If * is specified, the option will be applied to all columns. This option is allowed only in `COPY FROM`, and only when using `CSV` format. `FORCE_NULL` and `FORCE_NOT_NULL` can be used simultaneously on the same column. This results in converting quoted null strings to null values and unquoted null strings to empty strings. Explain it to me like I'm a 5-year-old, for a text column: `FORCE_NULL` looks for empty strings and registers them as `NULL` `FORCE_NOT_NULL` looks for null values and registers them as empty strings. --- src/test/regress/expected/pg17.out | 95 ++++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 90 ++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 99ed0c3e2..23efee4bd 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1238,6 +1238,101 @@ ERROR: Citus does not support COPY FROM with ON_ERROR option. COPY check_ign_err FROM STDIN WITH (log_verbosity verbose); ERROR: Citus does not support COPY FROM with LOG_VERBOSITY option. -- End of Test for COPY ON_ERROR option +-- Test FORCE_NOT_NULL and FORCE_NULL options +-- FORCE_NULL * and FORCE_NOT_NULL * options for COPY FROM were added in PG17 +-- Same tests as in PG copy2.sql, we just distribute the table first +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/f6d4c9cf1 +CREATE TABLE forcetest ( + a INT NOT NULL, + b TEXT NOT NULL, + c TEXT, + d TEXT, + e TEXT +); +\pset null NULL +SELECT create_distributed_table('forcetest', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +-- expected output for inserted row in test: +-- b | c +--------------------------------------------------------------------- +-- | NULL +--(1 row) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL(b), FORCE_NULL(c)); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 1; + b | c +--------------------------------------------------------------------- + | NULL +(1 row) + +-- should succeed, FORCE_NULL and FORCE_NOT_NULL can be both specified +-- expected output for inserted row in test: +-- c | d +--------------------------------------------------------------------- +-- | NULL +--(1 row) +BEGIN; +COPY forcetest (a, b, c, d) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL(c,d), FORCE_NULL(c,d)); +COMMIT; +SELECT c, d FROM forcetest WHERE a = 2; + c | d +--------------------------------------------------------------------- + | NULL +(1 row) + +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +-- expected output for inserted row in test: +-- b | c +--------------------------------------------------------------------- +-- | NULL +--(1 row) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; + b | c +--------------------------------------------------------------------- + | NULL +(1 row) + +-- should succeed with effect ("b" remains an empty string) +-- expected output for inserted row in test: +-- b | c +--------------------------------------------------------------------- +-- | +--(1 row) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; + b | c +--------------------------------------------------------------------- + | +(1 row) + +-- should succeed with effect ("c" remains NULL) +-- expected output for inserted row in test: +-- b | c +--------------------------------------------------------------------- +-- b | NULL +--(1 row) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; + b | c +--------------------------------------------------------------------- + b | NULL +(1 row) + +\pset null '' +-- End of Testing FORCE_NOT_NULL and FORCE_NULL options \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 2018d8b3b..ab8528f4b 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -621,6 +621,96 @@ COPY check_ign_err FROM STDIN WITH (log_verbosity verbose); -- End of Test for COPY ON_ERROR option +-- Test FORCE_NOT_NULL and FORCE_NULL options +-- FORCE_NULL * and FORCE_NOT_NULL * options for COPY FROM were added in PG17 +-- Same tests as in PG copy2.sql, we just distribute the table first +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/f6d4c9cf1 + +CREATE TABLE forcetest ( + a INT NOT NULL, + b TEXT NOT NULL, + c TEXT, + d TEXT, + e TEXT +); +\pset null NULL + +SELECT create_distributed_table('forcetest', 'a'); + +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +-- expected output for inserted row in test: +-- b | c +-----+------ +-- | NULL +--(1 row) + +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL(b), FORCE_NULL(c)); +1,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 1; + +-- should succeed, FORCE_NULL and FORCE_NOT_NULL can be both specified +-- expected output for inserted row in test: +-- c | d +-----+------ +-- | NULL +--(1 row) + +BEGIN; +COPY forcetest (a, b, c, d) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL(c,d), FORCE_NULL(c,d)); +2,'a',,"" +\. +COMMIT; +SELECT c, d FROM forcetest WHERE a = 2; + +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +-- expected output for inserted row in test: +-- b | c +-----+------ +-- | NULL +--(1 row) + +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +4,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; + +-- should succeed with effect ("b" remains an empty string) +-- expected output for inserted row in test: +-- b | c +-----+--- +-- | +--(1 row) + +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +5,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; + +-- should succeed with effect ("c" remains NULL) +-- expected output for inserted row in test: +-- b | c +-----+------ +-- b | NULL +--(1 row) + +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +6,"b","" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; + +\pset null '' + +-- End of Testing FORCE_NOT_NULL and FORCE_NULL options + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 62a00b1b3495d635b7af320802329a8f12b9299d Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Fri, 27 Dec 2024 15:07:38 +0300 Subject: [PATCH 098/155] Error out for ALTER TABLE ... SET ACCESS METHOD DEFAULT (#7803) PG17 introduced ALTER TABLE ... SET ACCESS METHOD DEFAULT This PR introduces and enforces an error check preventing ALTER TABLE ... SET ACCESS METHOD DEFAULT on both Citus local tables (added via citus_add_local_table_to_metadata) and distributed/partitioned distributed tables. The regression tests now demonstrate that each table type raises an error advising users to explicitly specify an access method, rather than relying on DEFAULT. This ensures consistent behavior across local and distributed environments in Citus. The reason why we currently don't support this is that we can't simply propagate the command as it is, because the default table access method may be different across Citus cluster nodes. Relevant PG commit: https://github.com/postgres/postgres/commit/d61a6cad6 --- src/backend/distributed/commands/table.c | 18 ++++++++++ src/test/regress/expected/pg17.out | 43 ++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 30 +++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index e65f57961..c395892b5 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -3666,6 +3666,24 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) #if PG_VERSION_NUM >= PG_VERSION_15 case AT_SetAccessMethod: + { + /* + * If command->name == NULL, that means the user is trying to use + * ALTER TABLE ... SET ACCESS METHOD DEFAULT + * which we don't support currently. + */ + if (command->name == NULL) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "DEFAULT option in ALTER TABLE ... SET ACCESS METHOD " + "is currently unsupported."), + errhint( + "You can rerun the command by explicitly writing the access method name."))); + } + break; + } + #endif case AT_SetNotNull: case AT_ReplicaIdentity: diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 23efee4bd..fe925516b 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1333,6 +1333,49 @@ SELECT b, c FROM forcetest WHERE a = 6; \pset null '' -- End of Testing FORCE_NOT_NULL and FORCE_NULL options +-- Test for ALTER TABLE SET ACCESS METHOD DEFAULT +-- Step 1: Local table setup (non-distributed) +CREATE TABLE test_local_table (id int); +SELECT citus_add_local_table_to_metadata('test_local_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- Step 2: Attempt to set access method to DEFAULT on a Citus local table (should fail) +ALTER TABLE test_local_table SET ACCESS METHOD DEFAULT; +ERROR: DEFAULT option in ALTER TABLE ... SET ACCESS METHOD is currently unsupported. +HINT: You can rerun the command by explicitly writing the access method name. +-- Step 3: Setup: create and distribute a table +CREATE TABLE test_alter_access_method (id int); +SELECT create_distributed_table('test_alter_access_method', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 4: Attempt to set access method to DEFAULT on a distributed table (should fail with your custom error) +ALTER TABLE test_alter_access_method SET ACCESS METHOD DEFAULT; +ERROR: DEFAULT option in ALTER TABLE ... SET ACCESS METHOD is currently unsupported. +HINT: You can rerun the command by explicitly writing the access method name. +-- Step 5: Create and distribute a partitioned table +CREATE TABLE test_partitioned_alter (id int, val text) PARTITION BY RANGE (id); +CREATE TABLE test_partitioned_alter_part1 PARTITION OF test_partitioned_alter FOR VALUES FROM (1) TO (100); +SELECT create_distributed_table('test_partitioned_alter', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 6: Attempt to set access method to DEFAULT on a partitioned, distributed table (should fail) +ALTER TABLE test_partitioned_alter SET ACCESS METHOD DEFAULT; +ERROR: DEFAULT option in ALTER TABLE ... SET ACCESS METHOD is currently unsupported. +HINT: You can rerun the command by explicitly writing the access method name. +-- Cleanup +DROP TABLE test_local_table CASCADE; +DROP TABLE test_alter_access_method CASCADE; +DROP TABLE test_partitioned_alter CASCADE; +-- End of Test for ALTER TABLE SET ACCESS METHOD DEFAULT \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index ab8528f4b..88d1a475f 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -711,6 +711,36 @@ SELECT b, c FROM forcetest WHERE a = 6; -- End of Testing FORCE_NOT_NULL and FORCE_NULL options +-- Test for ALTER TABLE SET ACCESS METHOD DEFAULT +-- Step 1: Local table setup (non-distributed) +CREATE TABLE test_local_table (id int); + +SELECT citus_add_local_table_to_metadata('test_local_table'); + +-- Step 2: Attempt to set access method to DEFAULT on a Citus local table (should fail) +ALTER TABLE test_local_table SET ACCESS METHOD DEFAULT; + +-- Step 3: Setup: create and distribute a table +CREATE TABLE test_alter_access_method (id int); +SELECT create_distributed_table('test_alter_access_method', 'id'); + +-- Step 4: Attempt to set access method to DEFAULT on a distributed table (should fail with your custom error) +ALTER TABLE test_alter_access_method SET ACCESS METHOD DEFAULT; + +-- Step 5: Create and distribute a partitioned table +CREATE TABLE test_partitioned_alter (id int, val text) PARTITION BY RANGE (id); +CREATE TABLE test_partitioned_alter_part1 PARTITION OF test_partitioned_alter FOR VALUES FROM (1) TO (100); +SELECT create_distributed_table('test_partitioned_alter', 'id'); + +-- Step 6: Attempt to set access method to DEFAULT on a partitioned, distributed table (should fail) +ALTER TABLE test_partitioned_alter SET ACCESS METHOD DEFAULT; + +-- Cleanup +DROP TABLE test_local_table CASCADE; +DROP TABLE test_alter_access_method CASCADE; +DROP TABLE test_partitioned_alter CASCADE; +-- End of Test for ALTER TABLE SET ACCESS METHOD DEFAULT + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 1a3316281c8025fe5ac5b9b47bd290bda881bcb7 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Fri, 27 Dec 2024 16:02:12 +0300 Subject: [PATCH 099/155] Error out for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION (#7814) PG17 added support for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION. Relevant PG commit: https://github.com/postgres/postgres/commit/5d06e99a3 We currently don't support propagating this command for Citus tables. It is added to future work. This PR disallows `ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION` on all Citus table types (local, distributed, and partitioned distributed) by adding an error check in `ErrorIfUnsupportedAlterTableStmt`. A new regression test verifies that each table type fails with a consistent error message when attempting to set an expression. --- src/backend/distributed/commands/table.c | 11 +++++++ src/test/regress/expected/pg17.out | 41 ++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 32 ++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index c395892b5..67b731a25 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -3664,6 +3664,17 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) break; } +#if PG_VERSION_NUM >= PG_VERSION_17 + case AT_SetExpression: + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION commands " + "are currently unsupported."))); + break; + } + +#endif #if PG_VERSION_NUM >= PG_VERSION_15 case AT_SetAccessMethod: { diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index fe925516b..4d086be82 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1376,6 +1376,47 @@ DROP TABLE test_local_table CASCADE; DROP TABLE test_alter_access_method CASCADE; DROP TABLE test_partitioned_alter CASCADE; -- End of Test for ALTER TABLE SET ACCESS METHOD DEFAULT +-- Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION +-- Step 1: Local table setup (non-distributed) +CREATE TABLE test_local_table_expr (id int, col int); +SELECT citus_add_local_table_to_metadata('test_local_table_expr'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- Step 2: Attempt to set expression on a Citus local table (should fail) +ALTER TABLE test_local_table_expr ALTER COLUMN col SET EXPRESSION AS (id * 4); +ERROR: ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION commands are currently unsupported. +-- Step 3: Create and distribute a table +CREATE TABLE test_distributed_table_expr (id int, col int); +SELECT create_distributed_table('test_distributed_table_expr', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 4: Attempt to set expression on a distributed table (should fail) +ALTER TABLE test_distributed_table_expr ALTER COLUMN col SET EXPRESSION AS (id * 4); +ERROR: ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION commands are currently unsupported. +-- Step 5: Create and distribute a partitioned table +CREATE TABLE test_partitioned_expr (id int, val text) PARTITION BY RANGE (id); +CREATE TABLE test_partitioned_expr_part1 PARTITION OF test_partitioned_expr + FOR VALUES FROM (1) TO (100); +SELECT create_distributed_table('test_partitioned_expr', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 6: Attempt to set expression on a partitioned, distributed table (should fail) +ALTER TABLE test_partitioned_expr ALTER COLUMN val SET EXPRESSION AS (id * 4); +ERROR: ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION commands are currently unsupported. +-- Cleanup +DROP TABLE test_local_table_expr CASCADE; +DROP TABLE test_distributed_table_expr CASCADE; +DROP TABLE test_partitioned_expr CASCADE; +-- End of Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 88d1a475f..6ca506267 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -741,6 +741,38 @@ DROP TABLE test_alter_access_method CASCADE; DROP TABLE test_partitioned_alter CASCADE; -- End of Test for ALTER TABLE SET ACCESS METHOD DEFAULT +-- Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION + +-- Step 1: Local table setup (non-distributed) +CREATE TABLE test_local_table_expr (id int, col int); + +SELECT citus_add_local_table_to_metadata('test_local_table_expr'); + +-- Step 2: Attempt to set expression on a Citus local table (should fail) +ALTER TABLE test_local_table_expr ALTER COLUMN col SET EXPRESSION AS (id * 4); + +-- Step 3: Create and distribute a table +CREATE TABLE test_distributed_table_expr (id int, col int); +SELECT create_distributed_table('test_distributed_table_expr', 'id'); + +-- Step 4: Attempt to set expression on a distributed table (should fail) +ALTER TABLE test_distributed_table_expr ALTER COLUMN col SET EXPRESSION AS (id * 4); + +-- Step 5: Create and distribute a partitioned table +CREATE TABLE test_partitioned_expr (id int, val text) PARTITION BY RANGE (id); +CREATE TABLE test_partitioned_expr_part1 PARTITION OF test_partitioned_expr + FOR VALUES FROM (1) TO (100); +SELECT create_distributed_table('test_partitioned_expr', 'id'); + +-- Step 6: Attempt to set expression on a partitioned, distributed table (should fail) +ALTER TABLE test_partitioned_expr ALTER COLUMN val SET EXPRESSION AS (id * 4); + +-- Cleanup +DROP TABLE test_local_table_expr CASCADE; +DROP TABLE test_distributed_table_expr CASCADE; +DROP TABLE test_partitioned_expr CASCADE; +-- End of Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 8a8b2f9a6d1ce86be9171e74983ed14b18e52024 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 30 Dec 2024 12:54:21 +0300 Subject: [PATCH 100/155] Add tests for inserting with AT LOCAL operator (#7815) PG17 has added support for AT LOCAL operator it converts the given time type to time stamp with the session's TimeZone value as time zone. Here we add tests that validate that we can use AT LOCAL at INSERT commands Relevant PG commit: https://github.com/postgres/postgres/commit/97957fdba With the tests, we verify that we evaluate AT LOCAL at the coordinator and then perform the insert remotely. --- src/test/regress/expected/pg17.out | 36 ++++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 24 ++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 4d086be82..ff1e57d74 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1417,6 +1417,42 @@ DROP TABLE test_local_table_expr CASCADE; DROP TABLE test_distributed_table_expr CASCADE; DROP TABLE test_partitioned_expr CASCADE; -- End of Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION +RESET citus.grep_remote_commands; +RESET citus.log_remote_commands; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 27122024; +-- PG17 has added support for AT LOCAL operator +-- it converts the given time type to +-- time stamp with the session's TimeZone value as time zone. +-- Here we add tests that validate that we can use AT LOCAL at INSERT commands +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/97957fdba +CREATE TABLE test_at_local (id int, time_example timestamp with time zone); +SELECT create_distributed_table('test_at_local', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +BEGIN; +SET LOCAL TimeZone TO 'Europe/Tirane'; +SELECT timestamp '2001-02-16 20:38:40' AT LOCAL; + timezone +--------------------------------------------------------------------- + Fri Feb 16 20:38:40 2001 CET +(1 row) + +-- verify that we evaluate AT LOCAL at the coordinator and then perform the insert remotely +SET citus.log_remote_commands TO on; +INSERT INTO test_at_local VALUES (1, timestamp '2001-02-16 20:38:40' AT LOCAL); +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing INSERT INTO pg17.test_at_local_27122024 (id, time_example) VALUES (1, 'Fri Feb 16 20:38:40 2001 CET'::timestamp with time zone) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +ROLLBACK; +NOTICE: issuing ROLLBACK +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- End of Testing AT LOCAL option \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 6ca506267..fd3a6ddfd 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -772,6 +772,30 @@ DROP TABLE test_local_table_expr CASCADE; DROP TABLE test_distributed_table_expr CASCADE; DROP TABLE test_partitioned_expr CASCADE; -- End of Test for ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION +RESET citus.grep_remote_commands; +RESET citus.log_remote_commands; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 27122024; + +-- PG17 has added support for AT LOCAL operator +-- it converts the given time type to +-- time stamp with the session's TimeZone value as time zone. +-- Here we add tests that validate that we can use AT LOCAL at INSERT commands +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/97957fdba + +CREATE TABLE test_at_local (id int, time_example timestamp with time zone); +SELECT create_distributed_table('test_at_local', 'id'); + +BEGIN; +SET LOCAL TimeZone TO 'Europe/Tirane'; +SELECT timestamp '2001-02-16 20:38:40' AT LOCAL; +-- verify that we evaluate AT LOCAL at the coordinator and then perform the insert remotely +SET citus.log_remote_commands TO on; +INSERT INTO test_at_local VALUES (1, timestamp '2001-02-16 20:38:40' AT LOCAL); +ROLLBACK; + +-- End of Testing AT LOCAL option \set VERBOSITY terse SET client_min_messages TO WARNING; From 9e3c3297fc4f64039f640d10929e1bfc38011799 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:19:07 +0300 Subject: [PATCH 101/155] Adds JSON_TABLE() support, and SQL/JSON constructor/query functions tests (#7816) DESCRIPTION: Adds JSON_TABLE() support PG17 has added basic `JSON_TABLE()` functionality `JSON_TABLE()` allows `JSON` data to be converted into a relational view and thus used, for example, in a `FROM` clause, like other tabular data. We treat `JSON_TABLE` the same as correlated functions (e.g., recurring tuples). In the end, for multi-shard `JSON_TABLE` commands, we apply the same restrictions as reference tables (e.g., cannot perform a lateral outer join when a distributed subquery references a (reference table)/(json table) etc.) Relevant PG17 commits: [basic JSON table](https://github.com/postgres/postgres/commit/de3600452), [nested paths in json table](https://github.com/postgres/postgres/commit/bb766cde6) Onder had previously added json table support for PG15BETA1, but we reverted that commit because json table was reverted in PG15. https://github.com/citusdata/citus/commit/ce7f1a530f84fb6bc4af935d4c9b2cebfb60b514 Previous relevant PG15Beta1 commit: https://github.com/postgres/postgres/commit/4e34747c8 Therefore, I referred to Onder's commit for this commit as well, with a few changes due to some differences between PG15/PG17: 1) In PG15Beta1, we had also `PLAN` clauses for `JSON_TABLE` https://github.com/postgres/postgres/commit/fadb48b00, and Onder's commit includes tests for those as well. However, `PLAN` nodes are _not_ added in PG17. Therefore, I didn't include the `json_table_select_only` test, which had mostly queries involving `PLAN`. I only included the last query from that test. 2) In PG15 timeline (Citus 11.1), we didn't support outer joins where the outer rel is a recurring one and the inner one is a non-recurring one. However, [Onur added support for that one in Citus 11.2](https://github.com/citusdata/citus/pull/6512), therefore I updated the tests from Onder's commit accordingly. 3) PG17 json table has nested paths and columns, therefore I added a test with a distributed table, which is exactly the same as the one in sqljson_jsontable in PG17. https://github.com/postgres/postgres/commit/bb766cde6 This pull request also adds some basic tests on validation of SQL/JSON constructor functions JSON(), JSON_SCALAR(), and JSON_SERIALIZE(), and also SQL/JSON query functions JSON_EXISTS(), JSON_QUERY(), and JSON_VALUE(). The relevant PG commits are the following: [JSON(), JSON_SCALAR(), JSON_SERIALIZE()](https://github.com/postgres/postgres/commit/03734a7fe) [JSON_EXISTS(), JSON_VALUE(), JSON_QUERY()](https://github.com/postgres/postgres/commit/6185c9737) --- .../planner/multi_logical_planner.c | 3 +- .../planner/query_pushdown_planning.c | 56 +- .../distributed/query_pushdown_planning.h | 1 + src/test/regress/expected/pg17_json.out | 578 ++++++++++++++++++ src/test/regress/expected/pg17_json_0.out | 18 + src/test/regress/multi_schedule | 2 +- src/test/regress/sql/pg17_json.sql | 379 ++++++++++++ 7 files changed, 1031 insertions(+), 6 deletions(-) create mode 100644 src/test/regress/expected/pg17_json.out create mode 100644 src/test/regress/expected/pg17_json_0.out create mode 100644 src/test/regress/sql/pg17_json.sql diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index ebc71e2a6..662d40503 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -1170,7 +1170,8 @@ HasComplexRangeTableType(Query *queryTree) if (rangeTableEntry->rtekind != RTE_RELATION && rangeTableEntry->rtekind != RTE_SUBQUERY && rangeTableEntry->rtekind != RTE_FUNCTION && - rangeTableEntry->rtekind != RTE_VALUES) + rangeTableEntry->rtekind != RTE_VALUES && + !IsJsonTableRTE(rangeTableEntry)) { hasComplexRangeTableType = true; } diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 65de8680c..20175eac3 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -61,7 +61,8 @@ typedef enum RecurringTuplesType RECURRING_TUPLES_FUNCTION, RECURRING_TUPLES_EMPTY_JOIN_TREE, RECURRING_TUPLES_RESULT_FUNCTION, - RECURRING_TUPLES_VALUES + RECURRING_TUPLES_VALUES, + RECURRING_TUPLES_JSON_TABLE } RecurringTuplesType; /* @@ -347,7 +348,8 @@ IsFunctionOrValuesRTE(Node *node) RangeTblEntry *rangeTblEntry = (RangeTblEntry *) node; if (rangeTblEntry->rtekind == RTE_FUNCTION || - rangeTblEntry->rtekind == RTE_VALUES) + rangeTblEntry->rtekind == RTE_VALUES || + IsJsonTableRTE(rangeTblEntry)) { return true; } @@ -700,6 +702,13 @@ DeferErrorIfFromClauseRecurs(Query *queryTree) "the FROM clause contains VALUES", NULL, NULL); } + else if (recurType == RECURRING_TUPLES_JSON_TABLE) + { + return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, + "correlated subqueries are not supported when " + "the FROM clause contains JSON_TABLE", NULL, + NULL); + } /* @@ -1204,7 +1213,8 @@ DeferErrorIfUnsupportedTableCombination(Query *queryTree) */ if (rangeTableEntry->rtekind == RTE_RELATION || rangeTableEntry->rtekind == RTE_SUBQUERY || - rangeTableEntry->rtekind == RTE_RESULT) + rangeTableEntry->rtekind == RTE_RESULT || + IsJsonTableRTE(rangeTableEntry)) { /* accepted */ } @@ -1372,6 +1382,13 @@ DeferErrorIfUnsupportedUnionQuery(Query *subqueryTree) "VALUES is not supported within a " "UNION", NULL); } + else if (recurType == RECURRING_TUPLES_JSON_TABLE) + { + return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, + "cannot push down this subquery", + "JSON_TABLE is not supported within a " + "UNION", NULL); + } return NULL; } @@ -1477,6 +1494,11 @@ RecurringTypeDescription(RecurringTuplesType recurType) return "a VALUES clause"; } + case RECURRING_TUPLES_JSON_TABLE: + { + return "a JSON_TABLE"; + } + case RECURRING_TUPLES_INVALID: { /* @@ -1673,7 +1695,8 @@ DeferredErrorIfUnsupportedLateralSubquery(PlannerInfo *plannerInfo, * strings anyway. */ if (recurType != RECURRING_TUPLES_VALUES && - recurType != RECURRING_TUPLES_RESULT_FUNCTION) + recurType != RECURRING_TUPLES_RESULT_FUNCTION && + recurType != RECURRING_TUPLES_JSON_TABLE) { recurTypeDescription = psprintf("%s (%s)", recurTypeDescription, recurringRangeTableEntry->eref-> @@ -1750,6 +1773,26 @@ ContainsRecurringRangeTable(List *rangeTable, RecurringTuplesType *recurType) } +/* + * IsJsonTableRTE checks whether the RTE refers to a JSON_TABLE + * table function, which was introduced in PostgreSQL 17. + */ +bool +IsJsonTableRTE(RangeTblEntry *rte) +{ +#if PG_VERSION_NUM >= PG_VERSION_17 + if (rte == NULL) + { + return false; + } + return (rte->rtekind == RTE_TABLEFUNC && + rte->tablefunc->functype == TFT_JSON_TABLE); +#endif + + return false; +} + + /* * HasRecurringTuples returns whether any part of the expression will generate * the same set of tuples in every query on shards when executing a distributed @@ -1811,6 +1854,11 @@ HasRecurringTuples(Node *node, RecurringTuplesType *recurType) *recurType = RECURRING_TUPLES_VALUES; return true; } + else if (IsJsonTableRTE(rangeTableEntry)) + { + *recurType = RECURRING_TUPLES_JSON_TABLE; + return true; + } return false; } diff --git a/src/include/distributed/query_pushdown_planning.h b/src/include/distributed/query_pushdown_planning.h index e0d4f25dd..47a34cee0 100644 --- a/src/include/distributed/query_pushdown_planning.h +++ b/src/include/distributed/query_pushdown_planning.h @@ -46,6 +46,7 @@ extern DeferredErrorMessage * DeferErrorIfCannotPushdownSubquery(Query *subquery bool outerMostQueryHasLimit); extern DeferredErrorMessage * DeferErrorIfUnsupportedUnionQuery(Query *queryTree); +extern bool IsJsonTableRTE(RangeTblEntry *rte); #endif /* QUERY_PUSHDOWN_PLANNING_H */ diff --git a/src/test/regress/expected/pg17_json.out b/src/test/regress/expected/pg17_json.out new file mode 100644 index 000000000..993ad985b --- /dev/null +++ b/src/test/regress/expected/pg17_json.out @@ -0,0 +1,578 @@ +-- +-- PG17_JSON +-- PG17 has added basic JSON_TABLE() functionality +-- JSON_TABLE() allows JSON data to be converted into a relational view +-- and thus used, for example, in a FROM clause, like other tabular +-- data. We treat JSON_TABLE the same as correlated functions (e.g., recurring tuples). +-- In the end, for multi-shard JSON_TABLE commands, we apply the same +-- restrictions as reference tables (e.g., cannot perform a lateral outer join +-- when a distributed subquery references a (reference table)/JSON_TABLE etc.) +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/de3600452 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +\else +\q +\endif +CREATE SCHEMA pg17_json; +SET search_path TO pg17_json; +SET citus.next_shard_id TO 1687000; +CREATE TABLE test_table(id bigserial, value text); +SELECT create_distributed_table('test_table', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_table (value) SELECT i::text FROM generate_series(0,100)i; +CREATE TABLE my_films(id bigserial, js jsonb); +SELECT create_distributed_table('my_films', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO my_films(js) VALUES ( +'{ "favorites" : [ + { "kind" : "comedy", "films" : [ { "title" : "Bananas", "director" : "Woody Allen"}, + { "title" : "The Dinner Game", "director" : "Francis Veber" } ] }, + { "kind" : "horror", "films" : [{ "title" : "Psycho", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "thriller", "films" : [{ "title" : "Vertigo", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "drama", "films" : [{ "title" : "Yojimbo", "director" : "Akira Kurosawa" } ] } + ] }'); +INSERT INTO my_films(js) VALUES ( +'{ "favorites" : [ + { "kind" : "comedy", "films" : [ { "title" : "Bananas2", "director" : "Woody Allen"}, + { "title" : "The Dinner Game2", "director" : "Francis Veber" } ] }, + { "kind" : "horror", "films" : [{ "title" : "Psycho2", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "thriller", "films" : [{ "title" : "Vertigo2", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "drama", "films" : [{ "title" : "Yojimbo2", "director" : "Akira Kurosawa" } ] } + ] }'); +-- a router query +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + WHERE my_films.id = 1 + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa +(5 rows) + +-- router query with an explicit LATEREL SUBQUERY +SELECT sub.* +FROM my_films, + lateral(SELECT * FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt) as sub +WHERE my_films.id = 1; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa +(5 rows) + +-- router query with an explicit LATEREL SUBQUERY and LIMIT +SELECT sub.* +FROM my_films, + lateral(SELECT * FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt ORDER BY id DESC LIMIT 1) as sub +WHERE my_films.id = 1; + id | kind | title | director +--------------------------------------------------------------------- + 4 | drama | Yojimbo | Akira Kurosawa +(1 row) + +-- set it DEBUG1 in case the plan changes +-- we can see details +SET client_min_messages TO DEBUG1; +-- a mult-shard query +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | Bananas2 | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 1 | comedy | The Dinner Game2 | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 2 | horror | Psycho2 | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 3 | thriller | Vertigo2 | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa + 4 | drama | Yojimbo2 | Akira Kurosawa +(10 rows) + +-- recursively plan subqueries that has JSON_TABLE +SELECT count(*) FROM +( + SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + LIMIT 1) as sub_with_json, test_table +WHERE test_table.id = sub_with_json.id; +DEBUG: push down of limit count: 1 +DEBUG: generating subplan XXX_1 for subquery SELECT jt.id, jt.kind, jt.title, jt.director FROM pg17_json.my_films, LATERAL JSON_TABLE(my_films.js, '$."favorites"[*]' AS json_table_path_0 COLUMNS (id FOR ORDINALITY, kind text PATH '$."kind"', NESTED PATH '$."films"[*]' AS json_table_path_1 COLUMNS (title text PATH '$."title"', director text PATH '$."director"'))) jt LIMIT 1 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.id, intermediate_result.kind, intermediate_result.title, intermediate_result.director FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, kind text, title text, director text)) sub_with_json, pg17_json.test_table WHERE (test_table.id OPERATOR(pg_catalog.=) sub_with_json.id) + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- multi-shard query with an explicit LATEREL SUBQUERY +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | Bananas2 | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 1 | comedy | The Dinner Game2 | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 2 | horror | Psycho2 | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 3 | thriller | Vertigo2 | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa + 4 | drama | Yojimbo2 | Akira Kurosawa +(10 rows) + +-- JSON_TABLE can be on the inner part of an outer joion +SELECT sub.* +FROM my_films LEFT JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | Bananas2 | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 1 | comedy | The Dinner Game2 | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 2 | horror | Psycho2 | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 3 | thriller | Vertigo2 | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa + 4 | drama | Yojimbo2 | Akira Kurosawa +(10 rows) + +-- we can pushdown this correlated subquery in WHERE clause +SELECT count(*) +FROM my_films WHERE + (SELECT count(*) > 0 + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000); + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- we can pushdown this correlated subquery in SELECT clause + SELECT (SELECT count(*) > 0 + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt) +FROM my_films; + ?column? +--------------------------------------------------------------------- + t + t +(2 rows) + +-- multi-shard query with an explicit LATEREL SUBQUERY +-- along with other tables +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) JOIN test_table ON(my_films.id = test_table.id) + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | Bananas2 | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 1 | comedy | The Dinner Game2 | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 2 | horror | Psycho2 | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 3 | thriller | Vertigo2 | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa + 4 | drama | Yojimbo2 | Akira Kurosawa +(10 rows) + +-- non-colocated join fails +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) JOIN test_table ON(my_films.id != test_table.id) + ORDER BY 1,2,3,4; +ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +-- JSON_TABLE can be in the outer part of the join +-- as long as there is a distributed table +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) LEFT JOIN test_table ON(my_films.id = test_table.id) + ORDER BY 1,2,3,4; + id | kind | title | director +--------------------------------------------------------------------- + 1 | comedy | Bananas | Woody Allen + 1 | comedy | Bananas2 | Woody Allen + 1 | comedy | The Dinner Game | Francis Veber + 1 | comedy | The Dinner Game2 | Francis Veber + 2 | horror | Psycho | Alfred Hitchcock + 2 | horror | Psycho2 | Alfred Hitchcock + 3 | thriller | Vertigo | Alfred Hitchcock + 3 | thriller | Vertigo2 | Alfred Hitchcock + 4 | drama | Yojimbo | Akira Kurosawa + 4 | drama | Yojimbo2 | Akira Kurosawa +(10 rows) + +-- JSON_TABLE can be on the outer side of the join +-- We support outer joins where the outer rel is a recurring one +-- and the inner one is a non-recurring one if we don't reference the outer from the inner +-- https://github.com/citusdata/citus/pull/6512 +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +LEFT JOIN LATERAL + (SELECT * + FROM my_films) AS foo on(foo.id = a); +DEBUG: recursively planning right side of the left join since the outer side is a recurring rel +DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel +DEBUG: generating subplan XXX_1 for subquery SELECT id, js FROM pg17_json.my_films +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT "json_table".id, "json_table".column_a, "json_table".column_b, "json_table".a, "json_table".b, "json_table".c, foo.id, foo.js FROM (JSON_TABLE('[{"a": 10, "b": 20}, {"a": 30, "b": 40}]'::jsonb, '$[*]' AS json_table_path_0 COLUMNS (id FOR ORDINALITY, column_a integer PATH '$."a"', column_b integer PATH '$."b"', a integer PATH '$."a"', b integer PATH '$."b"', c text PATH '$."c"')) LEFT JOIN LATERAL (SELECT intermediate_result.id, intermediate_result.js FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint, js jsonb)) foo ON ((foo.id OPERATOR(pg_catalog.=) "json_table".a))) + id | column_a | column_b | a | b | c | id | js +--------------------------------------------------------------------- + 1 | 10 | 20 | 10 | 20 | | | + 2 | 30 | 40 | 30 | 40 | | | +(2 rows) + +-- However we don't support +-- when we reference the JSON_TABLE from the non-recurring distributed table subquery +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (json_id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +LEFT JOIN LATERAL + (SELECT * + FROM my_films WHERE id::text LIKE c) AS foo on(foo.id = a); +DEBUG: recursively planning right side of the left join since the outer side is a recurring rel +DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel +ERROR: cannot perform a lateral outer join when a distributed subquery references a JSON_TABLE +-- JSON_TABLE cannot be on the FROM clause alone +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (json_id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) as foo +WHERE b > + (SELECT count(*) + FROM my_films WHERE id = foo.a); +ERROR: correlated subqueries are not supported when the FROM clause contains JSON_TABLE +-- we can recursively plan json_tables on set operations +(SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY)) ORDER BY id ASC LIMIT 1) +UNION +(SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY)) ORDER BY id ASC LIMIT 1) +UNION +(SELECT id FROM test_table ORDER BY id ASC LIMIT 1); +DEBUG: generating subplan XXX_1 for subquery SELECT id FROM JSON_TABLE('[{"a": 10, "b": 20}, {"a": 30, "b": 40}]'::jsonb, '$[*]' AS json_table_path_0 COLUMNS (id FOR ORDINALITY)) ORDER BY id LIMIT 1 +DEBUG: generating subplan XXX_2 for subquery SELECT id FROM JSON_TABLE('[{"a": 10, "b": 20}, {"a": 30, "b": 40}]'::jsonb, '$[*]' AS json_table_path_0 COLUMNS (id FOR ORDINALITY)) ORDER BY id LIMIT 1 +DEBUG: push down of limit count: 1 +DEBUG: generating subplan XXX_3 for subquery SELECT id FROM pg17_json.test_table ORDER BY id LIMIT 1 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer) UNION SELECT intermediate_result.id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer) UNION SELECT intermediate_result.id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(id bigint) + id +--------------------------------------------------------------------- + 1 +(1 row) + +-- LIMIT in subquery not supported when json_table exists +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +JOIN LATERAL + (SELECT * + FROM my_films WHERE json_table.id = a LIMIT 1) as foo ON (true); +ERROR: cannot push down this subquery +DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from a JSON_TABLE +RESET client_min_messages; +-- we can use JSON_TABLE in modification queries as well +-- use log level such that we can see trace changes +SET client_min_messages TO DEBUG1; +--the JSON_TABLE subquery is recursively planned +UPDATE test_table SET VALUE = 'XXX' FROM( +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) as foo WHERE foo.id = test_table.id; +DEBUG: generating subplan XXX_1 for subquery SELECT jt.id, jt.kind, jt.title, jt.director FROM pg17_json.my_films, LATERAL JSON_TABLE(my_films.js, '$."favorites"[*]' AS json_table_path_0 COLUMNS (id FOR ORDINALITY, kind text PATH '$."kind"', NESTED PATH '$."films"[*]' AS json_table_path_1 COLUMNS (title text PATH '$."title"', director text PATH '$."director"'))) jt +DEBUG: Plan XXX query after replacing subqueries and CTEs: UPDATE pg17_json.test_table SET value = 'XXX'::text FROM (SELECT intermediate_result.id, intermediate_result.kind, intermediate_result.title, intermediate_result.director FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, kind text, title text, director text)) foo WHERE (foo.id OPERATOR(pg_catalog.=) test_table.id) +-- Subquery with JSON table can be pushed down because two distributed tables +-- in the query are joined on distribution column +UPDATE test_table SET VALUE = 'XXX' FROM ( +SELECT my_films.id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) as foo WHERE foo.id = test_table.id; +-- we can pushdown with CTEs as well +WITH json_cte AS +(SELECT my_films.id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) +UPDATE test_table SET VALUE = 'XYZ' FROM json_cte + WHERE json_cte.id = test_table.id; + -- we can recursively with CTEs as well +WITH json_cte AS +(SELECT my_films.id as film_id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + id FOR ORDINALITY, + title text PATH '$.title', + director text PATH '$.director'))) AS jt ORDER BY jt.id LIMIT 1) +UPDATE test_table SET VALUE = 'XYZ' FROM json_cte + WHERE json_cte.film_id = test_table.id; +DEBUG: generating subplan XXX_1 for CTE json_cte: SELECT my_films.id AS film_id, jt.kind, jt.id, jt.title, jt.director FROM pg17_json.my_films, LATERAL JSON_TABLE(my_films.js, '$."favorites"[*]' AS json_table_path_0 COLUMNS (kind text PATH '$."kind"', NESTED PATH '$."films"[*]' AS json_table_path_1 COLUMNS (id FOR ORDINALITY, title text PATH '$."title"', director text PATH '$."director"'))) jt ORDER BY jt.id LIMIT 1 +DEBUG: push down of limit count: 1 +DEBUG: Plan XXX query after replacing subqueries and CTEs: UPDATE pg17_json.test_table SET value = 'XYZ'::text FROM (SELECT intermediate_result.film_id, intermediate_result.kind, intermediate_result.id, intermediate_result.title, intermediate_result.director FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(film_id bigint, kind text, id integer, title text, director text)) json_cte WHERE (json_cte.film_id OPERATOR(pg_catalog.=) test_table.id) +-- JSON_TABLE NESTED +-- JSON_TABLE: plan execution +-- Check output with Postgres table in sqljson_jsontable test +-- https://github.com/postgres/postgres/blob/REL_17_0/src/test/regress/expected/sqljson_jsontable.out#L776-L814 +CREATE TABLE jsonb_table_test (id bigserial, js jsonb); +DEBUG: CREATE TABLE will create implicit sequence "jsonb_table_test_id_seq" for serial column "jsonb_table_test.id" +SELECT create_distributed_table('jsonb_table_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO jsonb_table_test +VALUES (1, + '[ + {"a": 1, "b": [], "c": []}, + {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]}, + {"a": 3, "b": [1, 2], "c": []}, + {"x": "4", "b": [1, 2], "c": 123} + ]' +); +select + jt.* +from + jsonb_table_test jtt, + json_table ( + jtt.js,'strict $[*]' as p + columns ( + n for ordinality, + a int path 'lax $.a' default -1 on empty, + nested path 'strict $.b[*]' as pb columns (b_id for ordinality, b int path '$' ), + nested path 'strict $.c[*]' as pc columns (c_id for ordinality, c int path '$' ) + ) + ) jt; + n | a | b_id | b | c_id | c +--------------------------------------------------------------------- + 1 | 1 | | | | + 2 | 2 | 1 | 1 | | + 2 | 2 | 2 | 2 | | + 2 | 2 | 3 | 3 | | + 2 | 2 | | | 1 | 10 + 2 | 2 | | | 2 | + 2 | 2 | | | 3 | 20 + 3 | 3 | 1 | 1 | | + 3 | 3 | 2 | 2 | | + 4 | -1 | 1 | 1 | | + 4 | -1 | 2 | 2 | | +(11 rows) + +-- test some utility functions on the target list & where clause: json_exists() +select jsonb_path_exists(js, '$.favorites') from my_films; + jsonb_path_exists +--------------------------------------------------------------------- + t + t +(2 rows) + +select bool_and(JSON_EXISTS(js, '$.favorites.films.title')) from my_films; + bool_and +--------------------------------------------------------------------- + t +(1 row) + +SELECT count(*) FROM my_films WHERE jsonb_path_exists(js, '$.favorites'); + count +--------------------------------------------------------------------- + 2 +(1 row) + +SELECT count(*) FROM my_films WHERE JSON_EXISTS(js, '$.favorites.films.title'); + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- check constraint with json_exists, use json_scalar also +SET citus.shard_replication_factor TO 1; +create table user_profiles ( + id bigserial, + addresses jsonb, + anyjson jsonb, + serialized bytea, + check (json_exists( addresses, '$.main' )) -- we should insert a key named main +); +DEBUG: CREATE TABLE will create implicit sequence "user_profiles_id_seq" for serial column "user_profiles.id" +select create_distributed_table('user_profiles', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into user_profiles (addresses) VALUES (JSON_SCALAR('1')); +ERROR: new row for relation "user_profiles_1687012" violates check constraint "user_profiles_addresses_check" +DETAIL: Failing row contains (1, "1", null, null). +CONTEXT: while executing command on localhost:xxxxx +insert into user_profiles (addresses, anyjson) VALUES ('{"main":"value"}', JSON_SCALAR('1')) RETURNING *; + id | addresses | anyjson | serialized +--------------------------------------------------------------------- + 2 | {"main": "value"} | "1" | +(1 row) + +-- use json() - we cannot insert because WITH UNIQUE KEYS +insert into user_profiles (addresses) VALUES (JSON ('{"main":"value", "main":"value"}' WITH UNIQUE KEYS)); +ERROR: duplicate JSON object key value +-- we can insert with +insert into user_profiles (addresses) VALUES (JSON ('{"main":"value", "main":"value"}' WITHOUT UNIQUE KEYS)) RETURNING *; + id | addresses | anyjson | serialized +--------------------------------------------------------------------- + 4 | {"main": "value"} | | +(1 row) + +-- JSON predicates +TRUNCATE user_profiles; +INSERT INTO user_profiles (anyjson) VALUES ('12'), ('"abc"'), ('[1,2,3]'), ('{"a":12}'); +select anyjson, anyjson is json array as json_array, anyjson is json object as json_object, anyjson is json scalar as json_scalar, +anyjson is json with UNIQUE keys +from user_profiles WHERE anyjson IS NOT NULL ORDER BY 1; + anyjson | json_array | json_object | json_scalar | ?column? +--------------------------------------------------------------------- + "abc" | f | f | t | t + 12 | f | f | t | t + [1, 2, 3] | t | f | f | t + {"a": 12} | f | t | f | t +(4 rows) + +-- use json_serialize +-- it is evaluated in the worker +SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea); + json_serialize +--------------------------------------------------------------------- + \x7b20226122203a2031207d20 +(1 row) + +SET citus.log_remote_commands TO on; +INSERT INTO user_profiles (serialized) VALUES (JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea)) RETURNING *; +NOTICE: issuing INSERT INTO pg17_json.user_profiles_1687015 (id, serialized) VALUES ('9'::bigint, JSON_SERIALIZE('{ "a" : 1 } '::text RETURNING bytea)) RETURNING id, addresses, anyjson, serialized +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + id | addresses | anyjson | serialized +--------------------------------------------------------------------- + 9 | | | \x7b20226122203a2031207d20 +(1 row) + +RESET citus.log_remote_commands; +-- use json_query +SELECT i, + json_query('[{"x": "aaa"},{"x": "bbb"},{"x": "ccc"}]'::JSONB, '$[$i].x' passing id AS i RETURNING text omit quotes) +FROM generate_series(0, 3) i +JOIN my_films ON(id = i) ORDER BY 1; + i | json_query +--------------------------------------------------------------------- + 1 | bbb + 2 | ccc +(2 rows) + +-- use json_value +-- check output with sqljson_queryfuncs test +-- https://github.com/postgres/postgres/blob/REL_17_0/src/test/regress/expected/sqljson_queryfuncs.out#L439-L455 +SELECT i, + JSON_VALUE( + jsonb '{"a": 1, "b": 2}', + '$.* ? (@ > $i)' PASSING id AS i + RETURNING int + DEFAULT -1 ON EMPTY + DEFAULT -2 ON ERROR + ) +FROM generate_series(0, 3) i +JOIN my_films ON(id = i) ORDER BY 1; + i | json_value +--------------------------------------------------------------------- + 1 | 2 + 2 | -1 +(2 rows) + +SET client_min_messages TO ERROR; +DROP SCHEMA pg17_json CASCADE; diff --git a/src/test/regress/expected/pg17_json_0.out b/src/test/regress/expected/pg17_json_0.out new file mode 100644 index 000000000..7e603dc6c --- /dev/null +++ b/src/test/regress/expected/pg17_json_0.out @@ -0,0 +1,18 @@ +-- +-- PG17_JSON +-- PG17 has added basic JSON_TABLE() functionality +-- JSON_TABLE() allows JSON data to be converted into a relational view +-- and thus used, for example, in a FROM clause, like other tabular +-- data. We treat JSON_TABLE the same as correlated functions (e.g., recurring tuples). +-- In the end, for multi-shard JSON_TABLE commands, we apply the same +-- restrictions as reference tables (e.g., cannot perform a lateral outer join +-- when a distributed subquery references a (reference table)/JSON_TABLE etc.) +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/de3600452 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +\else +\q diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 0fa54bb38..136311a2a 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -66,7 +66,7 @@ test: pg14 test: pg15 test: pg15_jsonpath detect_conn_close test: pg16 -test: pg17 +test: pg17 pg17_json test: drop_column_partitioned_table test: tableam diff --git a/src/test/regress/sql/pg17_json.sql b/src/test/regress/sql/pg17_json.sql new file mode 100644 index 000000000..b28f5abea --- /dev/null +++ b/src/test/regress/sql/pg17_json.sql @@ -0,0 +1,379 @@ +-- +-- PG17_JSON +-- PG17 has added basic JSON_TABLE() functionality +-- JSON_TABLE() allows JSON data to be converted into a relational view +-- and thus used, for example, in a FROM clause, like other tabular +-- data. We treat JSON_TABLE the same as correlated functions (e.g., recurring tuples). +-- In the end, for multi-shard JSON_TABLE commands, we apply the same +-- restrictions as reference tables (e.g., cannot perform a lateral outer join +-- when a distributed subquery references a (reference table)/JSON_TABLE etc.) +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/de3600452 +-- + +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 17 AS server_version_ge_17 +\gset +\if :server_version_ge_17 +\else +\q +\endif + +CREATE SCHEMA pg17_json; +SET search_path TO pg17_json; + +SET citus.next_shard_id TO 1687000; + +CREATE TABLE test_table(id bigserial, value text); +SELECT create_distributed_table('test_table', 'id'); +INSERT INTO test_table (value) SELECT i::text FROM generate_series(0,100)i; + + +CREATE TABLE my_films(id bigserial, js jsonb); +SELECT create_distributed_table('my_films', 'id'); + +INSERT INTO my_films(js) VALUES ( +'{ "favorites" : [ + { "kind" : "comedy", "films" : [ { "title" : "Bananas", "director" : "Woody Allen"}, + { "title" : "The Dinner Game", "director" : "Francis Veber" } ] }, + { "kind" : "horror", "films" : [{ "title" : "Psycho", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "thriller", "films" : [{ "title" : "Vertigo", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "drama", "films" : [{ "title" : "Yojimbo", "director" : "Akira Kurosawa" } ] } + ] }'); + +INSERT INTO my_films(js) VALUES ( +'{ "favorites" : [ + { "kind" : "comedy", "films" : [ { "title" : "Bananas2", "director" : "Woody Allen"}, + { "title" : "The Dinner Game2", "director" : "Francis Veber" } ] }, + { "kind" : "horror", "films" : [{ "title" : "Psycho2", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "thriller", "films" : [{ "title" : "Vertigo2", "director" : "Alfred Hitchcock" } ] }, + { "kind" : "drama", "films" : [{ "title" : "Yojimbo2", "director" : "Akira Kurosawa" } ] } + ] }'); + +-- a router query +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + WHERE my_films.id = 1 + ORDER BY 1,2,3,4; + +-- router query with an explicit LATEREL SUBQUERY +SELECT sub.* +FROM my_films, + lateral(SELECT * FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt) as sub +WHERE my_films.id = 1; + +-- router query with an explicit LATEREL SUBQUERY and LIMIT +SELECT sub.* +FROM my_films, + lateral(SELECT * FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt ORDER BY id DESC LIMIT 1) as sub +WHERE my_films.id = 1; + +-- set it DEBUG1 in case the plan changes +-- we can see details +SET client_min_messages TO DEBUG1; + +-- a mult-shard query +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + ORDER BY 1,2,3,4; + +-- recursively plan subqueries that has JSON_TABLE +SELECT count(*) FROM +( + SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt + LIMIT 1) as sub_with_json, test_table +WHERE test_table.id = sub_with_json.id; + + +-- multi-shard query with an explicit LATEREL SUBQUERY +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) + ORDER BY 1,2,3,4; + +-- JSON_TABLE can be on the inner part of an outer joion +SELECT sub.* +FROM my_films LEFT JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) + ORDER BY 1,2,3,4; + +-- we can pushdown this correlated subquery in WHERE clause +SELECT count(*) +FROM my_films WHERE + (SELECT count(*) > 0 + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000); + +-- we can pushdown this correlated subquery in SELECT clause + SELECT (SELECT count(*) > 0 + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt) +FROM my_films; + +-- multi-shard query with an explicit LATEREL SUBQUERY +-- along with other tables +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) JOIN test_table ON(my_films.id = test_table.id) + ORDER BY 1,2,3,4; + +-- non-colocated join fails +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) JOIN test_table ON(my_films.id != test_table.id) + ORDER BY 1,2,3,4; + +-- JSON_TABLE can be in the outer part of the join +-- as long as there is a distributed table +SELECT sub.* +FROM my_films JOIN + lateral + (SELECT * + FROM JSON_TABLE (js, '$.favorites[*]' COLUMNS (id FOR ORDINALITY, + kind text PATH '$.kind', NESTED PATH '$.films[*]' + COLUMNS (title text PATH '$.title', director text PATH '$.director'))) AS jt + LIMIT 1000) AS sub ON (true) LEFT JOIN test_table ON(my_films.id = test_table.id) + ORDER BY 1,2,3,4; + +-- JSON_TABLE can be on the outer side of the join +-- We support outer joins where the outer rel is a recurring one +-- and the inner one is a non-recurring one if we don't reference the outer from the inner +-- https://github.com/citusdata/citus/pull/6512 + +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +LEFT JOIN LATERAL + (SELECT * + FROM my_films) AS foo on(foo.id = a); + +-- However we don't support +-- when we reference the JSON_TABLE from the non-recurring distributed table subquery +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (json_id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +LEFT JOIN LATERAL + (SELECT * + FROM my_films WHERE id::text LIKE c) AS foo on(foo.id = a); + +-- JSON_TABLE cannot be on the FROM clause alone +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (json_id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) as foo +WHERE b > + (SELECT count(*) + FROM my_films WHERE id = foo.a); + +-- we can recursively plan json_tables on set operations +(SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY)) ORDER BY id ASC LIMIT 1) +UNION +(SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY)) ORDER BY id ASC LIMIT 1) +UNION +(SELECT id FROM test_table ORDER BY id ASC LIMIT 1); + +-- LIMIT in subquery not supported when json_table exists +SELECT * +FROM json_table('[{"a":10,"b":20},{"a":30,"b":40}]'::JSONB, '$[*]' + COLUMNS (id FOR ORDINALITY, column_a int4 PATH '$.a', column_b int4 PATH '$.b', a int4, b int4, c text)) +JOIN LATERAL + (SELECT * + FROM my_films WHERE json_table.id = a LIMIT 1) as foo ON (true); + +RESET client_min_messages; + +-- we can use JSON_TABLE in modification queries as well + +-- use log level such that we can see trace changes +SET client_min_messages TO DEBUG1; + +--the JSON_TABLE subquery is recursively planned +UPDATE test_table SET VALUE = 'XXX' FROM( +SELECT jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + id FOR ORDINALITY, + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) as foo WHERE foo.id = test_table.id; + +-- Subquery with JSON table can be pushed down because two distributed tables +-- in the query are joined on distribution column +UPDATE test_table SET VALUE = 'XXX' FROM ( +SELECT my_films.id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) as foo WHERE foo.id = test_table.id; + +-- we can pushdown with CTEs as well +WITH json_cte AS +(SELECT my_films.id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + title text PATH '$.title', + director text PATH '$.director'))) AS jt) +UPDATE test_table SET VALUE = 'XYZ' FROM json_cte + WHERE json_cte.id = test_table.id; + + -- we can recursively with CTEs as well +WITH json_cte AS +(SELECT my_films.id as film_id, jt.* FROM + my_films, + JSON_TABLE ( js, '$.favorites[*]' COLUMNS ( + kind text PATH '$.kind', + NESTED PATH '$.films[*]' COLUMNS ( + id FOR ORDINALITY, + title text PATH '$.title', + director text PATH '$.director'))) AS jt ORDER BY jt.id LIMIT 1) +UPDATE test_table SET VALUE = 'XYZ' FROM json_cte + WHERE json_cte.film_id = test_table.id; + +-- JSON_TABLE NESTED +-- JSON_TABLE: plan execution +-- Check output with Postgres table in sqljson_jsontable test +-- https://github.com/postgres/postgres/blob/REL_17_0/src/test/regress/expected/sqljson_jsontable.out#L776-L814 + +CREATE TABLE jsonb_table_test (id bigserial, js jsonb); +SELECT create_distributed_table('jsonb_table_test', 'id'); + +INSERT INTO jsonb_table_test +VALUES (1, + '[ + {"a": 1, "b": [], "c": []}, + {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]}, + {"a": 3, "b": [1, 2], "c": []}, + {"x": "4", "b": [1, 2], "c": 123} + ]' +); + +select + jt.* +from + jsonb_table_test jtt, + json_table ( + jtt.js,'strict $[*]' as p + columns ( + n for ordinality, + a int path 'lax $.a' default -1 on empty, + nested path 'strict $.b[*]' as pb columns (b_id for ordinality, b int path '$' ), + nested path 'strict $.c[*]' as pc columns (c_id for ordinality, c int path '$' ) + ) + ) jt; + +-- test some utility functions on the target list & where clause: json_exists() +select jsonb_path_exists(js, '$.favorites') from my_films; +select bool_and(JSON_EXISTS(js, '$.favorites.films.title')) from my_films; +SELECT count(*) FROM my_films WHERE jsonb_path_exists(js, '$.favorites'); +SELECT count(*) FROM my_films WHERE JSON_EXISTS(js, '$.favorites.films.title'); + +-- check constraint with json_exists, use json_scalar also +SET citus.shard_replication_factor TO 1; +create table user_profiles ( + id bigserial, + addresses jsonb, + anyjson jsonb, + serialized bytea, + check (json_exists( addresses, '$.main' )) -- we should insert a key named main +); +select create_distributed_table('user_profiles', 'id'); + +insert into user_profiles (addresses) VALUES (JSON_SCALAR('1')); +insert into user_profiles (addresses, anyjson) VALUES ('{"main":"value"}', JSON_SCALAR('1')) RETURNING *; + +-- use json() - we cannot insert because WITH UNIQUE KEYS +insert into user_profiles (addresses) VALUES (JSON ('{"main":"value", "main":"value"}' WITH UNIQUE KEYS)); + +-- we can insert with +insert into user_profiles (addresses) VALUES (JSON ('{"main":"value", "main":"value"}' WITHOUT UNIQUE KEYS)) RETURNING *; + +-- JSON predicates +TRUNCATE user_profiles; +INSERT INTO user_profiles (anyjson) VALUES ('12'), ('"abc"'), ('[1,2,3]'), ('{"a":12}'); +select anyjson, anyjson is json array as json_array, anyjson is json object as json_object, anyjson is json scalar as json_scalar, +anyjson is json with UNIQUE keys +from user_profiles WHERE anyjson IS NOT NULL ORDER BY 1; + +-- use json_serialize +-- it is evaluated in the worker +SELECT JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea); +SET citus.log_remote_commands TO on; +INSERT INTO user_profiles (serialized) VALUES (JSON_SERIALIZE('{ "a" : 1 } ' RETURNING bytea)) RETURNING *; +RESET citus.log_remote_commands; + +-- use json_query +SELECT i, + json_query('[{"x": "aaa"},{"x": "bbb"},{"x": "ccc"}]'::JSONB, '$[$i].x' passing id AS i RETURNING text omit quotes) +FROM generate_series(0, 3) i +JOIN my_films ON(id = i) ORDER BY 1; + +-- use json_value +-- check output with sqljson_queryfuncs test +-- https://github.com/postgres/postgres/blob/REL_17_0/src/test/regress/expected/sqljson_queryfuncs.out#L439-L455 +SELECT i, + JSON_VALUE( + jsonb '{"a": 1, "b": 2}', + '$.* ? (@ > $i)' PASSING id AS i + RETURNING int + DEFAULT -1 ON EMPTY + DEFAULT -2 ON ERROR + ) +FROM generate_series(0, 3) i +JOIN my_films ON(id = i) ORDER BY 1; + +SET client_min_messages TO ERROR; +DROP SCHEMA pg17_json CASCADE; From 7dc97d3dc9d823f1fe23c15a32b200899b7faba7 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 30 Dec 2024 20:27:28 +0300 Subject: [PATCH 102/155] Disallow infinite values for partition interval in create_time_partitions udf (#7822) PG17 added +/- infinity values for the interval data type Relevant PG commit: https://github.com/postgres/postgres/commit/519fc1bd9 --- .../distributed/sql/citus--12.1-1--13.0-1.sql | 1 + .../sql/downgrades/citus--13.0-1--12.1-1.sql | 2 + .../udfs/create_time_partitions/13.0-1.sql | 58 +++++++++++++++++++ .../udfs/create_time_partitions/latest.sql | 4 ++ src/test/regress/expected/pg17.out | 15 +++++ src/test/regress/sql/pg17.sql | 15 +++++ 6 files changed, 95 insertions(+) create mode 100644 src/backend/distributed/sql/udfs/create_time_partitions/13.0-1.sql diff --git a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql index 216171664..b19ddfb75 100644 --- a/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--13.0-1.sql @@ -2,3 +2,4 @@ -- bump version to 13.0-1 #include "udfs/citus_prepare_pg_upgrade/13.0-1.sql" +#include "udfs/create_time_partitions/13.0-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql b/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql index 006349990..681ec6e82 100644 --- a/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql @@ -1,2 +1,4 @@ -- citus--13.0-1--12.1-1 -- this is an empty downgrade path since citus--12.1-1--13.0-1.sql is empty + +#include "../udfs/create_time_partitions/10.2-1.sql" diff --git a/src/backend/distributed/sql/udfs/create_time_partitions/13.0-1.sql b/src/backend/distributed/sql/udfs/create_time_partitions/13.0-1.sql new file mode 100644 index 000000000..566ba7163 --- /dev/null +++ b/src/backend/distributed/sql/udfs/create_time_partitions/13.0-1.sql @@ -0,0 +1,58 @@ +CREATE OR REPLACE FUNCTION pg_catalog.create_time_partitions( + table_name regclass, + partition_interval INTERVAL, + end_at timestamptz, + start_from timestamptz DEFAULT now()) +returns boolean +LANGUAGE plpgsql +AS $$ +DECLARE + -- partitioned table name + schema_name_text name; + table_name_text name; + + -- record for to-be-created partition + missing_partition_record record; + + -- result indiciates whether any partitions were created + partition_created bool := false; +BEGIN + IF start_from >= end_at THEN + RAISE 'start_from (%) must be older than end_at (%)', start_from, end_at; + END IF; + + IF NOT isfinite(partition_interval) THEN + RAISE 'Partition interval must be a finite value'; + END IF; + + SELECT nspname, relname + INTO schema_name_text, table_name_text + FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE pg_class.oid = table_name::oid; + + -- Get missing partition range info using the get_missing_partition_ranges + -- and create partitions using that info. + FOR missing_partition_record IN + SELECT * + FROM get_missing_time_partition_ranges(table_name, partition_interval, end_at, start_from) + LOOP + EXECUTE format('CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%L) TO (%L)', + schema_name_text, + missing_partition_record.partition_name, + schema_name_text, + table_name_text, + missing_partition_record.range_from_value, + missing_partition_record.range_to_value); + + partition_created := true; + END LOOP; + + RETURN partition_created; +END; +$$; +COMMENT ON FUNCTION pg_catalog.create_time_partitions( + table_name regclass, + partition_interval INTERVAL, + end_at timestamptz, + start_from timestamptz) +IS 'create time partitions for the given range'; diff --git a/src/backend/distributed/sql/udfs/create_time_partitions/latest.sql b/src/backend/distributed/sql/udfs/create_time_partitions/latest.sql index 11edcc5ac..566ba7163 100644 --- a/src/backend/distributed/sql/udfs/create_time_partitions/latest.sql +++ b/src/backend/distributed/sql/udfs/create_time_partitions/latest.sql @@ -21,6 +21,10 @@ BEGIN RAISE 'start_from (%) must be older than end_at (%)', start_from, end_at; END IF; + IF NOT isfinite(partition_interval) THEN + RAISE 'Partition interval must be a finite value'; + END IF; + SELECT nspname, relname INTO schema_name_text, table_name_text FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index ff1e57d74..1141efef0 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1453,6 +1453,21 @@ ROLLBACK; NOTICE: issuing ROLLBACK DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- End of Testing AT LOCAL option +-- interval can have infinite values +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/519fc1bd9 +-- disallow those in create_time_partitions +-- test create_time_partitions with infinity values +CREATE TABLE date_partitioned_table( + measureid integer, + eventdate date, + measure_data jsonb) PARTITION BY RANGE(eventdate); +SELECT create_time_partitions('date_partitioned_table', INTERVAL 'infinity', '2022-01-01', '2021-01-01'); +ERROR: Partition interval must be a finite value +CONTEXT: PL/pgSQL function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at RAISE +SELECT create_time_partitions('date_partitioned_table', INTERVAL '-infinity', '2022-01-01', '2021-01-01'); +ERROR: Partition interval must be a finite value +CONTEXT: PL/pgSQL function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at RAISE +-- end of testing interval with infinite values \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index fd3a6ddfd..888a0463c 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -797,6 +797,21 @@ ROLLBACK; -- End of Testing AT LOCAL option +-- interval can have infinite values +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/519fc1bd9 +-- disallow those in create_time_partitions + +-- test create_time_partitions with infinity values +CREATE TABLE date_partitioned_table( + measureid integer, + eventdate date, + measure_data jsonb) PARTITION BY RANGE(eventdate); + +SELECT create_time_partitions('date_partitioned_table', INTERVAL 'infinity', '2022-01-01', '2021-01-01'); +SELECT create_time_partitions('date_partitioned_table', INTERVAL '-infinity', '2022-01-01', '2021-01-01'); + +-- end of testing interval with infinite values + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From b3448a166192fb46c7d0b008d7c04892c453be88 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 30 Dec 2024 21:06:30 +0300 Subject: [PATCH 103/155] Add pg17 jsonpath methods tests (#7820) various jsonpath methods were added in PG17 Relevant PG commit: https://github.com/postgres/postgres/commit/66ea94e8e Here we add the same test as in pg15_jsonpath.sql for the new additions --- src/test/regress/expected/pg17.out | 51 ++++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 32 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 1141efef0..74504d340 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1468,6 +1468,57 @@ SELECT create_time_partitions('date_partitioned_table', INTERVAL '-infinity', '2 ERROR: Partition interval must be a finite value CONTEXT: PL/pgSQL function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at RAISE -- end of testing interval with infinite values +-- various jsonpath methods were added in PG17 +-- relevant PG commit: https://github.com/postgres/postgres/commit/66ea94e8e +-- here we add the same test as in pg15_jsonpath.sql for the new additions +CREATE TABLE jsonpath_test (id serial, sample text); +SELECT create_distributed_table('jsonpath_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\COPY jsonpath_test(sample) FROM STDIN +-- Cast the text into jsonpath on the worker nodes. +SELECT sample, sample::jsonpath FROM jsonpath_test ORDER BY id; + sample | sample +--------------------------------------------------------------------- + $.bigint().integer().number().decimal() | $.bigint().integer().number().decimal() + $.boolean() | $.boolean() + $.date() | $.date() + $.decimal(4,2) | $.decimal(4,2) + $.string() | $.string() + $.time() | $.time() + $.time(6) | $.time(6) + $.time_tz() | $.time_tz() + $.time_tz(4) | $.time_tz(4) + $.timestamp() | $.timestamp() + $.timestamp(2) | $.timestamp(2) + $.timestamp_tz() | $.timestamp_tz() + $.timestamp_tz(0) | $.timestamp_tz(0) +(13 rows) + +-- Pull the data, and cast on the coordinator node +WITH samples as (SELECT id, sample FROM jsonpath_test OFFSET 0) +SELECT sample, sample::jsonpath FROM samples ORDER BY id; + sample | sample +--------------------------------------------------------------------- + $.bigint().integer().number().decimal() | $.bigint().integer().number().decimal() + $.boolean() | $.boolean() + $.date() | $.date() + $.decimal(4,2) | $.decimal(4,2) + $.string() | $.string() + $.time() | $.time() + $.time(6) | $.time(6) + $.time_tz() | $.time_tz() + $.time_tz(4) | $.time_tz(4) + $.timestamp() | $.timestamp() + $.timestamp(2) | $.timestamp(2) + $.timestamp_tz() | $.timestamp_tz() + $.timestamp_tz(0) | $.timestamp_tz(0) +(13 rows) + +-- End of testing jsonpath methods \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 888a0463c..3d400a525 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -812,6 +812,38 @@ SELECT create_time_partitions('date_partitioned_table', INTERVAL '-infinity', '2 -- end of testing interval with infinite values +-- various jsonpath methods were added in PG17 +-- relevant PG commit: https://github.com/postgres/postgres/commit/66ea94e8e +-- here we add the same test as in pg15_jsonpath.sql for the new additions + +CREATE TABLE jsonpath_test (id serial, sample text); +SELECT create_distributed_table('jsonpath_test', 'id'); + +\COPY jsonpath_test(sample) FROM STDIN +$.bigint().integer().number().decimal() +$.boolean() +$.date() +$.decimal(4,2) +$.string() +$.time() +$.time(6) +$.time_tz() +$.time_tz(4) +$.timestamp() +$.timestamp(2) +$.timestamp_tz() +$.timestamp_tz(0) +\. + +-- Cast the text into jsonpath on the worker nodes. +SELECT sample, sample::jsonpath FROM jsonpath_test ORDER BY id; + +-- Pull the data, and cast on the coordinator node +WITH samples as (SELECT id, sample FROM jsonpath_test OFFSET 0) +SELECT sample, sample::jsonpath FROM samples ORDER BY id; + +-- End of testing jsonpath methods + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 80678bb07e0b74fe8d8556902c30de5d5a5900aa Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 30 Dec 2024 21:25:50 +0300 Subject: [PATCH 104/155] Allow configuring sslnegotiation using citus.node_conn_info (#7821) Relevant PG commit: https://github.com/postgres/postgres/commit/d39a49c1e PR similar to https://github.com/citusdata/citus/pull/5203 --- src/backend/distributed/shared_library_init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index ff16fa2df..d004b1989 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -2905,6 +2905,9 @@ NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source) "sslcrl", "sslkey", "sslmode", +#if PG_VERSION_NUM >= PG_VERSION_17 + "sslnegotiation", +#endif "sslrootcert", "tcp_user_timeout", }; From bee8e9cbb6086d9bc50dfa6e1e8a2985b7565ec5 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:49:37 +0300 Subject: [PATCH 105/155] Fix foreign_key_to_reference_shard_rebalance test (#7826) foreign_key_to_reference_shard_rebalance failed because partition of 2024 year does not exist, fixed by add default partition. Replaces https://github.com/citusdata/citus/pull/7396 by adding a rule that allows properly testing foreign_key_to_reference_shard_rebalance via run_test.py. Closes #7396 Co-authored-by: chuhx <148182736+cstarc1@users.noreply.github.com> (cherry picked from commit 968ac74cdef96b76f77fcd533c6464fad536d9b0) Co-authored-by: Onur Tirtir --- src/test/regress/citus_tests/run_test.py | 3 +++ .../expected/foreign_key_to_reference_shard_rebalance.out | 2 +- .../regress/sql/foreign_key_to_reference_shard_rebalance.sql | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 67993cecc..2b1c5210c 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -181,6 +181,9 @@ DEPS = { "multi_test_helpers_superuser", ], ), + "foreign_key_to_reference_shard_rebalance": TestDeps( + "minimal_schedule", ["remove_coordinator_from_metadata"] + ), } diff --git a/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out b/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out index bae1895f9..38b82ed68 100644 --- a/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out +++ b/src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out @@ -210,7 +210,7 @@ select create_distributed_table('partitioned_tbl_with_fkey','x'); create table partition_1_with_fkey partition of partitioned_tbl_with_fkey for values from ('2022-01-01') to ('2022-12-31'); create table partition_2_with_fkey partition of partitioned_tbl_with_fkey for values from ('2023-01-01') to ('2023-12-31'); -create table partition_3_with_fkey partition of partitioned_tbl_with_fkey for values from ('2024-01-01') to ('2024-12-31'); +create table partition_3_with_fkey partition of partitioned_tbl_with_fkey DEFAULT; insert into partitioned_tbl_with_fkey (x,y) select s,s%10 from generate_series(1,100) s; ALTER TABLE partitioned_tbl_with_fkey ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (y) REFERENCES ref_table_with_fkey(id); WITH shardid AS (SELECT shardid FROM pg_dist_shard where logicalrelid = 'partitioned_tbl_with_fkey'::regclass ORDER BY shardid LIMIT 1) diff --git a/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql b/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql index 0b9826cb7..b1017768d 100644 --- a/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql +++ b/src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql @@ -84,7 +84,7 @@ create table partitioned_tbl_with_fkey (x int, y int, t timestamptz default now( select create_distributed_table('partitioned_tbl_with_fkey','x'); create table partition_1_with_fkey partition of partitioned_tbl_with_fkey for values from ('2022-01-01') to ('2022-12-31'); create table partition_2_with_fkey partition of partitioned_tbl_with_fkey for values from ('2023-01-01') to ('2023-12-31'); -create table partition_3_with_fkey partition of partitioned_tbl_with_fkey for values from ('2024-01-01') to ('2024-12-31'); +create table partition_3_with_fkey partition of partitioned_tbl_with_fkey DEFAULT; insert into partitioned_tbl_with_fkey (x,y) select s,s%10 from generate_series(1,100) s; ALTER TABLE partitioned_tbl_with_fkey ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (y) REFERENCES ref_table_with_fkey(id); From 103125fa086908ebc7ce68c6031cdcaacc756d1f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 31 Dec 2024 17:36:53 +0300 Subject: [PATCH 106/155] Add tests with xmltext() and random(min, max) (#7824) xmltext() converts text into xml text nodes. Test with columnar and citus tables. Relevant PG17 commit: https://github.com/postgres/postgres/commit/526fe0d79 random(min, max) generates random numbers in a specified range Add tests like the ones for random() in aggregate_support.sql References: https://github.com/citusdata/citus/blob/main/src/test/regress/sql/aggregate_support.sql#L493-L532 https://github.com/citusdata/citus/pull/7183 Relevant PG17 commit: https://github.com/postgres/postgres/commit/e6341323a --- src/test/regress/expected/pg17.out | 87 ++++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 51 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 74504d340..15531d035 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1519,6 +1519,93 @@ SELECT sample, sample::jsonpath FROM samples ORDER BY id; (13 rows) -- End of testing jsonpath methods +-- xmltext() function added in PG17, test with columnar and distributed table +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/526fe0d79 +CREATE TABLE test_xml (id int, a xml) USING columnar; +-- expected to insert x<P>73</P>0.42truej +INSERT INTO test_xml VALUES (1, xmltext('x'|| '

73

'::xml || .42 || true || 'j'::char)); +SELECT * FROM test_xml ORDER BY 1; + id | a +--------------------------------------------------------------------- + 1 | x<P>73</P>0.42truej +(1 row) + +SELECT create_distributed_table('test_xml', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg17.test_xml$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- expected to insert foo & <"bar"> +INSERT INTO test_xml VALUES (2, xmltext('foo & <"bar">')); +SELECT * FROM test_xml ORDER BY 1; + id | a +--------------------------------------------------------------------- + 1 | x<P>73</P>0.42truej + 2 | foo & <"bar"> +(2 rows) + +-- end of xmltest() testing with Citus +-- +-- random(min, max) to generate random numbers in a specified range +-- adding here the same tests as the ones with random() in aggregate_support.sql +-- Relevant PG commit: https://github.com/postgres/postgres/commit/e6341323a +-- +CREATE TABLE dist_table (dist_col int, agg_col numeric); +SELECT create_distributed_table('dist_table', 'dist_col'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE ref_table (int_col int); +SELECT create_reference_table('ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Test the cases where the worker agg exec. returns no tuples. +SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col) +FROM (SELECT *, random(0, 1) FROM dist_table) a; + percentile_disc +--------------------------------------------------------------------- + +(1 row) + +SELECT PERCENTILE_DISC((2 > random(0, 1))::int::numeric / 10) + WITHIN GROUP (ORDER BY agg_col) +FROM dist_table +LEFT JOIN ref_table ON TRUE; + percentile_disc +--------------------------------------------------------------------- + +(1 row) + +-- run the same queries after loading some data +INSERT INTO dist_table VALUES (2, 11.2), (3, NULL), (6, 3.22), (3, 4.23), (5, 5.25), + (4, 63.4), (75, NULL), (80, NULL), (96, NULL), (8, 1078), (0, 1.19); +SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col) +FROM (SELECT *, random(0, 1) FROM dist_table) a; + percentile_disc +--------------------------------------------------------------------- + 3.22 +(1 row) + +SELECT PERCENTILE_DISC((2 > random_normal(0, 1))::int::numeric / 10) + WITHIN GROUP (ORDER BY agg_col) +FROM dist_table +LEFT JOIN ref_table ON TRUE; + percentile_disc +--------------------------------------------------------------------- + 1.19 +(1 row) + +-- End of random(min, max) testing with Citus \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 3d400a525..a1d9c4244 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -844,6 +844,57 @@ SELECT sample, sample::jsonpath FROM samples ORDER BY id; -- End of testing jsonpath methods +-- xmltext() function added in PG17, test with columnar and distributed table +-- Relevant PG17 commit: https://github.com/postgres/postgres/commit/526fe0d79 +CREATE TABLE test_xml (id int, a xml) USING columnar; +-- expected to insert x<P>73</P>0.42truej +INSERT INTO test_xml VALUES (1, xmltext('x'|| '

73

'::xml || .42 || true || 'j'::char)); +SELECT * FROM test_xml ORDER BY 1; + +SELECT create_distributed_table('test_xml', 'id'); +-- expected to insert foo & <"bar"> +INSERT INTO test_xml VALUES (2, xmltext('foo & <"bar">')); +SELECT * FROM test_xml ORDER BY 1; + +-- end of xmltest() testing with Citus + +-- +-- random(min, max) to generate random numbers in a specified range +-- adding here the same tests as the ones with random() in aggregate_support.sql +-- Relevant PG commit: https://github.com/postgres/postgres/commit/e6341323a +-- + +CREATE TABLE dist_table (dist_col int, agg_col numeric); +SELECT create_distributed_table('dist_table', 'dist_col'); + +CREATE TABLE ref_table (int_col int); +SELECT create_reference_table('ref_table'); + +-- Test the cases where the worker agg exec. returns no tuples. + +SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col) +FROM (SELECT *, random(0, 1) FROM dist_table) a; + +SELECT PERCENTILE_DISC((2 > random(0, 1))::int::numeric / 10) + WITHIN GROUP (ORDER BY agg_col) +FROM dist_table +LEFT JOIN ref_table ON TRUE; + +-- run the same queries after loading some data + +INSERT INTO dist_table VALUES (2, 11.2), (3, NULL), (6, 3.22), (3, 4.23), (5, 5.25), + (4, 63.4), (75, NULL), (80, NULL), (96, NULL), (8, 1078), (0, 1.19); + +SELECT PERCENTILE_DISC(.25) WITHIN GROUP (ORDER BY agg_col) +FROM (SELECT *, random(0, 1) FROM dist_table) a; + +SELECT PERCENTILE_DISC((2 > random_normal(0, 1))::int::numeric / 10) + WITHIN GROUP (ORDER BY agg_col) +FROM dist_table +LEFT JOIN ref_table ON TRUE; + +-- End of random(min, max) testing with Citus + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From a5780c519f9713fa9517809a23141e720517e3c5 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Tue, 31 Dec 2024 22:51:43 +0300 Subject: [PATCH 107/155] PG17 - Add Regression Test for Access Method Behavior on Partitioned Tables (#7818) This PR adds a regression test to verify the behavior of access methods for partitioned and distributed tables, including: - Creating partitioned tables with heap. - Distributing tables using create_distributed_table. - Switching access methods to columnar with ALTER TABLE. - Validating access method inheritance for new partitions. Relecant PG17 commit: https://github.com/postgres/postgres/commit/374c7a229 --- src/test/regress/expected/pg17.out | 118 +++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 92 ++++++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 15531d035..f2c4183f2 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1606,6 +1606,124 @@ LEFT JOIN ref_table ON TRUE; (1 row) -- End of random(min, max) testing with Citus +-- Test: Access Method Behavior for Partitioned Tables +-- This test verifies the ability to specify and modify table access methods for partitioned tables +-- using CREATE TABLE ... USING and ALTER TABLE ... SET ACCESS METHOD, including distributed tables. +-- Step 1: Create a partitioned table with a specified access method +CREATE TABLE test_partitioned_alter (id INT PRIMARY KEY, value TEXT) +PARTITION BY RANGE (id) +USING heap; +-- Step 2: Create partitions for the partitioned table +CREATE TABLE test_partition_1 PARTITION OF test_partitioned_alter + FOR VALUES FROM (0) TO (100); +CREATE TABLE test_partition_2 PARTITION OF test_partitioned_alter + FOR VALUES FROM (100) TO (200); +-- Step 3: Distribute the partitioned table +SELECT create_distributed_table('test_partitioned_alter', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Step 4: Verify that the table and partitions are created and distributed correctly on the coordinator +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + relname | relam +--------------------------------------------------------------------- + test_partitioned_alter | 2 +(1 row) + +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + relname | relam +--------------------------------------------------------------------- + test_partition_1 | 2 + test_partition_2 | 2 +(2 rows) + +-- Step 4 (Repeat on a Worker Node): Verify that the table and partitions are created correctly +\c - - - :worker_1_port +SET search_path TO pg17; +-- Verify the table's access method on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + relname | relam +--------------------------------------------------------------------- + test_partitioned_alter | 2 +(1 row) + +-- Verify the partitions' access methods on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + relname | relam +--------------------------------------------------------------------- + test_partition_1 | 2 + test_partition_2 | 2 +(2 rows) + +\c - - - :master_port +SET search_path TO pg17; +-- Step 5: Test ALTER TABLE ... SET ACCESS METHOD to a different method +ALTER TABLE test_partitioned_alter SET ACCESS METHOD columnar; +-- Verify the access method in the distributed parent and existing partitions +-- Note: Specifying an access method for a partitioned table lets the value be used for all +-- future partitions created under it, closely mirroring the behavior of the TABLESPACE +-- option for partitioned tables. Existing partitions are not modified. +-- Reference: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a2290429eac3217b0c7b0b485db9c2bcc72 +-- Verify the parent table's access method +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + relname | relam +--------------------------------------------------------------------- + test_partitioned_alter | 16413 +(1 row) + +-- Verify the partitions' access methods +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + relname | relam +--------------------------------------------------------------------- + test_partition_1 | 2 + test_partition_2 | 2 +(2 rows) + +-- Step 6: Verify the change is applied to future partitions +CREATE TABLE test_partition_3 PARTITION OF test_partitioned_alter + FOR VALUES FROM (200) TO (300); +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partition_3'; + relname | relam +--------------------------------------------------------------------- + test_partition_3 | 16413 +(1 row) + +-- Step 6 (Repeat on a Worker Node): Verify that the new partition is created correctly +\c - - - :worker_1_port +SET search_path TO pg17; +-- Verify the new partition's access method on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partition_3'; + relname | relam +--------------------------------------------------------------------- + test_partition_3 | 16413 +(1 row) + +\c - - - :master_port +SET search_path TO pg17; +-- Clean up +DROP TABLE test_partitioned_alter CASCADE; +-- End of Test: Access Method Behavior for Partitioned Tables \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index a1d9c4244..f6f7c5367 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -895,6 +895,98 @@ LEFT JOIN ref_table ON TRUE; -- End of random(min, max) testing with Citus +-- Test: Access Method Behavior for Partitioned Tables +-- This test verifies the ability to specify and modify table access methods for partitioned tables +-- using CREATE TABLE ... USING and ALTER TABLE ... SET ACCESS METHOD, including distributed tables. + +-- Step 1: Create a partitioned table with a specified access method +CREATE TABLE test_partitioned_alter (id INT PRIMARY KEY, value TEXT) +PARTITION BY RANGE (id) +USING heap; + +-- Step 2: Create partitions for the partitioned table +CREATE TABLE test_partition_1 PARTITION OF test_partitioned_alter + FOR VALUES FROM (0) TO (100); + +CREATE TABLE test_partition_2 PARTITION OF test_partitioned_alter + FOR VALUES FROM (100) TO (200); + +-- Step 3: Distribute the partitioned table +SELECT create_distributed_table('test_partitioned_alter', 'id'); + +-- Step 4: Verify that the table and partitions are created and distributed correctly on the coordinator +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + +-- Step 4 (Repeat on a Worker Node): Verify that the table and partitions are created correctly +\c - - - :worker_1_port +SET search_path TO pg17; + +-- Verify the table's access method on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + +-- Verify the partitions' access methods on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + +\c - - - :master_port +SET search_path TO pg17; + +-- Step 5: Test ALTER TABLE ... SET ACCESS METHOD to a different method +ALTER TABLE test_partitioned_alter SET ACCESS METHOD columnar; + +-- Verify the access method in the distributed parent and existing partitions +-- Note: Specifying an access method for a partitioned table lets the value be used for all +-- future partitions created under it, closely mirroring the behavior of the TABLESPACE +-- option for partitioned tables. Existing partitions are not modified. +-- Reference: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=374c7a2290429eac3217b0c7b0b485db9c2bcc72 + +-- Verify the parent table's access method +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partitioned_alter'; + +-- Verify the partitions' access methods +SELECT relname, relam +FROM pg_class +WHERE relname IN ('test_partition_1', 'test_partition_2') +ORDER BY relname; + +-- Step 6: Verify the change is applied to future partitions +CREATE TABLE test_partition_3 PARTITION OF test_partitioned_alter + FOR VALUES FROM (200) TO (300); + +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partition_3'; + +-- Step 6 (Repeat on a Worker Node): Verify that the new partition is created correctly +\c - - - :worker_1_port +SET search_path TO pg17; + +-- Verify the new partition's access method on the worker node +SELECT relname, relam +FROM pg_class +WHERE relname = 'test_partition_3'; + +\c - - - :master_port +SET search_path TO pg17; + +-- Clean up +DROP TABLE test_partitioned_alter CASCADE; + +-- End of Test: Access Method Behavior for Partitioned Tables + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 6d2a329da69fe055659b5aca89948d07a1b23df0 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:00:40 +0300 Subject: [PATCH 108/155] EXPLAIN generic_plan NOT supported in Citus (#7825) We thought we provided support for this in https://github.com/citusdata/citus/commit/b8c493f2c44efc1a19895fcadf5291b8285add7c However the use of parameters in SQL is not supported in Citus. Since generic plan queries use parameters, we can't support for now. Relevant PG16 commit https://github.com/postgres/postgres/commit/3c05284 Fixes #7813 with proper error message (cherry picked from commit 0a6adf4ccc908e373b7e7230ccc5b313ba63d9a4) --- .../distributed/planner/multi_explain.c | 14 +++++----- src/test/regress/expected/pg16.out | 26 +++---------------- src/test/regress/sql/pg16.sql | 4 +-- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 3f0120dae..c3de21208 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -190,6 +190,14 @@ PG_FUNCTION_INFO_V1(worker_save_query_explain_analyze); void CitusExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es) { +#if PG_VERSION_NUM >= PG_VERSION_16 + if (es->generic) + { + ereport(ERROR, (errmsg( + "EXPLAIN GENERIC_PLAN is currently not supported for Citus tables"))); + } +#endif + CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *distributedPlan = scanState->distributedPlan; EState *executorState = ScanStateGetExecutorState(scanState); @@ -1021,18 +1029,12 @@ BuildRemoteExplainQuery(char *queryString, ExplainState *es) appendStringInfo(explainQuery, "EXPLAIN (ANALYZE %s, VERBOSE %s, " "COSTS %s, BUFFERS %s, WAL %s, " -#if PG_VERSION_NUM >= PG_VERSION_16 - "GENERIC_PLAN %s, " -#endif "TIMING %s, SUMMARY %s, FORMAT %s) %s", es->analyze ? "TRUE" : "FALSE", es->verbose ? "TRUE" : "FALSE", es->costs ? "TRUE" : "FALSE", es->buffers ? "TRUE" : "FALSE", es->wal ? "TRUE" : "FALSE", -#if PG_VERSION_NUM >= PG_VERSION_16 - es->generic ? "TRUE" : "FALSE", -#endif es->timing ? "TRUE" : "FALSE", es->summary ? "TRUE" : "FALSE", formatStr, diff --git a/src/test/regress/expected/pg16.out b/src/test/regress/expected/pg16.out index 7689d21bd..d7c39a9fd 100644 --- a/src/test/regress/expected/pg16.out +++ b/src/test/regress/expected/pg16.out @@ -81,29 +81,9 @@ SELECT create_distributed_table('tenk1', 'unique1'); (1 row) SET citus.log_remote_commands TO on; -EXPLAIN (GENERIC_PLAN) SELECT unique1 FROM tenk1 WHERE thousand = 1000; -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SAVEPOINT citus_explain_savepoint -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing EXPLAIN (ANALYZE FALSE, VERBOSE FALSE, COSTS TRUE, BUFFERS FALSE, WAL FALSE, GENERIC_PLAN TRUE, TIMING FALSE, SUMMARY FALSE, FORMAT TEXT) SELECT unique1 FROM pg16.tenk1_950001 tenk1 WHERE (thousand OPERATOR(pg_catalog.=) 1000) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing ROLLBACK TO SAVEPOINT citus_explain_savepoint -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on tenk1_950001 tenk1 (cost=0.00..35.50 rows=10 width=4) - Filter: (thousand = 1000) -(7 rows) - -EXPLAIN (GENERIC_PLAN, ANALYZE) SELECT unique1 FROM tenk1 WHERE thousand = 1000; +EXPLAIN (GENERIC_PLAN) SELECT unique1 FROM tenk1 WHERE thousand = $1; +ERROR: EXPLAIN GENERIC_PLAN is currently not supported for Citus tables +EXPLAIN (GENERIC_PLAN, ANALYZE) SELECT unique1 FROM tenk1 WHERE thousand = $1; ERROR: EXPLAIN options ANALYZE and GENERIC_PLAN cannot be used together SET citus.log_remote_commands TO off; -- Proper error when creating statistics without a name on a Citus table diff --git a/src/test/regress/sql/pg16.sql b/src/test/regress/sql/pg16.sql index 736119914..ec4eb3fc7 100644 --- a/src/test/regress/sql/pg16.sql +++ b/src/test/regress/sql/pg16.sql @@ -58,8 +58,8 @@ CREATE TABLE tenk1 ( SELECT create_distributed_table('tenk1', 'unique1'); SET citus.log_remote_commands TO on; -EXPLAIN (GENERIC_PLAN) SELECT unique1 FROM tenk1 WHERE thousand = 1000; -EXPLAIN (GENERIC_PLAN, ANALYZE) SELECT unique1 FROM tenk1 WHERE thousand = 1000; +EXPLAIN (GENERIC_PLAN) SELECT unique1 FROM tenk1 WHERE thousand = $1; +EXPLAIN (GENERIC_PLAN, ANALYZE) SELECT unique1 FROM tenk1 WHERE thousand = $1; SET citus.log_remote_commands TO off; -- Proper error when creating statistics without a name on a Citus table From 48849ff3c2d202525f123cb664c5082c00f24978 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Thu, 2 Jan 2025 11:44:32 +0300 Subject: [PATCH 109/155] PG17 - Add Regression Test for REINDEX support in event triggers (#7819) This PR adds regression tests to verify REINDEX support with event triggers. Tests validates trigger execution, shard placement consistency, and distributed index rebuilding without disruption. --- src/test/regress/expected/pg17.out | 55 ++++++++++++++++++++++++++++++ src/test/regress/sql/pg17.sql | 52 ++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index f2c4183f2..1010c0d4b 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1724,6 +1724,61 @@ SET search_path TO pg17; -- Clean up DROP TABLE test_partitioned_alter CASCADE; -- End of Test: Access Method Behavior for Partitioned Tables +-- Test for REINDEX support in event triggers for Citus-related objects +-- Create a test table with a distributed setup +CREATE TABLE reindex_test (id SERIAL PRIMARY KEY, data TEXT); +SELECT create_distributed_table('reindex_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Create an index to test REINDEX functionality +CREATE INDEX reindex_test_data_idx ON reindex_test (data); +-- Create event triggers to capture REINDEX events (start and end) +CREATE OR REPLACE FUNCTION log_reindex_events() RETURNS event_trigger LANGUAGE plpgsql AS $$ +DECLARE + command_tag TEXT; + command_object JSONB; +BEGIN + command_tag := tg_tag; + command_object := jsonb_build_object( + 'object_type', tg_event, + 'command_tag', command_tag, + 'query', current_query() + ); + RAISE NOTICE 'Event Trigger Log: %', command_object::TEXT; +END; +$$; +CREATE EVENT TRIGGER reindex_event_trigger + ON ddl_command_start + WHEN TAG IN ('REINDEX') +EXECUTE FUNCTION log_reindex_events(); +CREATE EVENT TRIGGER reindex_event_trigger_end + ON ddl_command_end + WHEN TAG IN ('REINDEX') +EXECUTE FUNCTION log_reindex_events(); +-- Insert some data to create index bloat +INSERT INTO reindex_test (data) +SELECT 'value_' || g.i +FROM generate_series(1, 10000) g(i); +-- Perform REINDEX TABLE ... CONCURRENTLY and verify event trigger logs +REINDEX TABLE CONCURRENTLY reindex_test; +NOTICE: Event Trigger Log: {"query": "REINDEX TABLE CONCURRENTLY reindex_test;", "command_tag": "REINDEX", "object_type": "ddl_command_start"} +CONTEXT: PL/pgSQL function log_reindex_events() line XX at RAISE +NOTICE: Event Trigger Log: {"query": "REINDEX TABLE CONCURRENTLY reindex_test;", "command_tag": "REINDEX", "object_type": "ddl_command_end"} +CONTEXT: PL/pgSQL function log_reindex_events() line XX at RAISE +-- Perform REINDEX INDEX ... CONCURRENTLY and verify event trigger logs +REINDEX INDEX CONCURRENTLY reindex_test_data_idx; +NOTICE: Event Trigger Log: {"query": "REINDEX INDEX CONCURRENTLY reindex_test_data_idx;", "command_tag": "REINDEX", "object_type": "ddl_command_start"} +CONTEXT: PL/pgSQL function log_reindex_events() line XX at RAISE +NOTICE: Event Trigger Log: {"query": "REINDEX INDEX CONCURRENTLY reindex_test_data_idx;", "command_tag": "REINDEX", "object_type": "ddl_command_end"} +CONTEXT: PL/pgSQL function log_reindex_events() line XX at RAISE +-- Cleanup +DROP EVENT TRIGGER reindex_event_trigger; +DROP EVENT TRIGGER reindex_event_trigger_end; +DROP TABLE reindex_test CASCADE; +-- End of test for REINDEX support in event triggers for Citus-related objects \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index f6f7c5367..88d0eab0c 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -987,6 +987,58 @@ DROP TABLE test_partitioned_alter CASCADE; -- End of Test: Access Method Behavior for Partitioned Tables +-- Test for REINDEX support in event triggers for Citus-related objects +-- Create a test table with a distributed setup +CREATE TABLE reindex_test (id SERIAL PRIMARY KEY, data TEXT); +SELECT create_distributed_table('reindex_test', 'id'); + +-- Create an index to test REINDEX functionality +CREATE INDEX reindex_test_data_idx ON reindex_test (data); + +-- Create event triggers to capture REINDEX events (start and end) +CREATE OR REPLACE FUNCTION log_reindex_events() RETURNS event_trigger LANGUAGE plpgsql AS $$ +DECLARE + command_tag TEXT; + command_object JSONB; +BEGIN + command_tag := tg_tag; + command_object := jsonb_build_object( + 'object_type', tg_event, + 'command_tag', command_tag, + 'query', current_query() + ); + RAISE NOTICE 'Event Trigger Log: %', command_object::TEXT; +END; +$$; + +CREATE EVENT TRIGGER reindex_event_trigger + ON ddl_command_start + WHEN TAG IN ('REINDEX') +EXECUTE FUNCTION log_reindex_events(); + +CREATE EVENT TRIGGER reindex_event_trigger_end + ON ddl_command_end + WHEN TAG IN ('REINDEX') +EXECUTE FUNCTION log_reindex_events(); + +-- Insert some data to create index bloat +INSERT INTO reindex_test (data) +SELECT 'value_' || g.i +FROM generate_series(1, 10000) g(i); + +-- Perform REINDEX TABLE ... CONCURRENTLY and verify event trigger logs +REINDEX TABLE CONCURRENTLY reindex_test; + +-- Perform REINDEX INDEX ... CONCURRENTLY and verify event trigger logs +REINDEX INDEX CONCURRENTLY reindex_test_data_idx; + +-- Cleanup +DROP EVENT TRIGGER reindex_event_trigger; +DROP EVENT TRIGGER reindex_event_trigger_end; +DROP TABLE reindex_test CASCADE; + +-- End of test for REINDEX support in event triggers for Citus-related objects + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 9bad33f89d66fdeeb0ed62967a8aa55217a86374 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 2 Jan 2025 12:32:36 +0300 Subject: [PATCH 110/155] PG17 - Propagate EXPLAIN options: MEMORY and SERIALIZE (#7802) DESCRIPTION: Propagates MEMORY and SERIALIZE options of EXPLAIN The options for `MEMORY` can be true or false. Default is false. The options for `SERIALIZE` can be none, text or binary. Default is none. I referred to how we added support for WAL option in this PR [Support EXPLAIN(ANALYZE, WAL)](https://github.com/citusdata/citus/pull/4196). For the tests however, I used the same tests as Postgres, not like the tests in the WAL PR. I used exactly the same tests as Postgres does, I simply distributed the table beforehand. See below the relevant Postgres commits from where you can see the tests added as well: - [Add EXPLAIN (MEMORY)](https://github.com/postgres/postgres/commit/5de890e36) - [Invent SERIALIZE option for EXPLAIN.](https://github.com/postgres/postgres/commit/06286709e) This PR required a lot of copying of Postgres static functions regarding how `EXPLAIN` works for `MEMORY` and `SERIALIZE` options. Specifically, these copy-pastes were required for updating `ExplainWorkerPlan()` function, which is in fact based on postgres' `ExplainOnePlan()`: ```C /* copied from explain.c to update ExplainWorkerPlan() in citus according to ExplainOnePlan() in postgres */ #define BYTES_TO_KILOBYTES(b) typedef struct SerializeMetrics static bool peek_buffer_usage(ExplainState *es, const BufferUsage *usage); static void show_buffer_usage(ExplainState *es, const BufferUsage *usage); static void show_memory_counters(ExplainState *es, const MemoryContextCounters *mem_counters); static void ExplainIndentText(ExplainState *es); static void ExplainPrintSerialize(ExplainState *es, SerializeMetrics *metrics); static SerializeMetrics GetSerializationMetrics(DestReceiver *dest); ``` _Note_: it looks like we were missing some `buffers` option details as well. I put them together with the memory option, like the code in Postgres explain.c, as I didn't want to change the copied code. However, I tested locally and there is no big deal in previous Citus versions, and you can also see that existing Citus tests with `buffers true` didn't change. Therefore, I prefer not to backport "buffers" changes to previous versions. --- .../distributed/planner/multi_explain.c | 637 +++++++++++++++++- .../regress/expected/multi_test_helpers.out | 26 + src/test/regress/expected/pg17.out | 416 ++++++++++++ src/test/regress/sql/multi_test_helpers.sql | 28 + src/test/regress/sql/pg17.sql | 42 ++ 5 files changed, 1147 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index c3de21208..527fd073c 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -95,14 +95,24 @@ typedef struct bool wal; bool timing; bool summary; +#if PG_VERSION_NUM >= PG_VERSION_17 + bool memory; + ExplainSerializeOption serialize; +#endif ExplainFormat format; } ExplainOptions; /* EXPLAIN flags of current distributed explain */ +#if PG_VERSION_NUM >= PG_VERSION_17 +static ExplainOptions CurrentDistributedQueryExplainOptions = { + 0, 0, 0, 0, 0, 0, 0, EXPLAIN_SERIALIZE_NONE, EXPLAIN_FORMAT_TEXT +}; +#else static ExplainOptions CurrentDistributedQueryExplainOptions = { 0, 0, 0, 0, 0, 0, EXPLAIN_FORMAT_TEXT }; +#endif /* Result for a single remote EXPLAIN command */ typedef struct RemoteExplainPlan @@ -124,6 +134,59 @@ typedef struct ExplainAnalyzeDestination TupleDesc lastSavedExplainAnalyzeTupDesc; } ExplainAnalyzeDestination; +#if PG_VERSION_NUM >= PG_VERSION_17 + +/* + * Various places within need to convert bytes to kilobytes. Round these up + * to the next whole kilobyte. + * copied from explain.c + */ +#define BYTES_TO_KILOBYTES(b) (((b) + 1023) / 1024) + +/* copied from explain.c */ +/* Instrumentation data for SERIALIZE option */ +typedef struct SerializeMetrics +{ + uint64 bytesSent; /* # of bytes serialized */ + instr_time timeSpent; /* time spent serializing */ + BufferUsage bufferUsage; /* buffers accessed during serialization */ +} SerializeMetrics; + +/* copied from explain.c */ +static bool peek_buffer_usage(ExplainState *es, const BufferUsage *usage); +static void show_buffer_usage(ExplainState *es, const BufferUsage *usage); +static void show_memory_counters(ExplainState *es, + const MemoryContextCounters *mem_counters); +static void ExplainIndentText(ExplainState *es); +static void ExplainPrintSerialize(ExplainState *es, + SerializeMetrics *metrics); +static SerializeMetrics GetSerializationMetrics(DestReceiver *dest); + +/* + * DestReceiver functions for SERIALIZE option + * + * A DestReceiver for query tuples, that serializes passed rows into RowData + * messages while measuring the resources expended and total serialized size, + * while never sending the data to the client. This allows measuring the + * overhead of deTOASTing and datatype out/sendfuncs, which are not otherwise + * exercisable without actually hitting the network. + * + * copied from explain.c + */ +typedef struct SerializeDestReceiver +{ + DestReceiver pub; + ExplainState *es; /* this EXPLAIN statement's ExplainState */ + int8 format; /* text or binary, like pq wire protocol */ + TupleDesc attrinfo; /* the output tuple desc */ + int nattrs; /* current number of columns */ + FmgrInfo *finfos; /* precomputed call info for output fns */ + MemoryContext tmpcontext; /* per-row temporary memory context */ + StringInfoData buf; /* buffer to hold the constructed message */ + SerializeMetrics metrics; /* collected metrics */ +} SerializeDestReceiver; +#endif + /* Explain functions for distributed queries */ static void ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es); @@ -144,14 +207,27 @@ static void ExplainTaskPlacement(ShardPlacement *taskPlacement, List *explainOut ExplainState *es); static StringInfo BuildRemoteExplainQuery(char *queryString, ExplainState *es); static const char * ExplainFormatStr(ExplainFormat format); +#if PG_VERSION_NUM >= PG_VERSION_17 +static const char * ExplainSerializeStr(ExplainSerializeOption serializeOption); +#endif static void ExplainWorkerPlan(PlannedStmt *plannedStmt, DestReceiver *dest, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, +#if PG_VERSION_NUM >= PG_VERSION_17 + const BufferUsage *bufusage, + const MemoryContextCounters *mem_counters, +#endif double *executionDurationMillisec); static ExplainFormat ExtractFieldExplainFormat(Datum jsonbDoc, const char *fieldName, ExplainFormat defaultValue); +#if PG_VERSION_NUM >= PG_VERSION_17 +static ExplainSerializeOption ExtractFieldExplainSerialize(Datum jsonbDoc, + const char *fieldName, + ExplainSerializeOption + defaultValue); +#endif static TupleDestination * CreateExplainAnlyzeDestination(Task *task, TupleDestination *taskDest); static void ExplainAnalyzeDestPutTuple(TupleDestination *self, Task *task, @@ -1025,11 +1101,19 @@ BuildRemoteExplainQuery(char *queryString, ExplainState *es) { StringInfo explainQuery = makeStringInfo(); const char *formatStr = ExplainFormatStr(es->format); +#if PG_VERSION_NUM >= PG_VERSION_17 + const char *serializeStr = ExplainSerializeStr(es->serialize); +#endif + appendStringInfo(explainQuery, "EXPLAIN (ANALYZE %s, VERBOSE %s, " "COSTS %s, BUFFERS %s, WAL %s, " - "TIMING %s, SUMMARY %s, FORMAT %s) %s", + "TIMING %s, SUMMARY %s, " +#if PG_VERSION_NUM >= PG_VERSION_17 + "MEMORY %s, SERIALIZE %s, " +#endif + "FORMAT %s) %s", es->analyze ? "TRUE" : "FALSE", es->verbose ? "TRUE" : "FALSE", es->costs ? "TRUE" : "FALSE", @@ -1037,6 +1121,10 @@ BuildRemoteExplainQuery(char *queryString, ExplainState *es) es->wal ? "TRUE" : "FALSE", es->timing ? "TRUE" : "FALSE", es->summary ? "TRUE" : "FALSE", +#if PG_VERSION_NUM >= PG_VERSION_17 + es->memory ? "TRUE" : "FALSE", + serializeStr, +#endif formatStr, queryString); @@ -1075,6 +1163,42 @@ ExplainFormatStr(ExplainFormat format) } +#if PG_VERSION_NUM >= PG_VERSION_17 + +/* + * ExplainSerializeStr converts the given explain serialize option to string. + */ +static const char * +ExplainSerializeStr(ExplainSerializeOption serializeOption) +{ + switch (serializeOption) + { + case EXPLAIN_SERIALIZE_NONE: + { + return "none"; + } + + case EXPLAIN_SERIALIZE_TEXT: + { + return "text"; + } + + case EXPLAIN_SERIALIZE_BINARY: + { + return "binary"; + } + + default: + { + return "none"; + } + } +} + + +#endif + + /* * worker_last_saved_explain_analyze returns the last saved EXPLAIN ANALYZE output of * a worker task query. It returns NULL if nothing has been saved yet. @@ -1134,6 +1258,11 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) es->verbose = ExtractFieldBoolean(explainOptions, "verbose", es->verbose); es->timing = ExtractFieldBoolean(explainOptions, "timing", es->timing); es->format = ExtractFieldExplainFormat(explainOptions, "format", es->format); +#if PG_VERSION_NUM >= PG_VERSION_17 + es->memory = ExtractFieldBoolean(explainOptions, "memory", es->memory); + es->serialize = ExtractFieldExplainSerialize(explainOptions, "serialize", + es->serialize); +#endif TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); @@ -1179,6 +1308,36 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) /* plan query and record planning stats */ instr_time planStart; instr_time planDuration; +#if PG_VERSION_NUM >= PG_VERSION_17 + BufferUsage bufusage_start, + bufusage; + MemoryContextCounters mem_counters; + MemoryContext planner_ctx = NULL; + MemoryContext saved_ctx = NULL; + + if (es->memory) + { + /* + * Create a new memory context to measure planner's memory consumption + * accurately. Note that if the planner were to be modified to use a + * different memory context type, here we would be changing that to + * AllocSet, which might be undesirable. However, we don't have a way + * to create a context of the same type as another, so we pray and + * hope that this is OK. + * + * copied from explain.c + */ + planner_ctx = AllocSetContextCreate(CurrentMemoryContext, + "explain analyze planner context", + ALLOCSET_DEFAULT_SIZES); + saved_ctx = MemoryContextSwitchTo(planner_ctx); + } + + if (es->buffers) + { + bufusage_start = pgBufferUsage; + } +#endif INSTR_TIME_SET_CURRENT(planStart); @@ -1187,9 +1346,32 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) INSTR_TIME_SET_CURRENT(planDuration); INSTR_TIME_SUBTRACT(planDuration, planStart); +#if PG_VERSION_NUM >= PG_VERSION_17 + if (es->memory) + { + MemoryContextSwitchTo(saved_ctx); + MemoryContextMemConsumed(planner_ctx, &mem_counters); + } + + /* calc differences of buffer counters. */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + + /* do the actual EXPLAIN ANALYZE */ + ExplainWorkerPlan(plan, tupleStoreDest, es, queryString, boundParams, NULL, + &planDuration, + (es->buffers ? &bufusage : NULL), + (es->memory ? &mem_counters : NULL), + &executionDurationMillisec); +#else + /* do the actual EXPLAIN ANALYZE */ ExplainWorkerPlan(plan, tupleStoreDest, es, queryString, boundParams, NULL, &planDuration, &executionDurationMillisec); +#endif ExplainEndOutput(es); @@ -1258,6 +1440,50 @@ ExtractFieldExplainFormat(Datum jsonbDoc, const char *fieldName, ExplainFormat } +#if PG_VERSION_NUM >= PG_VERSION_17 + +/* + * ExtractFieldExplainSerialize gets value of fieldName from jsonbDoc, or returns + * defaultValue if it doesn't exist. + */ +static ExplainSerializeOption +ExtractFieldExplainSerialize(Datum jsonbDoc, const char *fieldName, ExplainSerializeOption + defaultValue) +{ + Datum jsonbDatum = 0; + bool found = ExtractFieldJsonbDatum(jsonbDoc, fieldName, &jsonbDatum); + if (!found) + { + return defaultValue; + } + + const char *serializeStr = DatumGetCString(DirectFunctionCall1(jsonb_out, + jsonbDatum)); + if (pg_strcasecmp(serializeStr, "\"none\"") == 0) + { + return EXPLAIN_SERIALIZE_NONE; + } + else if (pg_strcasecmp(serializeStr, "\"off\"") == 0) + { + return EXPLAIN_SERIALIZE_NONE; + } + else if (pg_strcasecmp(serializeStr, "\"text\"") == 0) + { + return EXPLAIN_SERIALIZE_TEXT; + } + else if (pg_strcasecmp(serializeStr, "\"binary\"") == 0) + { + return EXPLAIN_SERIALIZE_BINARY; + } + + ereport(ERROR, (errmsg("Invalid explain analyze serialize: %s", serializeStr))); + return 0; +} + + +#endif + + /* * CitusExplainOneQuery is the executor hook that is called when * postgres wants to explain a query. @@ -1275,6 +1501,10 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, CurrentDistributedQueryExplainOptions.summary = es->summary; CurrentDistributedQueryExplainOptions.timing = es->timing; CurrentDistributedQueryExplainOptions.format = es->format; +#if PG_VERSION_NUM >= PG_VERSION_17 + CurrentDistributedQueryExplainOptions.memory = es->memory; + CurrentDistributedQueryExplainOptions.serialize = es->serialize; +#endif /* rest is copied from ExplainOneQuery() */ instr_time planstart, @@ -1597,11 +1827,18 @@ WrapQueryForExplainAnalyze(const char *queryString, TupleDesc tupleDesc, StringInfo explainOptions = makeStringInfo(); appendStringInfo(explainOptions, "{\"verbose\": %s, \"costs\": %s, \"buffers\": %s, \"wal\": %s, " +#if PG_VERSION_NUM >= PG_VERSION_17 + "\"memory\": %s, \"serialize\": \"%s\", " +#endif "\"timing\": %s, \"summary\": %s, \"format\": \"%s\"}", CurrentDistributedQueryExplainOptions.verbose ? "true" : "false", CurrentDistributedQueryExplainOptions.costs ? "true" : "false", CurrentDistributedQueryExplainOptions.buffers ? "true" : "false", CurrentDistributedQueryExplainOptions.wal ? "true" : "false", +#if PG_VERSION_NUM >= PG_VERSION_17 + CurrentDistributedQueryExplainOptions.memory ? "true" : "false", + ExplainSerializeStr(CurrentDistributedQueryExplainOptions.serialize), +#endif CurrentDistributedQueryExplainOptions.timing ? "true" : "false", CurrentDistributedQueryExplainOptions.summary ? "true" : "false", ExplainFormatStr(CurrentDistributedQueryExplainOptions.format)); @@ -1826,7 +2063,12 @@ ExplainOneQuery(Query *query, int cursorOptions, static void ExplainWorkerPlan(PlannedStmt *plannedstmt, DestReceiver *dest, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, - const instr_time *planduration, double *executionDurationMillisec) + const instr_time *planduration, +#if PG_VERSION_NUM >= PG_VERSION_17 + const BufferUsage *bufusage, + const MemoryContextCounters *mem_counters, +#endif + double *executionDurationMillisec) { QueryDesc *queryDesc; instr_time starttime; @@ -1895,6 +2137,32 @@ ExplainWorkerPlan(PlannedStmt *plannedstmt, DestReceiver *dest, ExplainState *es /* Create textual dump of plan tree */ ExplainPrintPlan(es, queryDesc); +#if PG_VERSION_NUM >= PG_VERSION_17 + /* Show buffer and/or memory usage in planning */ + if (peek_buffer_usage(es, bufusage) || mem_counters) + { + ExplainOpenGroup("Planning", "Planning", true, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + appendStringInfoString(es->str, "Planning:\n"); + es->indent++; + } + + if (bufusage) + show_buffer_usage(es, bufusage); + + if (mem_counters) + show_memory_counters(es, mem_counters); + + if (es->format == EXPLAIN_FORMAT_TEXT) + es->indent--; + + ExplainCloseGroup("Planning", "Planning", true, es); + } +#endif + if (es->summary && planduration) { double plantime = INSTR_TIME_GET_DOUBLE(*planduration); @@ -1915,6 +2183,23 @@ ExplainWorkerPlan(PlannedStmt *plannedstmt, DestReceiver *dest, ExplainState *es if (es->costs) ExplainPrintJITSummary(es, queryDesc); +#if PG_VERSION_NUM >= PG_VERSION_17 + if (es->serialize != EXPLAIN_SERIALIZE_NONE) + { + /* the SERIALIZE option requires its own tuple receiver */ + DestReceiver *dest_serialize = CreateExplainSerializeDestReceiver(es); + + /* grab serialization metrics before we destroy the DestReceiver */ + SerializeMetrics serializeMetrics = GetSerializationMetrics(dest_serialize); + + /* call the DestReceiver's destroy method even during explain */ + dest_serialize->rDestroy(dest_serialize); + + /* Print info about serialization of output */ + ExplainPrintSerialize(es, &serializeMetrics); + } +#endif + /* * Close down the query and free resources. Include time for this in the * total execution time (although it should be pretty minimal). @@ -1963,3 +2248,351 @@ elapsed_time(instr_time *starttime) INSTR_TIME_SUBTRACT(endtime, *starttime); return INSTR_TIME_GET_DOUBLE(endtime); } + + +#if PG_VERSION_NUM >= PG_VERSION_17 +/* + * Return whether show_buffer_usage would have anything to print, if given + * the same 'usage' data. Note that when the format is anything other than + * text, we print even if the counters are all zeroes. + * + * Copied from explain.c. + */ +static bool +peek_buffer_usage(ExplainState *es, const BufferUsage *usage) +{ + bool has_shared; + bool has_local; + bool has_temp; + bool has_shared_timing; + bool has_local_timing; + bool has_temp_timing; + + if (usage == NULL) + return false; + + if (es->format != EXPLAIN_FORMAT_TEXT) + return true; + + has_shared = (usage->shared_blks_hit > 0 || + usage->shared_blks_read > 0 || + usage->shared_blks_dirtied > 0 || + usage->shared_blks_written > 0); + has_local = (usage->local_blks_hit > 0 || + usage->local_blks_read > 0 || + usage->local_blks_dirtied > 0 || + usage->local_blks_written > 0); + has_temp = (usage->temp_blks_read > 0 || + usage->temp_blks_written > 0); + has_shared_timing = (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)); + has_local_timing = (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->local_blk_write_time)); + has_temp_timing = (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)); + + return has_shared || has_local || has_temp || has_shared_timing || + has_local_timing || has_temp_timing; +} + + +/* + * Show buffer usage details. This better be sync with peek_buffer_usage. + * + * Copied from explain.c. + */ +static void +show_buffer_usage(ExplainState *es, const BufferUsage *usage) +{ + if (es->format == EXPLAIN_FORMAT_TEXT) + { + bool has_shared = (usage->shared_blks_hit > 0 || + usage->shared_blks_read > 0 || + usage->shared_blks_dirtied > 0 || + usage->shared_blks_written > 0); + bool has_local = (usage->local_blks_hit > 0 || + usage->local_blks_read > 0 || + usage->local_blks_dirtied > 0 || + usage->local_blks_written > 0); + bool has_temp = (usage->temp_blks_read > 0 || + usage->temp_blks_written > 0); + bool has_shared_timing = (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)); + bool has_local_timing = (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->local_blk_write_time)); + bool has_temp_timing = (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time) || + !INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)); + + /* Show only positive counter values. */ + if (has_shared || has_local || has_temp) + { + ExplainIndentText(es); + appendStringInfoString(es->str, "Buffers:"); + + if (has_shared) + { + appendStringInfoString(es->str, " shared"); + if (usage->shared_blks_hit > 0) + appendStringInfo(es->str, " hit=%lld", + (long long) usage->shared_blks_hit); + if (usage->shared_blks_read > 0) + appendStringInfo(es->str, " read=%lld", + (long long) usage->shared_blks_read); + if (usage->shared_blks_dirtied > 0) + appendStringInfo(es->str, " dirtied=%lld", + (long long) usage->shared_blks_dirtied); + if (usage->shared_blks_written > 0) + appendStringInfo(es->str, " written=%lld", + (long long) usage->shared_blks_written); + if (has_local || has_temp) + appendStringInfoChar(es->str, ','); + } + if (has_local) + { + appendStringInfoString(es->str, " local"); + if (usage->local_blks_hit > 0) + appendStringInfo(es->str, " hit=%lld", + (long long) usage->local_blks_hit); + if (usage->local_blks_read > 0) + appendStringInfo(es->str, " read=%lld", + (long long) usage->local_blks_read); + if (usage->local_blks_dirtied > 0) + appendStringInfo(es->str, " dirtied=%lld", + (long long) usage->local_blks_dirtied); + if (usage->local_blks_written > 0) + appendStringInfo(es->str, " written=%lld", + (long long) usage->local_blks_written); + if (has_temp) + appendStringInfoChar(es->str, ','); + } + if (has_temp) + { + appendStringInfoString(es->str, " temp"); + if (usage->temp_blks_read > 0) + appendStringInfo(es->str, " read=%lld", + (long long) usage->temp_blks_read); + if (usage->temp_blks_written > 0) + appendStringInfo(es->str, " written=%lld", + (long long) usage->temp_blks_written); + } + appendStringInfoChar(es->str, '\n'); + } + + /* As above, show only positive counter values. */ + if (has_shared_timing || has_local_timing || has_temp_timing) + { + ExplainIndentText(es); + appendStringInfoString(es->str, "I/O Timings:"); + + if (has_shared_timing) + { + appendStringInfoString(es->str, " shared"); + if (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time)) + appendStringInfo(es->str, " read=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->shared_blk_read_time)); + if (!INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)) + appendStringInfo(es->str, " write=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->shared_blk_write_time)); + if (has_local_timing || has_temp_timing) + appendStringInfoChar(es->str, ','); + } + if (has_local_timing) + { + appendStringInfoString(es->str, " local"); + if (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time)) + appendStringInfo(es->str, " read=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->local_blk_read_time)); + if (!INSTR_TIME_IS_ZERO(usage->local_blk_write_time)) + appendStringInfo(es->str, " write=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->local_blk_write_time)); + if (has_temp_timing) + appendStringInfoChar(es->str, ','); + } + if (has_temp_timing) + { + appendStringInfoString(es->str, " temp"); + if (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time)) + appendStringInfo(es->str, " read=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->temp_blk_read_time)); + if (!INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)) + appendStringInfo(es->str, " write=%0.3f", + INSTR_TIME_GET_MILLISEC(usage->temp_blk_write_time)); + } + appendStringInfoChar(es->str, '\n'); + } + } + else + { + ExplainPropertyInteger("Shared Hit Blocks", NULL, + usage->shared_blks_hit, es); + ExplainPropertyInteger("Shared Read Blocks", NULL, + usage->shared_blks_read, es); + ExplainPropertyInteger("Shared Dirtied Blocks", NULL, + usage->shared_blks_dirtied, es); + ExplainPropertyInteger("Shared Written Blocks", NULL, + usage->shared_blks_written, es); + ExplainPropertyInteger("Local Hit Blocks", NULL, + usage->local_blks_hit, es); + ExplainPropertyInteger("Local Read Blocks", NULL, + usage->local_blks_read, es); + ExplainPropertyInteger("Local Dirtied Blocks", NULL, + usage->local_blks_dirtied, es); + ExplainPropertyInteger("Local Written Blocks", NULL, + usage->local_blks_written, es); + ExplainPropertyInteger("Temp Read Blocks", NULL, + usage->temp_blks_read, es); + ExplainPropertyInteger("Temp Written Blocks", NULL, + usage->temp_blks_written, es); + if (track_io_timing) + { + ExplainPropertyFloat("Shared I/O Read Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->shared_blk_read_time), + 3, es); + ExplainPropertyFloat("Shared I/O Write Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->shared_blk_write_time), + 3, es); + ExplainPropertyFloat("Local I/O Read Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->local_blk_read_time), + 3, es); + ExplainPropertyFloat("Local I/O Write Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->local_blk_write_time), + 3, es); + ExplainPropertyFloat("Temp I/O Read Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->temp_blk_read_time), + 3, es); + ExplainPropertyFloat("Temp I/O Write Time", "ms", + INSTR_TIME_GET_MILLISEC(usage->temp_blk_write_time), + 3, es); + } + } +} + + +/* + * Indent a text-format line. + * + * We indent by two spaces per indentation level. However, when emitting + * data for a parallel worker there might already be data on the current line + * (cf. ExplainOpenWorker); in that case, don't indent any more. + * + * Copied from explain.c. + */ +static void +ExplainIndentText(ExplainState *es) +{ + Assert(es->format == EXPLAIN_FORMAT_TEXT); + if (es->str->len == 0 || es->str->data[es->str->len - 1] == '\n') + appendStringInfoSpaces(es->str, es->indent * 2); +} + + +/* + * Show memory usage details. + * + * Copied from explain.c. + */ +static void +show_memory_counters(ExplainState *es, const MemoryContextCounters *mem_counters) +{ + int64 memUsedkB = BYTES_TO_KILOBYTES(mem_counters->totalspace - + mem_counters->freespace); + int64 memAllocatedkB = BYTES_TO_KILOBYTES(mem_counters->totalspace); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + appendStringInfo(es->str, + "Memory: used=" INT64_FORMAT "kB allocated=" INT64_FORMAT "kB", + memUsedkB, memAllocatedkB); + appendStringInfoChar(es->str, '\n'); + } + else + { + ExplainPropertyInteger("Memory Used", "kB", memUsedkB, es); + ExplainPropertyInteger("Memory Allocated", "kB", memAllocatedkB, es); + } +} + + +/* + * ExplainPrintSerialize - + * Append information about query output volume to es->str. + * + * Copied from explain.c. + */ +static void +ExplainPrintSerialize(ExplainState *es, SerializeMetrics *metrics) +{ + const char *format; + + /* We shouldn't get called for EXPLAIN_SERIALIZE_NONE */ + if (es->serialize == EXPLAIN_SERIALIZE_TEXT) + format = "text"; + else + { + Assert(es->serialize == EXPLAIN_SERIALIZE_BINARY); + format = "binary"; + } + + ExplainOpenGroup("Serialization", "Serialization", true, es); + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + ExplainIndentText(es); + if (es->timing) + appendStringInfo(es->str, "Serialization: time=%.3f ms output=" UINT64_FORMAT "kB format=%s\n", + 1000.0 * INSTR_TIME_GET_DOUBLE(metrics->timeSpent), + BYTES_TO_KILOBYTES(metrics->bytesSent), + format); + else + appendStringInfo(es->str, "Serialization: output=" UINT64_FORMAT "kB format=%s\n", + BYTES_TO_KILOBYTES(metrics->bytesSent), + format); + + if (es->buffers && peek_buffer_usage(es, &metrics->bufferUsage)) + { + es->indent++; + show_buffer_usage(es, &metrics->bufferUsage); + es->indent--; + } + } + else + { + if (es->timing) + ExplainPropertyFloat("Time", "ms", + 1000.0 * INSTR_TIME_GET_DOUBLE(metrics->timeSpent), + 3, es); + ExplainPropertyUInteger("Output Volume", "kB", + BYTES_TO_KILOBYTES(metrics->bytesSent), es); + ExplainPropertyText("Format", format, es); + if (es->buffers) + show_buffer_usage(es, &metrics->bufferUsage); + } + + ExplainCloseGroup("Serialization", "Serialization", true, es); +} + + +/* + * GetSerializationMetrics - collect metrics + * + * We have to be careful here since the receiver could be an IntoRel + * receiver if the subject statement is CREATE TABLE AS. In that + * case, return all-zeroes stats. + * + * Copied from explain.c. + */ +static SerializeMetrics +GetSerializationMetrics(DestReceiver *dest) +{ + SerializeMetrics empty; + + if (dest->mydest == DestExplainSerialize) + return ((SerializeDestReceiver *) dest)->metrics; + + memset(&empty, 0, sizeof(SerializeMetrics)); + INSTR_TIME_SET_ZERO(empty.timeSpent); + + return empty; +} +#endif diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 8797f56b5..f3a0f2bba 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -605,3 +605,29 @@ BEGIN RETURN NEXT; END LOOP; END; $$ language plpgsql; +-- To produce stable regression test output, it's usually necessary to +-- ignore details such as exact costs or row counts. These filter +-- functions replace changeable output details with fixed strings. +-- Copied from PG explain.sql +create function explain_filter(text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in execute $1 + loop + -- Replace any numeric word with just 'N' + ln := regexp_replace(ln, '-?\m\d+\M', 'N', 'g'); + -- In sort output, the above won't match units-suffixed numbers + ln := regexp_replace(ln, '\m\d+kB', 'NkB', 'g'); + -- Ignore text-mode buffers output because it varies depending + -- on the system state + CONTINUE WHEN (ln ~ ' +Buffers: .*'); + -- Ignore text-mode "Planning:" line because whether it's output + -- varies depending on the system state + CONTINUE WHEN (ln = 'Planning:'); + return next ln; + end loop; +end; +$$; diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 1010c0d4b..dfd88e30e 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -1779,6 +1779,422 @@ DROP EVENT TRIGGER reindex_event_trigger; DROP EVENT TRIGGER reindex_event_trigger_end; DROP TABLE reindex_test CASCADE; -- End of test for REINDEX support in event triggers for Citus-related objects +-- Propagate EXPLAIN MEMORY +-- Relevant PG commit: https://github.com/postgres/postgres/commit/5de890e36 +-- Propagate EXPLAIN SERIALIZE +-- Relevant PG commit: https://github.com/postgres/postgres/commit/06286709e +SET citus.next_shard_id TO 12242024; +CREATE TABLE int8_tbl(q1 int8, q2 int8); +SELECT create_distributed_table('int8_tbl', 'q1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO int8_tbl VALUES + (' 123 ',' 456'), + ('123 ','4567890123456789'), + ('4567890123456789','123'), + (+4567890123456789,'4567890123456789'), + ('+4567890123456789','-4567890123456789'); +-- memory tests, same as postgres tests, we just distributed the table +-- we can see the memory used separately per each task in worker nodes +SET citus.log_remote_commands TO true; +-- for explain analyze, we run worker_save_query_explain_analyze query +-- for regular explain, we run EXPLAIN query +-- therefore let's grep the commands based on the shard id +SET citus.grep_remote_commands TO '%12242024%'; +select public.explain_filter('explain (memory) select * from int8_tbl i8'); +NOTICE: issuing EXPLAIN (ANALYZE FALSE, VERBOSE FALSE, COSTS TRUE, BUFFERS FALSE, WAL FALSE, TIMING FALSE, SUMMARY FALSE, MEMORY TRUE, SERIALIZE none, FORMAT TEXT) SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) + Task Count: N + Tasks Shown: One of N + -> Task + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) + Planning: + Memory: used=NkB allocated=NkB + Memory: used=NkB allocated=NkB +(9 rows) + +select public.explain_filter('explain (memory, analyze) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": false, "wal": false, "memory": true, "serialize": "none", "timing": true, "summary": true, "format": "TEXT"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Task Count: N + Tuple data received from nodes: N bytes + Tasks Shown: One of N + -> Task + Tuple data received from node: N bytes + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Planning: + Memory: used=NkB allocated=NkB + Planning Time: N.N ms + Execution Time: N.N ms + Memory: used=NkB allocated=NkB + Planning Time: N.N ms + Execution Time: N.N ms +(15 rows) + +select public.explain_filter('explain (memory, summary, format yaml) select * from int8_tbl i8'); +NOTICE: issuing EXPLAIN (ANALYZE FALSE, VERBOSE FALSE, COSTS TRUE, BUFFERS FALSE, WAL FALSE, TIMING FALSE, SUMMARY TRUE, MEMORY TRUE, SERIALIZE none, FORMAT YAML) SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + - Plan: + + Node Type: "Custom Scan" + + Custom Plan Provider: "Citus Adaptive" + + Parallel Aware: false + + Async Capable: false + + Startup Cost: N.N + + Total Cost: N.N + + Plan Rows: N + + Plan Width: N + + Distributed Query: + + Job: + + Task Count: N + + Tasks Shown: "One of N" + + Tasks: + + - Node: "host=localhost port=N dbname=regression"+ + Remote Plan: + + - Plan: + + Node Type: "Seq Scan" + + Parallel Aware: false + + Async Capable: false + + Relation Name: "int8_tbl_12242024" + + Alias: "i8" + + Startup Cost: N.N + + Total Cost: N.N + + Plan Rows: N + + Plan Width: N + + Planning: + + Memory Used: N + + Memory Allocated: N + + Planning Time: N.N + + + + Planning: + + Memory Used: N + + Memory Allocated: N + + Planning Time: N.N +(1 row) + +select public.explain_filter('explain (memory, analyze, format json) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": false, "wal": false, "memory": true, "serialize": "none", "timing": true, "summary": true, "format": "JSON"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + [ + + { + + "Plan": { + + "Node Type": "Custom Scan", + + "Custom Plan Provider": "Citus Adaptive", + + "Parallel Aware": false, + + "Async Capable": false, + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Actual Startup Time": N.N, + + "Actual Total Time": N.N, + + "Actual Rows": N, + + "Actual Loops": N, + + "Distributed Query": { + + "Job": { + + "Task Count": N, + + "Tuple data received from nodes": "N bytes", + + "Tasks Shown": "One of N", + + "Tasks": [ + + { + + "Tuple data received from node": "N bytes", + + "Node": "host=localhost port=N dbname=regression",+ + "Remote Plan": [ + + [ + + { + + "Plan": { + + "Node Type": "Seq Scan", + + "Parallel Aware": false, + + "Async Capable": false, + + "Relation Name": "int8_tbl_12242024", + + "Alias": "i8", + + "Startup Cost": N.N, + + "Total Cost": N.N, + + "Plan Rows": N, + + "Plan Width": N, + + "Actual Startup Time": N.N, + + "Actual Total Time": N.N, + + "Actual Rows": N, + + "Actual Loops": N + + }, + + "Planning": { + + "Memory Used": N, + + "Memory Allocated": N + + }, + + "Planning Time": N.N, + + "Triggers": [ + + ], + + "Execution Time": N.N + + } + + ] + + + + ] + + } + + ] + + } + + } + + }, + + "Planning": { + + "Memory Used": N, + + "Memory Allocated": N + + }, + + "Planning Time": N.N, + + "Triggers": [ + + ], + + "Execution Time": N.N + + } + + ] +(1 row) + +prepare int8_query as select * from int8_tbl i8; +select public.explain_filter('explain (memory) execute int8_query'); +NOTICE: issuing EXPLAIN (ANALYZE FALSE, VERBOSE FALSE, COSTS TRUE, BUFFERS FALSE, WAL FALSE, TIMING FALSE, SUMMARY FALSE, MEMORY TRUE, SERIALIZE none, FORMAT TEXT) SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) + Task Count: N + Tasks Shown: One of N + -> Task + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) + Planning: + Memory: used=NkB allocated=NkB + Memory: used=NkB allocated=NkB +(9 rows) + +-- serialize tests, same as postgres tests, we just distributed the table +select public.explain_filter('explain (analyze, serialize, buffers, format yaml) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": true, "wal": false, "memory": false, "serialize": "text", "timing": true, "summary": true, "format": "YAML"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + - Plan: + + Node Type: "Custom Scan" + + Custom Plan Provider: "Citus Adaptive" + + Parallel Aware: false + + Async Capable: false + + Startup Cost: N.N + + Total Cost: N.N + + Plan Rows: N + + Plan Width: N + + Actual Startup Time: N.N + + Actual Total Time: N.N + + Actual Rows: N + + Actual Loops: N + + Distributed Query: + + Job: + + Task Count: N + + Tuple data received from nodes: "N bytes" + + Tasks Shown: "One of N" + + Tasks: + + - Tuple data received from node: "N bytes" + + Node: "host=localhost port=N dbname=regression"+ + Remote Plan: + + - Plan: + + Node Type: "Seq Scan" + + Parallel Aware: false + + Async Capable: false + + Relation Name: "int8_tbl_12242024" + + Alias: "i8" + + Startup Cost: N.N + + Total Cost: N.N + + Plan Rows: N + + Plan Width: N + + Actual Startup Time: N.N + + Actual Total Time: N.N + + Actual Rows: N + + Actual Loops: N + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Planning: + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Planning Time: N.N + + Triggers: + + Serialization: + + Time: N.N + + Output Volume: N + + Format: "text" + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Execution Time: N.N + + + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Planning: + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Planning Time: N.N + + Triggers: + + Serialization: + + Time: N.N + + Output Volume: N + + Format: "text" + + Shared Hit Blocks: N + + Shared Read Blocks: N + + Shared Dirtied Blocks: N + + Shared Written Blocks: N + + Local Hit Blocks: N + + Local Read Blocks: N + + Local Dirtied Blocks: N + + Local Written Blocks: N + + Temp Read Blocks: N + + Temp Written Blocks: N + + Execution Time: N.N +(1 row) + +select public.explain_filter('explain (analyze,serialize) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": false, "wal": false, "memory": false, "serialize": "text", "timing": true, "summary": true, "format": "TEXT"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Task Count: N + Tuple data received from nodes: N bytes + Tasks Shown: One of N + -> Task + Tuple data received from node: N bytes + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=text + Execution Time: N.N ms + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=text + Execution Time: N.N ms +(14 rows) + +select public.explain_filter('explain (analyze,serialize text,buffers,timing off) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": true, "wal": false, "memory": false, "serialize": "text", "timing": false, "summary": true, "format": "TEXT"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) (actual rows=N loops=N) + Task Count: N + Tuple data received from nodes: N bytes + Tasks Shown: One of N + -> Task + Tuple data received from node: N bytes + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) (actual rows=N loops=N) + Planning Time: N.N ms + Serialization: output=NkB format=text + Execution Time: N.N ms + Planning Time: N.N ms + Serialization: output=NkB format=text + Execution Time: N.N ms +(14 rows) + +select public.explain_filter('explain (analyze,serialize binary,buffers,timing) select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": true, "wal": false, "memory": false, "serialize": "binary", "timing": true, "summary": true, "format": "TEXT"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Task Count: N + Tuple data received from nodes: N bytes + Tasks Shown: One of N + -> Task + Tuple data received from node: N bytes + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=binary + Execution Time: N.N ms + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=binary + Execution Time: N.N ms +(14 rows) + +-- this tests an edge case where we have no data to return +select public.explain_filter('explain (analyze,serialize) create temp table explain_temp as select * from int8_tbl i8'); +NOTICE: issuing SELECT * FROM worker_save_query_explain_analyze('SELECT q1, q2 FROM pg17.int8_tbl_12242024 i8 WHERE true', '{"verbose": false, "costs": true, "buffers": false, "wal": false, "memory": false, "serialize": "text", "timing": true, "summary": true, "format": "TEXT"}') AS (field_0 bigint, field_1 bigint) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXECUTE statement + explain_filter +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Task Count: N + Tuple data received from nodes: N bytes + Tasks Shown: One of N + -> Task + Tuple data received from node: N bytes + Node: host=localhost port=N dbname=regression + -> Seq Scan on int8_tbl_12242024 i8 (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N) + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=text + Execution Time: N.N ms + Planning Time: N.N ms + Serialization: time=N.N ms output=NkB format=text + Execution Time: N.N ms +(14 rows) + +RESET citus.log_remote_commands; +-- End of EXPLAIN MEMORY SERIALIZE tests \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 1cdcf4b39..57abf15a3 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -632,3 +632,31 @@ BEGIN RETURN NEXT; END LOOP; END; $$ language plpgsql; + +-- To produce stable regression test output, it's usually necessary to +-- ignore details such as exact costs or row counts. These filter +-- functions replace changeable output details with fixed strings. +-- Copied from PG explain.sql + +create function explain_filter(text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in execute $1 + loop + -- Replace any numeric word with just 'N' + ln := regexp_replace(ln, '-?\m\d+\M', 'N', 'g'); + -- In sort output, the above won't match units-suffixed numbers + ln := regexp_replace(ln, '\m\d+kB', 'NkB', 'g'); + -- Ignore text-mode buffers output because it varies depending + -- on the system state + CONTINUE WHEN (ln ~ ' +Buffers: .*'); + -- Ignore text-mode "Planning:" line because whether it's output + -- varies depending on the system state + CONTINUE WHEN (ln = 'Planning:'); + return next ln; + end loop; +end; +$$; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 88d0eab0c..70d5f68a8 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -1038,6 +1038,48 @@ DROP EVENT TRIGGER reindex_event_trigger_end; DROP TABLE reindex_test CASCADE; -- End of test for REINDEX support in event triggers for Citus-related objects +-- Propagate EXPLAIN MEMORY +-- Relevant PG commit: https://github.com/postgres/postgres/commit/5de890e36 +-- Propagate EXPLAIN SERIALIZE +-- Relevant PG commit: https://github.com/postgres/postgres/commit/06286709e + +SET citus.next_shard_id TO 12242024; +CREATE TABLE int8_tbl(q1 int8, q2 int8); +SELECT create_distributed_table('int8_tbl', 'q1'); +INSERT INTO int8_tbl VALUES + (' 123 ',' 456'), + ('123 ','4567890123456789'), + ('4567890123456789','123'), + (+4567890123456789,'4567890123456789'), + ('+4567890123456789','-4567890123456789'); + +-- memory tests, same as postgres tests, we just distributed the table +-- we can see the memory used separately per each task in worker nodes + +SET citus.log_remote_commands TO true; + +-- for explain analyze, we run worker_save_query_explain_analyze query +-- for regular explain, we run EXPLAIN query +-- therefore let's grep the commands based on the shard id +SET citus.grep_remote_commands TO '%12242024%'; + +select public.explain_filter('explain (memory) select * from int8_tbl i8'); +select public.explain_filter('explain (memory, analyze) select * from int8_tbl i8'); +select public.explain_filter('explain (memory, summary, format yaml) select * from int8_tbl i8'); +select public.explain_filter('explain (memory, analyze, format json) select * from int8_tbl i8'); +prepare int8_query as select * from int8_tbl i8; +select public.explain_filter('explain (memory) execute int8_query'); + +-- serialize tests, same as postgres tests, we just distributed the table +select public.explain_filter('explain (analyze, serialize, buffers, format yaml) select * from int8_tbl i8'); +select public.explain_filter('explain (analyze,serialize) select * from int8_tbl i8'); +select public.explain_filter('explain (analyze,serialize text,buffers,timing off) select * from int8_tbl i8'); +select public.explain_filter('explain (analyze,serialize binary,buffers,timing) select * from int8_tbl i8'); +-- this tests an edge case where we have no data to return +select public.explain_filter('explain (analyze,serialize) create temp table explain_temp as select * from int8_tbl i8'); + +RESET citus.log_remote_commands; +-- End of EXPLAIN MEMORY SERIALIZE tests \set VERBOSITY terse SET client_min_messages TO WARNING; From 32c058e4481b9db644116a8c172016c2df15e68c Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 8 Jan 2025 23:37:47 +0300 Subject: [PATCH 111/155] Remove Debian Buster support from packaging pipelines (#7828) (#7837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove Debian Buster support from packaging-test-pipelines Co-authored-by: Gürkan İndibay (cherry picked from commit 70f84e4aeeb326b690c57380eafca7e904ebcc1d) Co-authored-by: Seda Gündoğdu <69769369+sedagundogdu@users.noreply.github.com> --- .github/workflows/packaging-test-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 6c3672fed..34af216c0 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -115,7 +115,6 @@ jobs: # for each deb based image and we use POSTGRES_VERSION to set # PG_CONFIG variable in each of those runs. packaging_docker_image: - - debian-buster-all - debian-bookworm-all - debian-bullseye-all - ubuntu-focal-all From 3e924db90a255f0e2680775dd4c7e8b05de58fc2 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Thu, 9 Jan 2025 00:03:06 +0300 Subject: [PATCH 112/155] Propagate MERGE ... WHEN NOT MATCHED BY SOURCE (#7807) DESCRIPTION: Propagates MERGE ... WHEN NOT MATCHED BY SOURCE It seems like there is not much needed to be done here. `get_merge_query_def` from `ruleutils_17` is updated with "WHEN NOT MATCHED BY SOURCE" therefore `deparse_shard_query` parses the merge query for execution on the shard correctly. Relevant PG commit: https://github.com/postgres/postgres/commit/0294df2f1 --- .../distributed/planner/merge_planner.c | 4 +- src/test/regress/expected/pg17.out | 495 ++++++++++++++++++ src/test/regress/sql/pg17.sql | 370 +++++++++++++ 3 files changed, 867 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 5fee6e956..0abde46e4 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -1546,8 +1546,8 @@ FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) continue; } - /* NOT MATCHED can have either INSERT or DO NOTHING */ - if (action->commandType == CMD_NOTHING) + /* NOT MATCHED can have either INSERT, DO NOTHING or UPDATE(PG17) */ + if (action->commandType == CMD_NOTHING || action->commandType == CMD_UPDATE) { return NULL; } diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index dfd88e30e..83507bb15 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -2195,6 +2195,501 @@ CONTEXT: PL/pgSQL function public.explain_filter(text) line XX at FOR over EXEC RESET citus.log_remote_commands; -- End of EXPLAIN MEMORY SERIALIZE tests +-- Add support for MERGE ... WHEN NOT MATCHED BY SOURCE. +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/0294df2f1 +SET citus.next_shard_id TO 1072025; +-- Regular Postgres tables +CREATE TABLE postgres_target_1 (tid integer, balance float, val text); +CREATE TABLE postgres_target_2 (tid integer, balance float, val text); +CREATE TABLE postgres_source (sid integer, delta float); +INSERT INTO postgres_target_1 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_target_2 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Citus local tables +CREATE TABLE citus_local_target (tid integer, balance float, val text); +CREATE TABLE citus_local_source (sid integer, delta float); +SELECT citus_add_local_table_to_metadata('citus_local_target'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT citus_add_local_table_to_metadata('citus_local_source'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO citus_local_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_local_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Citus distributed tables +CREATE TABLE citus_distributed_target (tid integer, balance float, val text); +CREATE TABLE citus_distributed_source (sid integer, delta float); +SELECT create_distributed_table('citus_distributed_target', 'tid'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('citus_distributed_source', 'sid'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO citus_distributed_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_distributed_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Citus reference tables +CREATE TABLE citus_reference_target (tid integer, balance float, val text); +CREATE TABLE citus_reference_source (sid integer, delta float); +SELECT create_reference_table('citus_reference_target'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('citus_reference_source'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO citus_reference_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_reference_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Try all combinations of tables with two queries: +-- 1: Simple Merge +-- 2: Merge with a constant qual +-- Run the merge queries with the postgres tables +-- to save the expected output +-- try simple MERGE +MERGE INTO postgres_target_1 t + USING postgres_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_1 ORDER BY tid, val; + tid | balance | val +--------------------------------------------------------------------- + 1 | 110 | initial updated by merge + 2 | 20 | inserted by merge + 3 | 330 | initial updated by merge + 4 | 40 | inserted by merge + 5 | 550 | initial updated by merge + 6 | 60 | inserted by merge + 7 | 770 | initial updated by merge + 8 | 80 | inserted by merge + 9 | 990 | initial updated by merge + 10 | 100 | inserted by merge + 11 | 1210 | initial updated by merge + 12 | 120 | inserted by merge + 13 | 1430 | initial updated by merge + 14 | 140 | inserted by merge + 15 | 1500 | initial not matched by source +(15 rows) + +-- same with a constant qual +MERGE INTO postgres_target_2 t + USING postgres_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_2 ORDER BY tid, val; + tid | balance | val +--------------------------------------------------------------------- + 1 | 110 | initial updated by merge + 2 | 20 | inserted by merge + 3 | 300 | initial not matched by source + 3 | 30 | inserted by merge + 4 | 40 | inserted by merge + 5 | 500 | initial not matched by source + 5 | 50 | inserted by merge + 6 | 60 | inserted by merge + 7 | 700 | initial not matched by source + 7 | 70 | inserted by merge + 8 | 80 | inserted by merge + 9 | 900 | initial not matched by source + 9 | 90 | inserted by merge + 10 | 100 | inserted by merge + 11 | 1100 | initial not matched by source + 11 | 110 | inserted by merge + 12 | 120 | inserted by merge + 13 | 1300 | initial not matched by source + 13 | 130 | inserted by merge + 14 | 140 | inserted by merge + 15 | 1500 | initial not matched by source +(21 rows) + +-- function to compare the output from Citus tables +-- with the expected output from Postgres tables +CREATE OR REPLACE FUNCTION compare_tables(table1 TEXT, table2 TEXT) RETURNS BOOLEAN AS $$ +DECLARE ret BOOL; +BEGIN +EXECUTE 'select count(*) = 0 from (( + SELECT * FROM ' || table1 || + ' EXCEPT + SELECT * FROM ' || table2 || ' ) + UNION ALL ( + SELECT * FROM ' || table2 || + ' EXCEPT + SELECT * FROM ' || table1 || ' ))' INTO ret; +RETURN ret; +END +$$ LANGUAGE PLPGSQL; +-- Local-Local +-- Let's also print the command here +-- try simple MERGE +BEGIN; +SET citus.log_local_commands TO on; +MERGE INTO citus_local_target t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_local_target t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Local-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_local_target t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_local_target t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Local-Distributed - Merge currently not supported, Feature in development. +-- try simple MERGE +MERGE INTO citus_local_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +ERROR: MERGE involving repartition of rows is supported only if the target is distributed +-- Distributed-Local +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Distributed-Distributed +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_distributed_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Distributed-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Reference-N/A - Reference table as target is not allowed in Merge +-- try simple MERGE +MERGE INTO citus_reference_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +ERROR: Reference table as target is not allowed in MERGE command +-- Complex repartition query example with a mix of tables +-- Example from blog post +-- https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge +-- Contains information about the machines in the manufacturing facility +CREATE TABLE machines ( + machine_id NUMERIC PRIMARY KEY, + machine_name VARCHAR(100), + location VARCHAR(50), + status VARCHAR(20) +); +SELECT create_reference_table('machines'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Holds data on the various sensors installed on each machine +CREATE TABLE sensors ( + sensor_id NUMERIC PRIMARY KEY, + sensor_name VARCHAR(100), + machine_id NUMERIC, + sensor_type VARCHAR(50) +); +SELECT create_distributed_table('sensors', 'sensor_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Stores real-time readings from the sensors +CREATE TABLE sensor_readings ( + reading_id NUMERIC , + sensor_id NUMERIC, + reading_value NUMERIC, + reading_timestamp TIMESTAMP +); +SELECT create_distributed_table('sensor_readings', 'sensor_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Holds real-time sensor readings for machines on 'Production Floor 1' +CREATE TABLE real_sensor_readings ( + real_reading_id NUMERIC , + sensor_id NUMERIC, + reading_value NUMERIC, + reading_timestamp TIMESTAMP +); +SELECT create_distributed_table('real_sensor_readings', 'sensor_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Insert data into the machines table +INSERT INTO machines (machine_id, machine_name, location, status) +VALUES + (1, 'Machine A', 'Production Floor 1', 'Active'), + (2, 'Machine B', 'Production Floor 2', 'Active'), + (3, 'Machine C', 'Production Floor 1', 'Inactive'); +-- Insert data into the sensors table +INSERT INTO sensors (sensor_id, sensor_name, machine_id, sensor_type) +VALUES + (1, 'Temperature Sensor 1', 1, 'Temperature'), + (2, 'Pressure Sensor 1', 1, 'Pressure'), + (3, 'Temperature Sensor 2', 2, 'Temperature'), + (4, 'Vibration Sensor 1', 3, 'Vibration'); +-- Insert data into the real_sensor_readings table +INSERT INTO real_sensor_readings (real_reading_id, sensor_id, reading_value, reading_timestamp) +VALUES + (1, 1, 35.6, TIMESTAMP '2023-07-20 10:15:00'), + (2, 1, 36.8, TIMESTAMP '2023-07-20 10:30:00'), + (3, 2, 100.5, TIMESTAMP '2023-07-20 10:15:00'), + (4, 2, 101.2, TIMESTAMP '2023-07-20 10:30:00'), + (5, 3, 36.2, TIMESTAMP '2023-07-20 10:15:00'), + (6, 3, 36.5, TIMESTAMP '2023-07-20 10:30:00'), + (7, 4, 0.02, TIMESTAMP '2023-07-20 10:15:00'), + (8, 4, 0.03, TIMESTAMP '2023-07-20 10:30:00'); +-- Insert DUMMY data to use for WHEN NOT MATCHED BY SOURCE +INSERT INTO sensor_readings VALUES (0, 0, 0, TIMESTAMP '2023-07-20 10:15:00'); +SET client_min_messages TO DEBUG1; +-- Complex merge query which needs repartitioning +MERGE INTO sensor_readings SR +USING (SELECT +rsr.sensor_id, +AVG(rsr.reading_value) AS average_reading, +MAX(rsr.reading_timestamp) AS last_reading_timestamp, +MAX(rsr.real_reading_id) AS rid +FROM sensors s +INNER JOIN machines m ON s.machine_id = m.machine_id +INNER JOIN real_sensor_readings rsr ON s.sensor_id = rsr.sensor_id +WHERE m.location = 'Production Floor 1' +GROUP BY rsr.sensor_id +) NEW_READINGS +ON (SR.sensor_id = NEW_READINGS.sensor_id) +-- Existing reading, update it +WHEN MATCHED THEN +UPDATE SET reading_value = NEW_READINGS.average_reading, reading_timestamp = NEW_READINGS.last_reading_timestamp +-- New reading, record it +WHEN NOT MATCHED BY TARGET THEN +INSERT (reading_id, sensor_id, reading_value, reading_timestamp) +VALUES (NEW_READINGS.rid, NEW_READINGS.sensor_id, +NEW_READINGS.average_reading, NEW_READINGS.last_reading_timestamp) +-- Target has dummy entry not matched by source +-- dummy move change reading_value to 100 to notice the change +WHEN NOT MATCHED BY SOURCE THEN +UPDATE SET reading_value = 100; +DEBUG: A mix of distributed and reference table, try repartitioning +DEBUG: A mix of distributed and reference table, routable query is not possible +DEBUG: Creating MERGE repartition plan +DEBUG: Using column - index:0 from the source list to redistribute +DEBUG: Executing subplans of the source query and storing the results at the respective node(s) +DEBUG: Redistributing source result rows across nodes +DEBUG: Executing final MERGE on workers using intermediate results +DEBUG: +DEBUG: +RESET client_min_messages; +-- Expected output is: +-- reading_id | sensor_id | reading_value | reading_timestamp +-- ------------+-----------+------------------------+--------------------- +-- 0 | 0 | 100 | 2023-07-20 10:15:00 +-- 2 | 1 | 36.2000000000000000 | 2023-07-20 10:30:00 +-- 4 | 2 | 100.8500000000000000 | 2023-07-20 10:30:00 +-- 8 | 4 | 0.02500000000000000000 | 2023-07-20 10:30:00 +SELECT * FROM sensor_readings ORDER BY 1; + reading_id | sensor_id | reading_value | reading_timestamp +--------------------------------------------------------------------- + 0 | 0 | 100 | Thu Jul 20 10:15:00 2023 + 2 | 1 | 36.2000000000000000 | Thu Jul 20 10:30:00 2023 + 4 | 2 | 100.8500000000000000 | Thu Jul 20 10:30:00 2023 + 8 | 4 | 0.02500000000000000000 | Thu Jul 20 10:30:00 2023 +(4 rows) + +-- End of MERGE ... WHEN NOT MATCHED BY SOURCE tests \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index 70d5f68a8..e4843db44 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -1080,6 +1080,376 @@ select public.explain_filter('explain (analyze,serialize) create temp table expl RESET citus.log_remote_commands; -- End of EXPLAIN MEMORY SERIALIZE tests +-- Add support for MERGE ... WHEN NOT MATCHED BY SOURCE. +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/0294df2f1 + +SET citus.next_shard_id TO 1072025; + +-- Regular Postgres tables +CREATE TABLE postgres_target_1 (tid integer, balance float, val text); +CREATE TABLE postgres_target_2 (tid integer, balance float, val text); +CREATE TABLE postgres_source (sid integer, delta float); +INSERT INTO postgres_target_1 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_target_2 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_source SELECT id, id * 10 FROM generate_series(1,14) AS id; + +-- Citus local tables +CREATE TABLE citus_local_target (tid integer, balance float, val text); +CREATE TABLE citus_local_source (sid integer, delta float); +SELECT citus_add_local_table_to_metadata('citus_local_target'); +SELECT citus_add_local_table_to_metadata('citus_local_source'); +INSERT INTO citus_local_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_local_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Citus distributed tables +CREATE TABLE citus_distributed_target (tid integer, balance float, val text); +CREATE TABLE citus_distributed_source (sid integer, delta float); +SELECT create_distributed_table('citus_distributed_target', 'tid'); +SELECT create_distributed_table('citus_distributed_source', 'sid'); +INSERT INTO citus_distributed_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_distributed_source SELECT id, id * 10 FROM generate_series(1,14) AS id; +-- Citus reference tables +CREATE TABLE citus_reference_target (tid integer, balance float, val text); +CREATE TABLE citus_reference_source (sid integer, delta float); +SELECT create_reference_table('citus_reference_target'); +SELECT create_reference_table('citus_reference_source'); +INSERT INTO citus_reference_target SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO citus_reference_source SELECT id, id * 10 FROM generate_series(1,14) AS id; + +-- Try all combinations of tables with two queries: +-- 1: Simple Merge +-- 2: Merge with a constant qual + +-- Run the merge queries with the postgres tables +-- to save the expected output + +-- try simple MERGE +MERGE INTO postgres_target_1 t + USING postgres_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_1 ORDER BY tid, val; + +-- same with a constant qual +MERGE INTO postgres_target_2 t + USING postgres_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_2 ORDER BY tid, val; + +-- function to compare the output from Citus tables +-- with the expected output from Postgres tables + +CREATE OR REPLACE FUNCTION compare_tables(table1 TEXT, table2 TEXT) RETURNS BOOLEAN AS $$ +DECLARE ret BOOL; +BEGIN +EXECUTE 'select count(*) = 0 from (( + SELECT * FROM ' || table1 || + ' EXCEPT + SELECT * FROM ' || table2 || ' ) + UNION ALL ( + SELECT * FROM ' || table2 || + ' EXCEPT + SELECT * FROM ' || table1 || ' ))' INTO ret; +RETURN ret; +END +$$ LANGUAGE PLPGSQL; + +-- Local-Local +-- Let's also print the command here +-- try simple MERGE +BEGIN; +SET citus.log_local_commands TO on; +MERGE INTO citus_local_target t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_local_target t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_2'); +ROLLBACK; + +-- Local-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_local_target t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_local_target t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_local_target', 'postgres_target_2'); +ROLLBACK; + +-- Local-Distributed - Merge currently not supported, Feature in development. +-- try simple MERGE +MERGE INTO citus_local_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; + +-- Distributed-Local +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); +ROLLBACK; + +-- Distributed-Distributed +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_distributed_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); +ROLLBACK; + +-- Distributed-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target', 'postgres_target_2'); +ROLLBACK; + +-- Reference-N/A - Reference table as target is not allowed in Merge +-- try simple MERGE +MERGE INTO citus_reference_target t + USING citus_distributed_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; + +-- Complex repartition query example with a mix of tables +-- Example from blog post +-- https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge + +-- Contains information about the machines in the manufacturing facility +CREATE TABLE machines ( + machine_id NUMERIC PRIMARY KEY, + machine_name VARCHAR(100), + location VARCHAR(50), + status VARCHAR(20) +); +SELECT create_reference_table('machines'); + +-- Holds data on the various sensors installed on each machine +CREATE TABLE sensors ( + sensor_id NUMERIC PRIMARY KEY, + sensor_name VARCHAR(100), + machine_id NUMERIC, + sensor_type VARCHAR(50) +); +SELECT create_distributed_table('sensors', 'sensor_id'); + +-- Stores real-time readings from the sensors +CREATE TABLE sensor_readings ( + reading_id NUMERIC , + sensor_id NUMERIC, + reading_value NUMERIC, + reading_timestamp TIMESTAMP +); +SELECT create_distributed_table('sensor_readings', 'sensor_id'); + +-- Holds real-time sensor readings for machines on 'Production Floor 1' +CREATE TABLE real_sensor_readings ( + real_reading_id NUMERIC , + sensor_id NUMERIC, + reading_value NUMERIC, + reading_timestamp TIMESTAMP +); +SELECT create_distributed_table('real_sensor_readings', 'sensor_id'); + +-- Insert data into the machines table +INSERT INTO machines (machine_id, machine_name, location, status) +VALUES + (1, 'Machine A', 'Production Floor 1', 'Active'), + (2, 'Machine B', 'Production Floor 2', 'Active'), + (3, 'Machine C', 'Production Floor 1', 'Inactive'); + +-- Insert data into the sensors table +INSERT INTO sensors (sensor_id, sensor_name, machine_id, sensor_type) +VALUES + (1, 'Temperature Sensor 1', 1, 'Temperature'), + (2, 'Pressure Sensor 1', 1, 'Pressure'), + (3, 'Temperature Sensor 2', 2, 'Temperature'), + (4, 'Vibration Sensor 1', 3, 'Vibration'); + +-- Insert data into the real_sensor_readings table +INSERT INTO real_sensor_readings (real_reading_id, sensor_id, reading_value, reading_timestamp) +VALUES + (1, 1, 35.6, TIMESTAMP '2023-07-20 10:15:00'), + (2, 1, 36.8, TIMESTAMP '2023-07-20 10:30:00'), + (3, 2, 100.5, TIMESTAMP '2023-07-20 10:15:00'), + (4, 2, 101.2, TIMESTAMP '2023-07-20 10:30:00'), + (5, 3, 36.2, TIMESTAMP '2023-07-20 10:15:00'), + (6, 3, 36.5, TIMESTAMP '2023-07-20 10:30:00'), + (7, 4, 0.02, TIMESTAMP '2023-07-20 10:15:00'), + (8, 4, 0.03, TIMESTAMP '2023-07-20 10:30:00'); + +-- Insert DUMMY data to use for WHEN NOT MATCHED BY SOURCE +INSERT INTO sensor_readings VALUES (0, 0, 0, TIMESTAMP '2023-07-20 10:15:00'); + +SET client_min_messages TO DEBUG1; +-- Complex merge query which needs repartitioning +MERGE INTO sensor_readings SR +USING (SELECT +rsr.sensor_id, +AVG(rsr.reading_value) AS average_reading, +MAX(rsr.reading_timestamp) AS last_reading_timestamp, +MAX(rsr.real_reading_id) AS rid +FROM sensors s +INNER JOIN machines m ON s.machine_id = m.machine_id +INNER JOIN real_sensor_readings rsr ON s.sensor_id = rsr.sensor_id +WHERE m.location = 'Production Floor 1' +GROUP BY rsr.sensor_id +) NEW_READINGS + +ON (SR.sensor_id = NEW_READINGS.sensor_id) + +-- Existing reading, update it +WHEN MATCHED THEN +UPDATE SET reading_value = NEW_READINGS.average_reading, reading_timestamp = NEW_READINGS.last_reading_timestamp + +-- New reading, record it +WHEN NOT MATCHED BY TARGET THEN +INSERT (reading_id, sensor_id, reading_value, reading_timestamp) +VALUES (NEW_READINGS.rid, NEW_READINGS.sensor_id, +NEW_READINGS.average_reading, NEW_READINGS.last_reading_timestamp) + +-- Target has dummy entry not matched by source +-- dummy move change reading_value to 100 to notice the change +WHEN NOT MATCHED BY SOURCE THEN +UPDATE SET reading_value = 100; + +RESET client_min_messages; + +-- Expected output is: +-- reading_id | sensor_id | reading_value | reading_timestamp +-- ------------+-----------+------------------------+--------------------- +-- 0 | 0 | 100 | 2023-07-20 10:15:00 +-- 2 | 1 | 36.2000000000000000 | 2023-07-20 10:30:00 +-- 4 | 2 | 100.8500000000000000 | 2023-07-20 10:30:00 +-- 8 | 4 | 0.02500000000000000000 | 2023-07-20 10:30:00 +SELECT * FROM sensor_readings ORDER BY 1; + +-- End of MERGE ... WHEN NOT MATCHED BY SOURCE tests \set VERBOSITY terse SET client_min_messages TO WARNING; From f18c21c2b40db3e876d42605efe973b86879de5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emel=20=C5=9Eim=C5=9Fek?= Date: Mon, 9 Oct 2023 22:33:08 +0300 Subject: [PATCH 113/155] Send keepalive messages in split decoder periodically to avoid wal receiver timeouts during large shard splits. (#7229) DESCRIPTION: Send keepalive messages during the logical replication phase of large shard splits to avoid timeouts. During the logical replication part of the shard split process, split decoder filters out the wal records produced by the initial copy. If the number of wal records is big, then split decoder ends up processing for a long time before sending out any wal records through pgoutput. Hence the wal receiver may time out and restarts repeatedly causing our split driver code catch up logic to fail. Notes: 1. If the wal_receiver_timeout is set to a very small number e.g. 600ms, it may time out before receiving the keepalives. My tests show that this code works best when the` wal_receiver_timeout `is set to 1minute, which is the default value. 2. Once a logical replication worker time outs, a new one gets launched. The new logical replication worker sets the pg_stat_subscription columns to initial values. E.g. the latest_end_lsn is set to 0. Our driver logic in `WaitForGroupedLogicalRepTargetsToCatchUp` can not handle LSN value to go back. This is the main reason for it to get stuck in the infinite loop. (cherry picked from commit e9035f6d32ae58405a3ecd64cd77cebe34ab7286) --- .../shardsplit/shardsplit_decoder.c | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/backend/distributed/shardsplit/shardsplit_decoder.c b/src/backend/distributed/shardsplit/shardsplit_decoder.c index bcd25ce2e..20dd01b0c 100644 --- a/src/backend/distributed/shardsplit/shardsplit_decoder.c +++ b/src/backend/distributed/shardsplit/shardsplit_decoder.c @@ -94,6 +94,46 @@ replication_origin_filter_cb(LogicalDecodingContext *ctx, RepOriginId origin_id) } +/* + * update_replication_progress is copied from Postgres 15. We use it to send keepalive + * messages when we are filtering out the wal changes resulting from the initial copy. + * If we do not send out messages long enough, wal reciever will time out. + * Postgres 16 has refactored this code such that keepalive messages are sent during + * reordering phase which is above change_cb. So we do not need to send keepalive in + * change_cb. + */ +#if (PG_VERSION_NUM < PG_VERSION_16) +static void +update_replication_progress(LogicalDecodingContext *ctx, bool skipped_xact) +{ + static int changes_count = 0; + + /* + * We don't want to try sending a keepalive message after processing each + * change as that can have overhead. Tests revealed that there is no + * noticeable overhead in doing it after continuously processing 100 or so + * changes. + */ +#define CHANGES_THRESHOLD 100 + + /* + * After continuously processing CHANGES_THRESHOLD changes, we + * try to send a keepalive message if required. + */ + if (ctx->end_xact || ++changes_count >= CHANGES_THRESHOLD) + { +#if (PG_VERSION_NUM >= PG_VERSION_15) + OutputPluginUpdateProgress(ctx, skipped_xact); +#else + OutputPluginUpdateProgress(ctx); +#endif + changes_count = 0; + } +} + + +#endif + /* * shard_split_change_cb function emits the incoming tuple change * to the appropriate destination shard. @@ -112,6 +152,12 @@ shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, return; } +#if (PG_VERSION_NUM < PG_VERSION_16) + + /* Send replication keepalive. */ + update_replication_progress(ctx, false); +#endif + /* check if the relation is publishable.*/ if (!is_publishable_relation(relation)) { From 1a8349aeaf91ca7a532b46ebbe32145e7c8bc6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Villemain?= Date: Thu, 2 Nov 2023 13:15:24 +0100 Subject: [PATCH 114/155] Fix #7242, CALL(@0) crash backend (#7288) When executing a prepared CALL, which is not pure SQL but available with some drivers like npgsql and jpgdbc, Citus entered a code path where a plan is not defined, while trying to increase its cost. Thus SIG11 when plan is a NULL pointer. Fix by only increasing plan cost when plan is not null. However, it is a bit suspicious to get here with a NULL plan and maybe a better change will be to not call ShardPlacementForFunctionColocatedWithDistTable() with a NULL plan at all (in call.c:134) bug hit with for example: ``` CallableStatement proc = con.prepareCall("{CALL p(?)}"); proc.registerOutParameter(1, java.sql.Types.BIGINT); proc.setInt(1, -100); proc.execute(); ``` where `p(bigint)` is a distributed "function" and the param the distribution key (also in a distributed table), see #7242 for details Fixes #7242 (cherry picked from commit 0678a2fd895670e0dd0a3b594915144d468f20e4) --- .../distributed/planner/distributed_planner.c | 1 + .../planner/function_call_delegation.c | 12 ++++++-- src/test/regress/citus_tests/common.py | 8 +++++ .../test/test_prepared_statements.py | 30 +++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/test/regress/citus_tests/test/test_prepared_statements.py diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 4e88155de..0ec3883da 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -703,6 +703,7 @@ DissuadePlannerFromUsingPlan(PlannedStmt *plan) * Arbitrarily high cost, but low enough that it can be added up * without overflowing by choose_custom_plan(). */ + Assert(plan != NULL); plan->planTree->total_cost = FLT_MAX / 100000000; } diff --git a/src/backend/distributed/planner/function_call_delegation.c b/src/backend/distributed/planner/function_call_delegation.c index 43f7d8f7f..4a79dc25a 100644 --- a/src/backend/distributed/planner/function_call_delegation.c +++ b/src/backend/distributed/planner/function_call_delegation.c @@ -531,8 +531,16 @@ ShardPlacementForFunctionColocatedWithDistTable(DistObjectCacheEntry *procedure, if (partitionParam->paramkind == PARAM_EXTERN) { - /* Don't log a message, we should end up here again without a parameter */ - DissuadePlannerFromUsingPlan(plan); + /* + * Don't log a message, we should end up here again without a + * parameter. + * Note that "plan" can be null, for example when a CALL statement + * is prepared. + */ + if (plan) + { + DissuadePlannerFromUsingPlan(plan); + } return NULL; } } diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 3459af33d..3460afc1b 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -582,6 +582,14 @@ class QueryRunner(ABC): with self.cur(**kwargs) as cur: cur.execute(query, params=params) + def sql_prepared(self, query, params=None, **kwargs): + """Run an SQL query, with prepare=True + + This opens a new connection and closes it once the query is done + """ + with self.cur(**kwargs) as cur: + cur.execute(query, params=params, prepare=True) + def sql_row(self, query, params=None, allow_empty_result=False, **kwargs): """Run an SQL query that returns a single row and returns this row diff --git a/src/test/regress/citus_tests/test/test_prepared_statements.py b/src/test/regress/citus_tests/test/test_prepared_statements.py new file mode 100644 index 000000000..761ecc30c --- /dev/null +++ b/src/test/regress/citus_tests/test/test_prepared_statements.py @@ -0,0 +1,30 @@ +def test_call_param(cluster): + # create a distributed table and an associated distributed procedure + # to ensure parameterized CALL succeed, even when the param is the + # distribution key. + coord = cluster.coordinator + coord.sql("CREATE TABLE test(i int)") + coord.sql( + """ + CREATE PROCEDURE p(_i INT) LANGUAGE plpgsql AS $$ + BEGIN + INSERT INTO test(i) VALUES (_i); + END; $$ + """ + ) + sql = "CALL p(%s)" + + # prepare/exec before distributing + coord.sql_prepared(sql, (1,)) + + coord.sql("SELECT create_distributed_table('test', 'i')") + coord.sql( + "SELECT create_distributed_function('p(int)', distribution_arg_name := '_i', colocate_with := 'test')" + ) + + # prepare/exec after distribution + coord.sql_prepared(sql, (2,)) + + sum_i = coord.sql_value("select sum(i) from test;") + + assert sum_i == 3 From 01945bd3a35951107f367a6239dbdde5715f8986 Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Mon, 8 Jan 2024 19:09:30 +0300 Subject: [PATCH 115/155] Fix getting heap tuple size (#7387) This fixes #7230. First of all, using HeapTupleHeaderGetDatumLength(heapTuple) is definetly wrong, it gives a number that's 4 times less than the correct tuple size (heapTuple.t_len). See https://github.com/postgres/postgres/blob/REL_16_0/src/include/access/htup_details.h#L455-L456 https://github.com/postgres/postgres/blob/REL_16_0/src/include/varatt.h#L279 https://github.com/postgres/postgres/blob/REL_16_0/src/include/varatt.h#L225-L226 When I fixed it, the limit_intermediate_size test failed, so I tried to understand what's going on there. In original commit fd546cf these queries were supposed to fail. Then in b3af63c three of the queries that were supposed to fail suddenly worked and tests were changed to pass without understanding why the output had changed or how to keep test testing what it had to test. Even comments saying that these queries should fail were left untouched. Commit message gives no clue about why exactly test has changed: > It seems that when we use adaptive executor instead of task tracker, we > exceed the intermediate result size less in the test. Therefore updated > the tests accordingly. Then 3fda2c3 also blindly raised the limit for one of the queries to keep it working: https://github.com/citusdata/citus/commit/3fda2c325413a3726824ce5f463d60ea84281eb9#diff-a9b7b617f9dfd345318cb8987d5897143ca1b723c87b81049bbadd94dcc86570R19 When in fe3caf3 that HeapTupleHeaderGetDatumLength(heapTuple) call was finally added, one of those test queries became failing again. The other two of them now also failing after the fix. I don't understand how exactly the calculation of "intermediate result size" that is limited by citus.max_intermediate_result_size had changed through b3af63c and fe3caf3, but these numbers are now closer to what they originally were when this limitation was added in fd546cf. So these queries should fail, like in the original version of the limit_intermediate_size test. Co-authored-by: Karina Litskevich (cherry picked from commit 20dc58cf5dfbf7cbb37a134afc26e75f3a79e144) --- .../distributed/executor/tuple_destination.c | 2 +- src/test/regress/citus_tests/run_test.py | 1 + .../expected/limit_intermediate_size.out | 28 ++++++------------- .../regress/sql/limit_intermediate_size.sql | 3 +- 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/backend/distributed/executor/tuple_destination.c b/src/backend/distributed/executor/tuple_destination.c index 3c44d21c0..b3c4b509c 100644 --- a/src/backend/distributed/executor/tuple_destination.c +++ b/src/backend/distributed/executor/tuple_destination.c @@ -109,7 +109,7 @@ TupleStoreTupleDestPutTuple(TupleDestination *self, Task *task, uint64 tupleSize = tupleLibpqSize; if (tupleSize == 0) { - tupleSize = HeapTupleHeaderGetDatumLength(heapTuple); + tupleSize = heapTuple->t_len; } /* diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 2b1c5210c..0f6153c93 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -184,6 +184,7 @@ DEPS = { "foreign_key_to_reference_shard_rebalance": TestDeps( "minimal_schedule", ["remove_coordinator_from_metadata"] ), + "limit_intermediate_size": TestDeps("base_schedule"), } diff --git a/src/test/regress/expected/limit_intermediate_size.out b/src/test/regress/expected/limit_intermediate_size.out index e6fd0e798..996c6536b 100644 --- a/src/test/regress/expected/limit_intermediate_size.out +++ b/src/test/regress/expected/limit_intermediate_size.out @@ -16,7 +16,8 @@ SELECT cte.user_id, cte.value_2 FROM cte,cte2 ORDER BY 1,2 LIMIT 10; ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 2 kB) DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. -SET citus.max_intermediate_result_size TO 17; +SET citus.max_intermediate_result_size TO 9; +-- regular adaptive executor CTE should fail WITH cte AS MATERIALIZED ( SELECT @@ -38,20 +39,9 @@ FROM ORDER BY 1,2 LIMIT 10; - user_id | value_2 ---------------------------------------------------------------------- - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 - 1 | 0 -(10 rows) - +ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 9 kB) +DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. +HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. -- router queries should be able to get limitted too SET citus.max_intermediate_result_size TO 2; -- this should pass, since we fetch small portions in each subplan @@ -117,11 +107,9 @@ WITH cte AS MATERIALIZED ( AND EXISTS (select * from cte2, cte3) ) SELECT count(*) FROM cte WHERE EXISTS (select * from cte); - count ---------------------------------------------------------------------- - 105 -(1 row) - +ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 4 kB) +DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. +HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. SET citus.max_intermediate_result_size TO 3; -- this should fail since the cte-subplan exceeds the limit even if the -- cte2 and cte3 does not diff --git a/src/test/regress/sql/limit_intermediate_size.sql b/src/test/regress/sql/limit_intermediate_size.sql index 8f64c31fd..38ef734e7 100644 --- a/src/test/regress/sql/limit_intermediate_size.sql +++ b/src/test/regress/sql/limit_intermediate_size.sql @@ -17,7 +17,8 @@ cte2 AS MATERIALIZED ( SELECT cte.user_id, cte.value_2 FROM cte,cte2 ORDER BY 1,2 LIMIT 10; -SET citus.max_intermediate_result_size TO 17; +SET citus.max_intermediate_result_size TO 9; +-- regular adaptive executor CTE should fail WITH cte AS MATERIALIZED ( SELECT From f42e8556ec946ca4dd00725a6b214bc467d9f6eb Mon Sep 17 00:00:00 2001 From: zhjwpku Date: Wed, 10 Jan 2024 18:15:19 +0800 Subject: [PATCH 116/155] [performance improvement] remove duplicate LoadShardList call (#7380) LoadShardList is called twice, which is not neccessary, and there is no need to sort the shard placement list since we only want to know the list length. (cherry picked from commit 8e979f7ac6cd96cfe4f63e0fd1a79f74d42eb640) --- src/backend/distributed/metadata/metadata_cache.c | 2 +- src/backend/distributed/utils/colocation_utils.c | 6 ++---- src/backend/distributed/utils/shardinterval_utils.c | 5 ++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 09560c0fb..2fa97bbbe 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -379,7 +379,7 @@ EnsureModificationsCanRun(void) /* - * EnsureModificationsCanRunOnRelation firsts calls into EnsureModificationsCanRun() and + * EnsureModificationsCanRunOnRelation first calls into EnsureModificationsCanRun() and * then does one more additional check. The additional check is to give a proper error * message if any relation that is modified is replicated, as replicated tables use * 2PC and 2PC cannot happen when recovery is in progress. diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index c87db3b95..e2af11a1d 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -362,10 +362,8 @@ ErrorIfShardPlacementsNotColocated(Oid leftRelationId, Oid rightRelationId) leftRelationName, rightRelationName))); } - List *leftPlacementList = ShardPlacementListSortedByWorker( - leftShardId); - List *rightPlacementList = ShardPlacementListSortedByWorker( - rightShardId); + List *leftPlacementList = ShardPlacementList(leftShardId); + List *rightPlacementList = ShardPlacementList(rightShardId); if (list_length(leftPlacementList) != list_length(rightPlacementList)) { diff --git a/src/backend/distributed/utils/shardinterval_utils.c b/src/backend/distributed/utils/shardinterval_utils.c index aa45c50cb..05df7d816 100644 --- a/src/backend/distributed/utils/shardinterval_utils.c +++ b/src/backend/distributed/utils/shardinterval_utils.c @@ -470,12 +470,11 @@ SingleReplicatedTable(Oid relationId) return false; } - List *shardIntervalList = LoadShardList(relationId); uint64 *shardIdPointer = NULL; - foreach_declared_ptr(shardIdPointer, shardIntervalList) + foreach_declared_ptr(shardIdPointer, shardList) { uint64 shardId = *shardIdPointer; - shardPlacementList = ShardPlacementListSortedByWorker(shardId); + shardPlacementList = ShardPlacementList(shardId); if (list_length(shardPlacementList) != 1) { From ddef97212ac7d5fc468ca5f7725bd713b64f1d63 Mon Sep 17 00:00:00 2001 From: eaydingol <60466783+eaydingol@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:32:49 +0300 Subject: [PATCH 117/155] Generate qualified relation name (#7427) This change refactors the code by using generate_qualified_relation_name from id instead of using a sequence of functions to generate the relation name. Fixes #6602 (cherry picked from commit ee11492a0ed080b4c669460ae523cb437ab2faeb) --- .../distributed/commands/alter_table.c | 75 +++++++------------ .../citus_add_local_table_to_metadata.c | 8 +- .../commands/create_distributed_table.c | 5 +- src/backend/distributed/commands/multi_copy.c | 8 +- src/backend/distributed/commands/view.c | 4 +- .../executor/insert_select_executor.c | 7 +- .../distributed/operations/shard_transfer.c | 6 +- .../distributed/worker/worker_drop_protocol.c | 6 +- 8 files changed, 36 insertions(+), 83 deletions(-) diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index 5e830fa47..d2f8348da 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -209,12 +209,9 @@ static void ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommand static bool HasAnyGeneratedStoredColumns(Oid relationId); static List * GetNonGeneratedStoredColumnNameList(Oid relationId); static void CheckAlterDistributedTableConversionParameters(TableConversionState *con); -static char * CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, - char *sequenceName, - char *sourceSchemaName, - char *sourceName, - char *targetSchemaName, - char *targetName); +static char * CreateWorkerChangeSequenceDependencyCommand(char *qualifiedSequeceName, + char *qualifiedSourceName, + char *qualifiedTargetName); static void ErrorIfMatViewSizeExceedsTheLimit(Oid matViewOid); static char * CreateMaterializedViewDDLCommand(Oid matViewOid); static char * GetAccessMethodForMatViewIfExists(Oid viewOid); @@ -791,13 +788,15 @@ ConvertTableInternal(TableConversionState *con) justBeforeDropCommands = lappend(justBeforeDropCommands, detachFromParentCommand); } + char *qualifiedRelationName = quote_qualified_identifier(con->schemaName, + con->relationName); + if (PartitionedTable(con->relationId)) { if (!con->suppressNoticeMessages) { ereport(NOTICE, (errmsg("converting the partitions of %s", - quote_qualified_identifier(con->schemaName, - con->relationName)))); + qualifiedRelationName))); } List *partitionList = PartitionList(con->relationId); @@ -870,9 +869,7 @@ ConvertTableInternal(TableConversionState *con) if (!con->suppressNoticeMessages) { - ereport(NOTICE, (errmsg("creating a new table for %s", - quote_qualified_identifier(con->schemaName, - con->relationName)))); + ereport(NOTICE, (errmsg("creating a new table for %s", qualifiedRelationName))); } TableDDLCommand *tableCreationCommand = NULL; @@ -999,8 +996,6 @@ ConvertTableInternal(TableConversionState *con) { continue; } - char *qualifiedRelationName = quote_qualified_identifier(con->schemaName, - con->relationName); TableConversionParameters cascadeParam = { .relationId = colocatedTableId, @@ -1750,9 +1745,7 @@ CreateMaterializedViewDDLCommand(Oid matViewOid) { StringInfo query = makeStringInfo(); - char *viewName = get_rel_name(matViewOid); - char *schemaName = get_namespace_name(get_rel_namespace(matViewOid)); - char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName); + char *qualifiedViewName = generate_qualified_relation_name(matViewOid); /* here we need to get the access method of the view to recreate it */ char *accessMethodName = GetAccessMethodForMatViewIfExists(matViewOid); @@ -1801,9 +1794,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, bool suppressNoticeMessages) { char *sourceName = get_rel_name(sourceId); - char *targetName = get_rel_name(targetId); - Oid schemaId = get_rel_namespace(sourceId); - char *schemaName = get_namespace_name(schemaId); + char *qualifiedSourceName = generate_qualified_relation_name(sourceId); + char *qualifiedTargetName = generate_qualified_relation_name(targetId); StringInfo query = makeStringInfo(); @@ -1811,8 +1803,7 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, { if (!suppressNoticeMessages) { - ereport(NOTICE, (errmsg("moving the data of %s", - quote_qualified_identifier(schemaName, sourceName)))); + ereport(NOTICE, (errmsg("moving the data of %s", qualifiedSourceName))); } if (!HasAnyGeneratedStoredColumns(sourceId)) @@ -1822,8 +1813,7 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, * "INSERT INTO .. SELECT *"". */ appendStringInfo(query, "INSERT INTO %s SELECT * FROM %s", - quote_qualified_identifier(schemaName, targetName), - quote_qualified_identifier(schemaName, sourceName)); + qualifiedTargetName, qualifiedSourceName); } else { @@ -1838,9 +1828,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, char *insertColumnString = StringJoin(nonStoredColumnNameList, ','); appendStringInfo(query, "INSERT INTO %s (%s) OVERRIDING SYSTEM VALUE SELECT %s FROM %s", - quote_qualified_identifier(schemaName, targetName), - insertColumnString, insertColumnString, - quote_qualified_identifier(schemaName, sourceName)); + qualifiedTargetName, insertColumnString, + insertColumnString, qualifiedSourceName); } ExecuteQueryViaSPI(query->data, SPI_OK_INSERT); @@ -1864,14 +1853,11 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, */ if (ShouldSyncTableMetadata(targetId)) { - Oid sequenceSchemaOid = get_rel_namespace(sequenceOid); - char *sequenceSchemaName = get_namespace_name(sequenceSchemaOid); - char *sequenceName = get_rel_name(sequenceOid); + char *qualifiedSequenceName = generate_qualified_relation_name(sequenceOid); char *workerChangeSequenceDependencyCommand = - CreateWorkerChangeSequenceDependencyCommand(sequenceSchemaName, - sequenceName, - schemaName, sourceName, - schemaName, targetName); + CreateWorkerChangeSequenceDependencyCommand(qualifiedSequenceName, + qualifiedSourceName, + qualifiedTargetName); SendCommandToWorkersWithMetadata(workerChangeSequenceDependencyCommand); } else if (ShouldSyncTableMetadata(sourceId)) @@ -1894,25 +1880,23 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, if (!suppressNoticeMessages) { - ereport(NOTICE, (errmsg("dropping the old %s", - quote_qualified_identifier(schemaName, sourceName)))); + ereport(NOTICE, (errmsg("dropping the old %s", qualifiedSourceName))); } resetStringInfo(query); appendStringInfo(query, "DROP %sTABLE %s CASCADE", IsForeignTable(sourceId) ? "FOREIGN " : "", - quote_qualified_identifier(schemaName, sourceName)); + qualifiedSourceName); ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); if (!suppressNoticeMessages) { - ereport(NOTICE, (errmsg("renaming the new table to %s", - quote_qualified_identifier(schemaName, sourceName)))); + ereport(NOTICE, (errmsg("renaming the new table to %s", qualifiedSourceName))); } resetStringInfo(query); appendStringInfo(query, "ALTER TABLE %s RENAME TO %s", - quote_qualified_identifier(schemaName, targetName), + qualifiedTargetName, quote_identifier(sourceName)); ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); } @@ -2172,18 +2156,13 @@ CheckAlterDistributedTableConversionParameters(TableConversionState *con) * worker_change_sequence_dependency query with the parameters. */ static char * -CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, char *sequenceName, - char *sourceSchemaName, char *sourceName, - char *targetSchemaName, char *targetName) +CreateWorkerChangeSequenceDependencyCommand(char *qualifiedSequeceName, + char *qualifiedSourceName, + char *qualifiedTargetName) { - char *qualifiedSchemaName = quote_qualified_identifier(sequenceSchemaName, - sequenceName); - char *qualifiedSourceName = quote_qualified_identifier(sourceSchemaName, sourceName); - char *qualifiedTargetName = quote_qualified_identifier(targetSchemaName, targetName); - StringInfo query = makeStringInfo(); appendStringInfo(query, "SELECT worker_change_sequence_dependency(%s, %s, %s)", - quote_literal_cstr(qualifiedSchemaName), + quote_literal_cstr(qualifiedSequeceName), quote_literal_cstr(qualifiedSourceName), quote_literal_cstr(qualifiedTargetName)); diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index e2baa1025..dfc57f096 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -1160,9 +1160,7 @@ DropIdentitiesOnTable(Oid relationId) if (attributeForm->attidentity) { - char *tableName = get_rel_name(relationId); - char *schemaName = get_namespace_name(get_rel_namespace(relationId)); - char *qualifiedTableName = quote_qualified_identifier(schemaName, tableName); + char *qualifiedTableName = generate_qualified_relation_name(relationId); StringInfo dropCommand = makeStringInfo(); @@ -1222,9 +1220,7 @@ DropViewsOnTable(Oid relationId) Oid viewId = InvalidOid; foreach_declared_oid(viewId, reverseOrderedViews) { - char *viewName = get_rel_name(viewId); - char *schemaName = get_namespace_name(get_rel_namespace(viewId)); - char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName); + char *qualifiedViewName = generate_qualified_relation_name(viewId); StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "DROP %sVIEW IF EXISTS %s", diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 2cba7e1a5..7af6f2dd0 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -1325,10 +1325,7 @@ CreateCitusTable(Oid relationId, CitusTableType tableType, { List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; - Oid namespaceId = get_rel_namespace(relationId); - char *schemaName = get_namespace_name(namespaceId); - char *relationName = get_rel_name(relationId); - char *parentRelationName = quote_qualified_identifier(schemaName, relationName); + char *parentRelationName = generate_qualified_relation_name(relationId); /* * when there are many partitions, each call to CreateDistributedTable diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 3d79e0cbc..4b772ffb8 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -2549,12 +2549,8 @@ ShardIdForTuple(CitusCopyDestReceiver *copyDest, Datum *columnValues, bool *colu if (columnNulls[partitionColumnIndex]) { - Oid relationId = copyDest->distributedRelationId; - char *relationName = get_rel_name(relationId); - Oid schemaOid = get_rel_namespace(relationId); - char *schemaName = get_namespace_name(schemaOid); - char *qualifiedTableName = quote_qualified_identifier(schemaName, - relationName); + char *qualifiedTableName = generate_qualified_relation_name( + copyDest->distributedRelationId); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("the partition column of table %s cannot be NULL", diff --git a/src/backend/distributed/commands/view.c b/src/backend/distributed/commands/view.c index aa9675a4d..0ffd00ec4 100644 --- a/src/backend/distributed/commands/view.c +++ b/src/backend/distributed/commands/view.c @@ -392,9 +392,7 @@ CreateViewDDLCommand(Oid viewOid) static void AppendQualifiedViewNameToCreateViewCommand(StringInfo buf, Oid viewOid) { - char *viewName = get_rel_name(viewOid); - char *schemaName = get_namespace_name(get_rel_namespace(viewOid)); - char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName); + char *qualifiedViewName = generate_qualified_relation_name(viewOid); appendStringInfo(buf, "%s ", qualifiedViewName); } diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index 582ec0d3c..76dde345f 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -143,15 +143,10 @@ NonPushableInsertSelectExecScan(CustomScanState *node) targetRelation->partitionColumn); if (distributionColumnIndex == -1) { - char *relationName = get_rel_name(targetRelationId); - Oid schemaOid = get_rel_namespace(targetRelationId); - char *schemaName = get_namespace_name(schemaOid); - ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg( "the partition column of table %s should have a value", - quote_qualified_identifier(schemaName, - relationName)))); + generate_qualified_relation_name(targetRelationId)))); } TargetEntry *selectPartitionTE = list_nth(selectQuery->targetList, diff --git a/src/backend/distributed/operations/shard_transfer.c b/src/backend/distributed/operations/shard_transfer.c index 8e3306a2d..aa70efe0a 100644 --- a/src/backend/distributed/operations/shard_transfer.c +++ b/src/backend/distributed/operations/shard_transfer.c @@ -1940,11 +1940,7 @@ ConstructQualifiedShardName(ShardInterval *shardInterval) static List * RecreateTableDDLCommandList(Oid relationId) { - const char *relationName = get_rel_name(relationId); - Oid relationSchemaId = get_rel_namespace(relationId); - const char *relationSchemaName = get_namespace_name(relationSchemaId); - const char *qualifiedRelationName = quote_qualified_identifier(relationSchemaName, - relationName); + const char *qualifiedRelationName = generate_qualified_relation_name(relationId); StringInfo dropCommand = makeStringInfo(); diff --git a/src/backend/distributed/worker/worker_drop_protocol.c b/src/backend/distributed/worker/worker_drop_protocol.c index 1103d6c90..c4c2fe5b5 100644 --- a/src/backend/distributed/worker/worker_drop_protocol.c +++ b/src/backend/distributed/worker/worker_drop_protocol.c @@ -170,14 +170,10 @@ WorkerDropDistributedTable(Oid relationId) */ if (!IsAnyObjectAddressOwnedByExtension(list_make1(distributedTableObject), NULL)) { - char *relName = get_rel_name(relationId); - Oid schemaId = get_rel_namespace(relationId); - char *schemaName = get_namespace_name(schemaId); - StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "DROP%sTABLE %s CASCADE", IsForeignTable(relationId) ? " FOREIGN " : " ", - quote_qualified_identifier(schemaName, relName)); + generate_qualified_relation_name(relationId)); Node *dropCommandNode = ParseTreeNode(dropCommand->data); From 657575114c513ff7b893121042c9bb17ac537521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Sedl=C3=A1k?= Date: Wed, 7 Feb 2024 13:04:52 +0100 Subject: [PATCH 118/155] Fail early when shard can't be safely moved to a new node (#7467) DESCRIPTION: citus_move_shard_placement now fails early when shard cannot be safely moved The implementation is quite simplistic - `citus_move_shard_placement(...)` will fail with an error if there's any new node in the cluster that doesn't have reference tables yet. It could have been finer-grained, i.e. erroring only when trying to move a shard to an unitialized node. Looking at the related functions - `replicate_reference_tables()` or `citus_rebalance_start()`, I think it's acceptable behaviour. These other functions also treat "any" unitialized node as a temporary anomaly. Fixes #7426 --------- Co-authored-by: Jelte Fennema-Nio --- .../distributed/operations/shard_transfer.c | 11 +++ .../regress/expected/shard_rebalancer.out | 68 +++++++++++++++++++ src/test/regress/sql/shard_rebalancer.sql | 37 ++++++++++ 3 files changed, 116 insertions(+) diff --git a/src/backend/distributed/operations/shard_transfer.c b/src/backend/distributed/operations/shard_transfer.c index aa70efe0a..34e8e4693 100644 --- a/src/backend/distributed/operations/shard_transfer.c +++ b/src/backend/distributed/operations/shard_transfer.c @@ -294,6 +294,17 @@ citus_move_shard_placement(PG_FUNCTION_ARGS) CheckCitusVersion(ERROR); EnsureCoordinator(); + List *referenceTableIdList = NIL; + + if (HasNodesWithMissingReferenceTables(&referenceTableIdList)) + { + ereport(ERROR, (errmsg("there are missing reference tables on some nodes"), + errhint("Copy reference tables first with " + "replicate_reference_tables() or use " + "citus_rebalance_start() that will do it automatically." + ))); + } + int64 shardId = PG_GETARG_INT64(0); char *sourceNodeName = text_to_cstring(PG_GETARG_TEXT_P(1)); int32 sourceNodePort = PG_GETARG_INT32(2); diff --git a/src/test/regress/expected/shard_rebalancer.out b/src/test/regress/expected/shard_rebalancer.out index bae42b558..2b47bac72 100644 --- a/src/test/regress/expected/shard_rebalancer.out +++ b/src/test/regress/expected/shard_rebalancer.out @@ -2374,6 +2374,74 @@ SELECT count(*) FROM pg_dist_partition; 0 (1 row) +-- verify a system with a new node won't copy distributed table shards without reference tables +SELECT 1 from master_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT public.wait_until_metadata_sync(30000); + wait_until_metadata_sync +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE r1 (a int PRIMARY KEY, b int); +SELECT create_reference_table('r1'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE d1 (a int PRIMARY KEY, b int); +SELECT create_distributed_table('d1', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER SEQUENCE pg_dist_groupid_seq RESTART WITH 15; +SELECT 1 from master_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- count the number of placements for the reference table to verify it is not available on +-- all nodes +SELECT count(*) +FROM pg_dist_shard +JOIN pg_dist_shard_placement USING (shardid) +WHERE logicalrelid = 'r1'::regclass; + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- #7426 We can't move shards to the fresh node before we copy reference tables there. +-- rebalance_table_shards() will do the copy, but the low-level +-- citus_move_shard_placement() should raise an error +SELECT citus_move_shard_placement(pg_dist_shard.shardid, nodename, nodeport, 'localhost', :worker_2_port) + FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid) + WHERE logicalrelid = 'd1'::regclass AND nodename = 'localhost' AND nodeport = :worker_1_port LIMIT 1; +ERROR: there are missing reference tables on some nodes +SELECT replicate_reference_tables(); + replicate_reference_tables +--------------------------------------------------------------------- + +(1 row) + +-- After replication, the move should succeed. +SELECT citus_move_shard_placement(pg_dist_shard.shardid, nodename, nodeport, 'localhost', :worker_2_port) + FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid) + WHERE logicalrelid = 'd1'::regclass AND nodename = 'localhost' AND nodeport = :worker_1_port LIMIT 1; + citus_move_shard_placement +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE d1, r1; -- verify a system having only reference tables will copy the reference tables when -- executing the rebalancer SELECT 1 from master_remove_node('localhost', :worker_2_port); diff --git a/src/test/regress/sql/shard_rebalancer.sql b/src/test/regress/sql/shard_rebalancer.sql index 07efa8617..8c1c45855 100644 --- a/src/test/regress/sql/shard_rebalancer.sql +++ b/src/test/regress/sql/shard_rebalancer.sql @@ -1326,6 +1326,43 @@ DROP TABLE t1, r1, r2; -- test suites should clean up their distributed tables. SELECT count(*) FROM pg_dist_partition; +-- verify a system with a new node won't copy distributed table shards without reference tables + +SELECT 1 from master_remove_node('localhost', :worker_2_port); +SELECT public.wait_until_metadata_sync(30000); + +CREATE TABLE r1 (a int PRIMARY KEY, b int); +SELECT create_reference_table('r1'); + +CREATE TABLE d1 (a int PRIMARY KEY, b int); +SELECT create_distributed_table('d1', 'a'); + +ALTER SEQUENCE pg_dist_groupid_seq RESTART WITH 15; +SELECT 1 from master_add_node('localhost', :worker_2_port); + +-- count the number of placements for the reference table to verify it is not available on +-- all nodes +SELECT count(*) +FROM pg_dist_shard +JOIN pg_dist_shard_placement USING (shardid) +WHERE logicalrelid = 'r1'::regclass; + +-- #7426 We can't move shards to the fresh node before we copy reference tables there. +-- rebalance_table_shards() will do the copy, but the low-level +-- citus_move_shard_placement() should raise an error +SELECT citus_move_shard_placement(pg_dist_shard.shardid, nodename, nodeport, 'localhost', :worker_2_port) + FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid) + WHERE logicalrelid = 'd1'::regclass AND nodename = 'localhost' AND nodeport = :worker_1_port LIMIT 1; + +SELECT replicate_reference_tables(); + +-- After replication, the move should succeed. +SELECT citus_move_shard_placement(pg_dist_shard.shardid, nodename, nodeport, 'localhost', :worker_2_port) + FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid) + WHERE logicalrelid = 'd1'::regclass AND nodename = 'localhost' AND nodeport = :worker_1_port LIMIT 1; + +DROP TABLE d1, r1; + -- verify a system having only reference tables will copy the reference tables when -- executing the rebalancer From e2a03ce6a03d18ff26b1409ef1d5f96a0ba9ebf6 Mon Sep 17 00:00:00 2001 From: eaydingol <60466783+eaydingol@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:46:49 +0300 Subject: [PATCH 119/155] Fix: store the previous shard cost for order verification (#7550) Store the previous shard cost so that the invariant checking performs as expected. --- src/backend/distributed/operations/shard_rebalancer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index 653c7ce23..a4c7364be 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -384,6 +384,7 @@ CheckRebalanceStateInvariants(const RebalanceState *state) Assert(shardCost->cost <= prevShardCost->cost); } totalCost += shardCost->cost; + prevShardCost = shardCost; } /* Check that utilization field is up to date. */ From 89ccafff50db046099c8e1728353a960e48495b9 Mon Sep 17 00:00:00 2001 From: Karina <55838532+Green-Chan@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:08:19 +0300 Subject: [PATCH 120/155] Fix server crash when trying to execute activate_node_snapshot() on a single-node cluster (#7552) This fixes #7551 reported by Egor Chindyaskin Function activate_node_snapshot() is not meant to be called on a cluster without worker nodes. This commit adds ERROR report for such case to prevent server crash. --- src/backend/distributed/test/metadata_sync.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/backend/distributed/test/metadata_sync.c b/src/backend/distributed/test/metadata_sync.c index 744183968..2c0634754 100644 --- a/src/backend/distributed/test/metadata_sync.c +++ b/src/backend/distributed/test/metadata_sync.c @@ -50,6 +50,13 @@ activate_node_snapshot(PG_FUNCTION_ARGS) * so we are using first primary worker node just for test purposes. */ WorkerNode *dummyWorkerNode = GetFirstPrimaryWorkerNode(); + if (dummyWorkerNode == NULL) + { + ereport(ERROR, (errmsg("no worker nodes found"), + errdetail("Function activate_node_snapshot is meant to be " + "used when running tests on a multi-node cluster " + "with workers."))); + } /* * Create MetadataSyncContext which is used throughout nodes' activation. From ef946e44afd7f236cc144e839a1eb3b9385bd499 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Wed, 20 Mar 2024 14:38:33 +0300 Subject: [PATCH 121/155] Fix incorrect "VALID UNTIL" assumption made for roles in node activation (#7534) Fixes https://github.com/citusdata/citus/issues/7533. DESCRIPTION: Fixes incorrect `VALID UNTIL` setting assumption made for roles when syncing them to new nodes --- src/backend/distributed/commands/role.c | 13 ++++++------- .../expected/create_role_propagation.out | 18 +++++++++--------- .../regress/expected/upgrade_post_11_after.out | 14 ++++++++++++++ src/test/regress/sql/upgrade_post_11_after.sql | 15 +++++++++++++++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index dd45c98d8..208539018 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -491,18 +491,17 @@ GenerateRoleOptionsList(HeapTuple tuple) options = lappend(options, makeDefElem("password", NULL, -1)); } - /* load valid unitl data from the heap tuple, use default of infinity if not set */ + /* load valid until data from the heap tuple */ Datum rolValidUntilDatum = SysCacheGetAttr(AUTHNAME, tuple, Anum_pg_authid_rolvaliduntil, &isNull); - char *rolValidUntil = "infinity"; if (!isNull) { - rolValidUntil = pstrdup((char *) timestamptz_to_str(rolValidUntilDatum)); - } + char *rolValidUntil = pstrdup((char *) timestamptz_to_str(rolValidUntilDatum)); - Node *validUntilStringNode = (Node *) makeString(rolValidUntil); - DefElem *validUntilOption = makeDefElem("validUntil", validUntilStringNode, -1); - options = lappend(options, validUntilOption); + Node *validUntilStringNode = (Node *) makeString(rolValidUntil); + DefElem *validUntilOption = makeDefElem("validUntil", validUntilStringNode, -1); + options = lappend(options, validUntilOption); + } return options; } diff --git a/src/test/regress/expected/create_role_propagation.out b/src/test/regress/expected/create_role_propagation.out index 59f7948a1..92c5a7a45 100644 --- a/src/test/regress/expected/create_role_propagation.out +++ b/src/test/regress/expected/create_role_propagation.out @@ -129,17 +129,17 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port); SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, (rolpassword != '') as pass_not_empty, rolvaliduntil FROM pg_authid WHERE rolname LIKE 'create\_%' ORDER BY rolname; rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | pass_not_empty | rolvaliduntil --------------------------------------------------------------------- - create_group | f | t | f | f | f | f | f | -1 | | infinity - create_group_2 | f | t | f | f | f | f | f | -1 | | infinity - create_role | f | t | f | f | f | f | f | -1 | | infinity - create_role"edge | f | t | f | f | f | f | f | -1 | | infinity - create_role'edge | f | t | f | f | f | f | f | -1 | | infinity - create_role_2 | f | t | f | f | f | f | f | -1 | | infinity - create_role_sysid | f | t | f | f | f | f | f | -1 | | infinity + create_group | f | t | f | f | f | f | f | -1 | | + create_group_2 | f | t | f | f | f | f | f | -1 | | + create_role | f | t | f | f | f | f | f | -1 | | + create_role"edge | f | t | f | f | f | f | f | -1 | | + create_role'edge | f | t | f | f | f | f | f | -1 | | + create_role_2 | f | t | f | f | f | f | f | -1 | | + create_role_sysid | f | t | f | f | f | f | f | -1 | | create_role_with_everything | t | t | t | t | t | t | t | 105 | t | Thu May 04 17:00:00 2045 PDT create_role_with_nothing | f | f | f | f | f | f | f | 3 | t | Mon May 04 17:00:00 2015 PDT - create_user | f | t | f | f | t | f | f | -1 | | infinity - create_user_2 | f | t | f | f | t | f | f | -1 | | infinity + create_user | f | t | f | f | t | f | f | -1 | | + create_user_2 | f | t | f | f | t | f | f | -1 | | (11 rows) SELECT roleid::regrole::text AS role, member::regrole::text, grantor::regrole::text, admin_option FROM pg_auth_members WHERE roleid::regrole::text LIKE 'create\_%' ORDER BY 1, 2; diff --git a/src/test/regress/expected/upgrade_post_11_after.out b/src/test/regress/expected/upgrade_post_11_after.out index 422bc846f..49bd20432 100644 --- a/src/test/regress/expected/upgrade_post_11_after.out +++ b/src/test/regress/expected/upgrade_post_11_after.out @@ -67,6 +67,20 @@ SELECT 1 FROM run_command_on_workers($$SELECT pg_reload_conf()$$); 1 (2 rows) +-- In the version that we use for upgrade tests (v10.2.0), we propagate +-- "valid until" to the workers as "infinity" even if it's not set. And +-- given that "postgres" role is created in the older version, "valid until" +-- is set to "infinity" on the workers while this is not the case for +-- coordinator. See https://github.com/citusdata/citus/issues/7533. +-- +-- We're fixing this for new versions of Citus and we'll probably backport +-- this to some older versions too. However, v10.2.0 won't ever have this +-- fix. +-- +-- For this reason, here we set "valid until" to "infinity" for all the +-- nodes so that below query doesn't report any difference between the +-- metadata on coordinator and workers. +ALTER ROLE postgres WITH VALID UNTIL 'infinity'; -- make sure that the metadata is consistent across all nodes -- we exclude the distributed_object_data as they are -- not sorted in the same order (as OIDs differ on the nodes) diff --git a/src/test/regress/sql/upgrade_post_11_after.sql b/src/test/regress/sql/upgrade_post_11_after.sql index ba9b12f3b..6d948ec34 100644 --- a/src/test/regress/sql/upgrade_post_11_after.sql +++ b/src/test/regress/sql/upgrade_post_11_after.sql @@ -27,6 +27,21 @@ SET datestyle = "ISO, YMD"; SELECT 1 FROM run_command_on_workers($$ALTER SYSTEM SET datestyle = "ISO, YMD";$$); SELECT 1 FROM run_command_on_workers($$SELECT pg_reload_conf()$$); +-- In the version that we use for upgrade tests (v10.2.0), we propagate +-- "valid until" to the workers as "infinity" even if it's not set. And +-- given that "postgres" role is created in the older version, "valid until" +-- is set to "infinity" on the workers while this is not the case for +-- coordinator. See https://github.com/citusdata/citus/issues/7533. +-- +-- We're fixing this for new versions of Citus and we'll probably backport +-- this to some older versions too. However, v10.2.0 won't ever have this +-- fix. +-- +-- For this reason, here we set "valid until" to "infinity" for all the +-- nodes so that below query doesn't report any difference between the +-- metadata on coordinator and workers. +ALTER ROLE postgres WITH VALID UNTIL 'infinity'; + -- make sure that the metadata is consistent across all nodes -- we exclude the distributed_object_data as they are -- not sorted in the same order (as OIDs differ on the nodes) From 5eb037ac50da642e4b5d7e4f50ca804163dce826 Mon Sep 17 00:00:00 2001 From: Evgeny Nechayev Date: Tue, 28 May 2024 03:39:13 +0300 Subject: [PATCH 122/155] =?UTF-8?q?Use=20macro=20wrapper=20to=20access=20P?= =?UTF-8?q?GPROC=20data,=20which=20allow=20to=20improve=20compa=E2=80=A6?= =?UTF-8?q?=20(#7607)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESCRIPTION: Use macro wrapper to access PGPROC data, to improve compatibility with PostgreSQL forks. --- src/backend/distributed/transaction/backend_data.c | 6 +++--- .../transaction/distributed_deadlock_detection.c | 2 +- src/backend/distributed/transaction/lock_graph.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index f23d58bd0..e2afd18f7 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -395,7 +395,7 @@ StoreAllActiveTransactions(Tuplestorestate *tupleStore, TupleDesc tupleDescripto bool showCurrentBackendDetails = showAllBackends; BackendData *currentBackend = &backendManagementShmemData->backends[backendIndex]; - PGPROC *currentProc = &ProcGlobal->allProcs[backendIndex]; + PGPROC *currentProc = GetPGProcByNumber(backendIndex); /* to work on data after releasing g spinlock to protect against errors */ uint64 transactionNumber = 0; @@ -420,7 +420,7 @@ StoreAllActiveTransactions(Tuplestorestate *tupleStore, TupleDesc tupleDescripto } Oid databaseId = currentBackend->databaseId; - int backendPid = ProcGlobal->allProcs[backendIndex].pid; + int backendPid = GetPGProcByNumber(backendIndex)->pid; /* * We prefer to use worker_query instead of distributedCommandOriginator in @@ -1280,7 +1280,7 @@ ActiveDistributedTransactionNumbers(void) /* build list of starting procs */ for (int curBackend = 0; curBackend < MaxBackends; curBackend++) { - PGPROC *currentProc = &ProcGlobal->allProcs[curBackend]; + PGPROC *currentProc = GetPGProcByNumber(curBackend); BackendData currentBackendData; if (currentProc->pid == 0) diff --git a/src/backend/distributed/transaction/distributed_deadlock_detection.c b/src/backend/distributed/transaction/distributed_deadlock_detection.c index 834fcbe4d..30b423028 100644 --- a/src/backend/distributed/transaction/distributed_deadlock_detection.c +++ b/src/backend/distributed/transaction/distributed_deadlock_detection.c @@ -375,7 +375,7 @@ AssociateDistributedTransactionWithBackendProc(TransactionNode *transactionNode) for (int backendIndex = 0; backendIndex < MaxBackends; ++backendIndex) { - PGPROC *currentProc = &ProcGlobal->allProcs[backendIndex]; + PGPROC *currentProc = GetPGProcByNumber(backendIndex); BackendData currentBackendData; /* we're not interested in processes that are not active or waiting on a lock */ diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index dadcfe0a4..fab7e8491 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -561,7 +561,7 @@ BuildLocalWaitGraph(bool onlyDistributedTx) /* build list of starting procs */ for (int curBackend = 0; curBackend < totalProcs; curBackend++) { - PGPROC *currentProc = &ProcGlobal->allProcs[curBackend]; + PGPROC *currentProc = GetPGProcByNumber(curBackend); BackendData currentBackendData; if (currentProc->pid == 0) From a115b3043a24729dadcac6f9be25d816a343cfe8 Mon Sep 17 00:00:00 2001 From: eaydingol <60466783+eaydingol@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:53:38 +0300 Subject: [PATCH 123/155] Check if the limit is null (#7665) DESCRIPTION: Add a check to see if the given limit is null. Fixes a bug by checking if the limit given in the query is null when the actual limit is computed with respect to the given offset. Prior to this change, null is interpreted as 0 during the limit calculation when both limit and offset are given. Fixes #7663 --- .../planner/multi_logical_optimizer.c | 29 +++++-- .../regress/expected/multi_limit_clause.out | 80 +++++++++++++++++++ src/test/regress/sql/multi_limit_clause.sql | 19 +++++ 3 files changed, 120 insertions(+), 8 deletions(-) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 393a5a0d3..4bf3169a3 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -4753,22 +4753,35 @@ WorkerLimitCount(Node *limitCount, Node *limitOffset, OrderByLimitReference if (workerLimitNode != NULL && limitOffset != NULL) { Const *workerLimitConst = (Const *) workerLimitNode; - Const *workerOffsetConst = (Const *) limitOffset; - int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); - int64 workerOffsetCount = DatumGetInt64(workerOffsetConst->constvalue); - workerLimitCount = workerLimitCount + workerOffsetCount; - workerLimitNode = (Node *) MakeIntegerConstInt64(workerLimitCount); + /* Only update the worker limit if the const is not null.*/ + if (!workerLimitConst->constisnull) + { + Const *workerOffsetConst = (Const *) limitOffset; + int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); + + /* If the offset is null, it defaults to 0 when cast to int64. */ + int64 workerOffsetCount = DatumGetInt64(workerOffsetConst->constvalue); + workerLimitCount = workerLimitCount + workerOffsetCount; + workerLimitNode = (Node *) MakeIntegerConstInt64(workerLimitCount); + } } /* display debug message on limit push down */ if (workerLimitNode != NULL) { Const *workerLimitConst = (Const *) workerLimitNode; - int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); + if (!workerLimitConst->constisnull) + { + int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); - ereport(DEBUG1, (errmsg("push down of limit count: " INT64_FORMAT, - workerLimitCount))); + ereport(DEBUG1, (errmsg("push down of limit count: " INT64_FORMAT, + workerLimitCount))); + } + else + { + ereport(DEBUG1, (errmsg("push down of limit count: ALL"))); + } } return workerLimitNode; diff --git a/src/test/regress/expected/multi_limit_clause.out b/src/test/regress/expected/multi_limit_clause.out index 65304b777..83cd05837 100644 --- a/src/test/regress/expected/multi_limit_clause.out +++ b/src/test/regress/expected/multi_limit_clause.out @@ -521,6 +521,86 @@ SELECT 1 | 1 (1 row) +-- check if we can correctly push the limit when it is null +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey LIMIT null; +DEBUG: push down of limit count: ALL + l_orderkey +--------------------------------------------------------------------- + 1 + 1 + 1 + 1 + 1 + 1 + 2 +(7 rows) + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET 1 LIMIT null; +DEBUG: push down of limit count: ALL + l_orderkey +--------------------------------------------------------------------- + 1 + 1 + 1 + 1 + 1 + 2 +(6 rows) + +SELECT count(*) FROM lineitem LIMIT null; +DEBUG: push down of limit count: ALL + count +--------------------------------------------------------------------- + 12000 +(1 row) + +SELECT count(*) FROM lineitem OFFSET 0 LIMIT null; +DEBUG: push down of limit count: ALL + count +--------------------------------------------------------------------- + 12000 +(1 row) + +-- check if we push the right limit when both offset and limit are given +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET 1 LIMIT 3; +DEBUG: push down of limit count: 4 + l_orderkey +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET null LIMIT 1; +DEBUG: push down of limit count: 1 + l_orderkey +--------------------------------------------------------------------- + 1 +(1 row) + +-- check if we can correctly push the limit when it is all +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 2 LIMIT all; +DEBUG: push down of limit count: ALL + l_orderkey +--------------------------------------------------------------------- + 1 + 1 + 1 + 1 + 1 + 1 +(6 rows) + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 2 OFFSET 2 LIMIT all; +DEBUG: push down of limit count: ALL + l_orderkey +--------------------------------------------------------------------- + 1 + 1 + 1 + 1 +(4 rows) + SET client_min_messages TO NOTICE; -- non constants should not push down CREATE OR REPLACE FUNCTION my_limit() diff --git a/src/test/regress/sql/multi_limit_clause.sql b/src/test/regress/sql/multi_limit_clause.sql index 8d14bbbc8..5e3b3e3de 100644 --- a/src/test/regress/sql/multi_limit_clause.sql +++ b/src/test/regress/sql/multi_limit_clause.sql @@ -222,6 +222,25 @@ SELECT ORDER BY 2 DESC, 1 LIMIT 5; +-- check if we can correctly push the limit when it is null +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey LIMIT null; + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET 1 LIMIT null; + +SELECT count(*) FROM lineitem LIMIT null; + +SELECT count(*) FROM lineitem OFFSET 0 LIMIT null; + +-- check if we push the right limit when both offset and limit are given +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET 1 LIMIT 3; + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 3 ORDER BY l_orderkey OFFSET null LIMIT 1; + +-- check if we can correctly push the limit when it is all +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 2 LIMIT all; + +SELECT l_orderkey FROM lineitem WHERE l_orderkey < 2 OFFSET 2 LIMIT all; + SET client_min_messages TO NOTICE; -- non constants should not push down From 063cff908ed5ad7bad7e7b8e9019c0d3e9bd7c5a Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Mon, 9 Sep 2024 17:09:56 +0300 Subject: [PATCH 124/155] Fix race condition in citus_set_coordinator_host when adding multiple coordinator nodes concurrently (#7682) When multiple sessions concurrently attempt to add the same coordinator node using `citus_set_coordinator_host`, there is a potential race condition. Both sessions may pass the initial metadata check (`isCoordinatorInMetadata`), but only one will succeed in adding the node. The other session will fail with an assertion error (`Assert(!nodeAlreadyExists)`), causing the server to crash. Even though the `AddNodeMetadata` function takes an exclusive lock, it appears that the lock is not preventing the race condition before the initial metadata check. - **Issue**: The current logic allows concurrent sessions to pass the check for existing coordinators, leading to an attempt to insert duplicate nodes, which triggers the assertion failure. - **Impact**: This race condition leads to crashes during operations that involve concurrent coordinator additions, as seen in https://github.com/citusdata/citus/issues/7646. **Test Plan:** - Isolation Test Limitation: An isolation test was added to simulate concurrent additions of the same coordinator node, but due to the behavior of PostgreSQL locking mechanisms, the test does not trigger the edge case. The lock applied within the function serializes the operations, preventing the race condition from occurring in the isolation test environment. While the edge case is difficult to reproduce in an isolation test, the fix addresses the core issue by ensuring concurrency control through proper locking. - Existing Tests: All existing tests related to node metadata and coordinator management have been run to ensure that no regressions were introduced. **After the Fix:** - Concurrent attempts to add the same coordinator node will be serialized. One session will succeed in adding the node, while the others will skip the operation without crashing the server. Co-authored-by: Mehmet YILMAZ (cherry picked from commit 477571569178ca8f48321bc396f1db07b6f2244f) --- src/backend/distributed/metadata/node_metadata.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c index 4e728f9ed..39edfec1d 100644 --- a/src/backend/distributed/metadata/node_metadata.c +++ b/src/backend/distributed/metadata/node_metadata.c @@ -217,6 +217,9 @@ citus_set_coordinator_host(PG_FUNCTION_ARGS) EnsureTransactionalMetadataSyncMode(); } + /* prevent concurrent modification */ + LockRelationOid(DistNodeRelationId(), RowExclusiveLock); + bool isCoordinatorInMetadata = false; WorkerNode *coordinatorNode = PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &isCoordinatorInMetadata); From 1893d9a900335385d559cd9c529dd4bcc8b0d263 Mon Sep 17 00:00:00 2001 From: Parag Jain <40451840+paragikjain@users.noreply.github.com> Date: Fri, 13 Sep 2024 09:46:39 +0530 Subject: [PATCH 125/155] [Bug Fix] : writing incorrect data to target Merge repartition Command (#7659) We were writing incorrect data to target collection in some cases of merge command. In case of repartition when source query is RELATION. We were referring to incorrect attribute number that was resulting into this incorrect behavior. Example : ![image](https://github.com/user-attachments/assets/a101cb36-7976-459c-befb-96a55a5b3dc1) ![image](https://github.com/user-attachments/assets/e5c83b7b-5b8e-4d79-a927-95684dc9ba49) I have added fixed tests as part of this PR , Thanks. (cherry picked from commit 5bad6c6a1def94e78dafe1583c5ac4487353ba15) --- .../distributed/planner/merge_planner.c | 2 +- .../planner/query_colocation_checker.c | 4 +- .../distributed/query_colocation_checker.h | 2 + src/test/regress/expected/merge_vcore.out | 106 ++++++++++++++++++ src/test/regress/sql/merge_vcore.sql | 84 ++++++++++++++ 5 files changed, 194 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 0abde46e4..9c0ba3cd3 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -904,7 +904,7 @@ ConvertRelationRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, newRangeTableRef->rtindex = SINGLE_RTE_INDEX; sourceResultsQuery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); sourceResultsQuery->targetList = - CreateAllTargetListForRelation(sourceRte->relid, requiredAttributes); + CreateFilteredTargetListForRelation(sourceRte->relid, requiredAttributes); List *restrictionList = GetRestrictInfoListForRelation(sourceRte, plannerRestrictionContext); List *copyRestrictionList = copyObject(restrictionList); diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index bef91618e..d298b0f46 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -45,8 +45,6 @@ static RangeTblEntry * AnchorRte(Query *subquery); static List * UnionRelationRestrictionLists(List *firstRelationList, List *secondRelationList); -static List * CreateFilteredTargetListForRelation(Oid relationId, - List *requiredAttributes); static List * CreateDummyTargetList(Oid relationId, List *requiredAttributes); static TargetEntry * CreateTargetEntryForColumn(Form_pg_attribute attributeTuple, Index rteIndex, @@ -378,7 +376,7 @@ CreateAllTargetListForRelation(Oid relationId, List *requiredAttributes) * only the required columns of the given relation. If there is not required * columns then a dummy NULL column is put as the only entry. */ -static List * +List * CreateFilteredTargetListForRelation(Oid relationId, List *requiredAttributes) { Relation relation = relation_open(relationId, AccessShareLock); diff --git a/src/include/distributed/query_colocation_checker.h b/src/include/distributed/query_colocation_checker.h index 2a46d364c..485e4a033 100644 --- a/src/include/distributed/query_colocation_checker.h +++ b/src/include/distributed/query_colocation_checker.h @@ -39,5 +39,7 @@ extern Query * WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, List *requiredAttributes, RTEPermissionInfo *perminfo); extern List * CreateAllTargetListForRelation(Oid relationId, List *requiredAttributes); +extern List * CreateFilteredTargetListForRelation(Oid relationId, + List *requiredAttributes); #endif /* QUERY_COLOCATION_CHECKER_H */ diff --git a/src/test/regress/expected/merge_vcore.out b/src/test/regress/expected/merge_vcore.out index 03f6f8820..0eccb811b 100644 --- a/src/test/regress/expected/merge_vcore.out +++ b/src/test/regress/expected/merge_vcore.out @@ -476,6 +476,112 @@ WHEN MATCHED THEN DO NOTHING; Filter: ('2'::bigint = id) (11 rows) +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- Bug Fix Test as part of this PR +-- Test 1 +CREATE TABLE source ( + id int, + age int, + salary int +); +CREATE TABLE target ( + id int, + age int, + salary int +); +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (id, salary) VALUES (source.id, source.salary); +SELECT * FROM TARGET; + id | age | salary +--------------------------------------------------------------------- + 1 | | 100000 +(1 row) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- Test 2 +CREATE TABLE source ( + id int, + age int, + salary int +); +CREATE TABLE target ( + id int, + age int, + salary int +); +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (salary, id) VALUES (source.salary, source.id); +SELECT * FROM TARGET; + id | age | salary +--------------------------------------------------------------------- + 1 | | 100000 +(1 row) + +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; +-- Test 3 +CREATE TABLE source ( + id int, + age int, + salary int +); +CREATE TABLE target ( + id int, + age int, + salary int +); +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (salary, id, age) VALUES (source.age, source.id, source.salary); +SELECT * FROM TARGET; + id | age | salary +--------------------------------------------------------------------- + 1 | 100000 | 30 +(1 row) + DROP TABLE IF EXISTS source; DROP TABLE IF EXISTS target; DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; diff --git a/src/test/regress/sql/merge_vcore.sql b/src/test/regress/sql/merge_vcore.sql index 472bbfe91..2ab95e874 100644 --- a/src/test/regress/sql/merge_vcore.sql +++ b/src/test/regress/sql/merge_vcore.sql @@ -312,6 +312,90 @@ WHEN MATCHED THEN DO NOTHING; DROP TABLE IF EXISTS source; DROP TABLE IF EXISTS target; + +-- Bug Fix Test as part of this PR +-- Test 1 +CREATE TABLE source ( + id int, + age int, + salary int +); + +CREATE TABLE target ( + id int, + age int, + salary int +); + +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); + +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (id, salary) VALUES (source.id, source.salary); + +SELECT * FROM TARGET; +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- Test 2 +CREATE TABLE source ( + id int, + age int, + salary int +); + +CREATE TABLE target ( + id int, + age int, + salary int +); + +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); + +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (salary, id) VALUES (source.salary, source.id); + +SELECT * FROM TARGET; +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + +-- Test 3 +CREATE TABLE source ( + id int, + age int, + salary int +); + +CREATE TABLE target ( + id int, + age int, + salary int +); + +SELECT create_distributed_table('source', 'id', colocate_with=>'none'); +SELECT create_distributed_table('target', 'id', colocate_with=>'none'); + +INSERT INTO source (id, age, salary) VALUES (1,30, 100000); + +MERGE INTO ONLY target USING source ON (source.id = target.id) +WHEN NOT MATCHED THEN +INSERT (salary, id, age) VALUES (source.age, source.id, source.salary); + +SELECT * FROM TARGET; +DROP TABLE IF EXISTS source; +DROP TABLE IF EXISTS target; + + + DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; From 353033d3f0104f2c596db2da1dfd696343459c53 Mon Sep 17 00:00:00 2001 From: Colm McHugh Date: Fri, 25 Oct 2024 11:24:01 +0000 Subject: [PATCH 126/155] [Bug Fix] [SEGFAULT] Querying distributed tables with window partition may cause segfault #7705 In function MasterAggregateMutator(), when the original Node is a Var node use makeVar() instead of copyObject() when constructing the Var node for the target list of the combine query. The varnullingrels field of the original Var node is ignored because it is not relevant for the combine query; copying this cause the problem in issue 7705, where a coordinator query had a Var with a reference to a non-existent join relation. (cherry picked from commit c52f36019fceeb8894f4c7a0e52d7225a0f78d09) --- .../planner/multi_logical_optimizer.c | 7 +- src/test/regress/expected/issue_7705.out | 248 ++++++++++++++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/issue_7705.sql | 72 +++++ 4 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/issue_7705.out create mode 100644 src/test/regress/sql/issue_7705.sql diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 4bf3169a3..029de7707 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -1557,9 +1557,10 @@ MasterAggregateMutator(Node *originalNode, MasterAggregateWalkerContext *walkerC } else if (IsA(originalNode, Var)) { - Var *newColumn = copyObject((Var *) originalNode); - newColumn->varno = masterTableId; - newColumn->varattno = walkerContext->columnId; + Var *origColumn = (Var *) originalNode; + Var *newColumn = makeVar(masterTableId, walkerContext->columnId, + origColumn->vartype, origColumn->vartypmod, + origColumn->varcollid, origColumn->varlevelsup); walkerContext->columnId++; newNode = (Node *) newColumn; diff --git a/src/test/regress/expected/issue_7705.out b/src/test/regress/expected/issue_7705.out new file mode 100644 index 000000000..20b078226 --- /dev/null +++ b/src/test/regress/expected/issue_7705.out @@ -0,0 +1,248 @@ +--- Test for verifying that column references (var nodes) in targets that cannot be pushed down +--- do not cause issues for the postgres planner, in particular postgres versions 16+, where the +--- varnullingrels field of a VAR node may contain relids of join relations that can make the var +--- NULL; in a rewritten distributed query without a join such relids do not have a meaning. +--- Issue #7705: [SEGFAULT] Querying distributed tables with window partition causes segmentation fault +--- https://github.com/citusdata/citus/issues/7705 +CREATE SCHEMA issue_7705; +SET search_path to 'issue_7705'; +SET citus.next_shard_id TO 30070000; +SET citus.shard_replication_factor TO 1; +SET citus.enable_local_execution TO ON; +CREATE TABLE t1 (id INT PRIMARY KEY); +INSERT INTO t1 VALUES (1), (2); +CREATE TABLE t2 (id INT, account_id INT, a2 INT, PRIMARY KEY(id, account_id)); +INSERT INTO t2 VALUES (3, 1, 10), (4, 2, 20), (5, 1, NULL); +SELECT create_distributed_table('t1', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$issue_7705.t1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('t2', 'account_id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$issue_7705.t2$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Test the issue seen in #7705; a target expression with +-- a window function that cannot be pushed down because the +-- partion by is not on the distribution column also includes +-- a column from the inner side of a left outer join, which +-- produces a non-empty varnullingrels set in PG 16 (and higher) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | 10 + 2 | 20 + 1 | +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on issue_7705.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on issue_7705.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | 10 + 2 | 20 + 1 | +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t2_30070004 t2 RIGHT JOIN issue_7705.t1_30070000 t1 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on issue_7705.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on issue_7705.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | + 1 | 10 + 2 | 20 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.id, (max(remote_scan.max) OVER (?)), remote_scan.worker_column_3 + Group Key: remote_scan.id, max(remote_scan.max) OVER (?) + -> WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on issue_7705.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on issue_7705.t1_30070000 t1 + Output: t1.id +(25 rows) + +CREATE SEQUENCE test_seq START 101; +CREATE OR REPLACE FUNCTION TEST_F(int) returns INT language sql stable as $$ select $1 + 42; $$ ; +-- Issue #7705 also occurs if a target expression includes a column +-- of a distributed table that is on the inner side of a left outer +-- join and a call to nextval(), because nextval() cannot be pushed +-- down, and must be run on the coordinator +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + id | test_f +--------------------------------------------------------------------- + 1 | 153 + 1 | + 2 | 165 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Result + Output: remote_scan.id, ((remote_scan.test_f + (nextval('test_seq'::regclass))::integer) + 42) + -> Sort + Output: remote_scan.id, remote_scan.test_f + Sort Key: remote_scan.id + -> Custom Scan (Citus Adaptive) + Output: remote_scan.id, remote_scan.test_f + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS test_f FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on issue_7705.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on issue_7705.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT t1.id, CASE nextval('test_seq') % 2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + id | case +--------------------------------------------------------------------- + 1 | 10 + 1 | 1 + 2 | 20 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, CASE nextval('test_seq') %2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Result + Output: remote_scan.id, CASE ((nextval('test_seq'::regclass) % '2'::bigint) = 0) WHEN CASE_TEST_EXPR THEN remote_scan."case" ELSE 1 END + -> Sort + Output: remote_scan.id, remote_scan."case" + Sort Key: remote_scan.id + -> Custom Scan (Citus Adaptive) + Output: remote_scan.id, remote_scan."case" + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS "case" FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on issue_7705.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on issue_7705.t1_30070000 t1 + Output: t1.id +(22 rows) + +--- cleanup +\set VERBOSITY TERSE +DROP SCHEMA issue_7705 CASCADE; +NOTICE: drop cascades to 4 other objects +RESET all; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 136311a2a..ae9d9ac6c 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -103,7 +103,7 @@ test: multi_dropped_column_aliases foreign_key_restriction_enforcement test: binary_protocol test: alter_table_set_access_method test: alter_distributed_table -test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 +test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 issue_7705 test: object_propagation_debug test: undistribute_table test: run_command_on_all_nodes diff --git a/src/test/regress/sql/issue_7705.sql b/src/test/regress/sql/issue_7705.sql new file mode 100644 index 000000000..950933017 --- /dev/null +++ b/src/test/regress/sql/issue_7705.sql @@ -0,0 +1,72 @@ +--- Test for verifying that column references (var nodes) in targets that cannot be pushed down +--- do not cause issues for the postgres planner, in particular postgres versions 16+, where the +--- varnullingrels field of a VAR node may contain relids of join relations that can make the var +--- NULL; in a rewritten distributed query without a join such relids do not have a meaning. +--- Issue #7705: [SEGFAULT] Querying distributed tables with window partition causes segmentation fault +--- https://github.com/citusdata/citus/issues/7705 + +CREATE SCHEMA issue_7705; +SET search_path to 'issue_7705'; +SET citus.next_shard_id TO 30070000; +SET citus.shard_replication_factor TO 1; +SET citus.enable_local_execution TO ON; + +CREATE TABLE t1 (id INT PRIMARY KEY); +INSERT INTO t1 VALUES (1), (2); + +CREATE TABLE t2 (id INT, account_id INT, a2 INT, PRIMARY KEY(id, account_id)); +INSERT INTO t2 VALUES (3, 1, 10), (4, 2, 20), (5, 1, NULL); + +SELECT create_distributed_table('t1', 'id'); +SELECT create_distributed_table('t2', 'account_id'); + +-- Test the issue seen in #7705; a target expression with +-- a window function that cannot be pushed down because the +-- partion by is not on the distribution column also includes +-- a column from the inner side of a left outer join, which +-- produces a non-empty varnullingrels set in PG 16 (and higher) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; + +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + +CREATE SEQUENCE test_seq START 101; +CREATE OR REPLACE FUNCTION TEST_F(int) returns INT language sql stable as $$ select $1 + 42; $$ ; + +-- Issue #7705 also occurs if a target expression includes a column +-- of a distributed table that is on the inner side of a left outer +-- join and a call to nextval(), because nextval() cannot be pushed +-- down, and must be run on the coordinator +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + +SELECT t1.id, CASE nextval('test_seq') % 2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, CASE nextval('test_seq') %2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + +--- cleanup +\set VERBOSITY TERSE +DROP SCHEMA issue_7705 CASCADE; +RESET all; From cdded256ef2122b24bc077826eaeacc31dfb867b Mon Sep 17 00:00:00 2001 From: Pavel Seleznev Date: Tue, 3 Dec 2024 17:10:36 +0300 Subject: [PATCH 127/155] Remove warnings on some builds (#7680) Co-authored-by: Pavel Seleznev (cherry picked from commit fe6d198ab2cb87073fe552ab8c45b011c97230c2) --- src/backend/columnar/columnar_tableam.c | 2 ++ src/backend/distributed/metadata/metadata_cache.c | 2 ++ src/backend/distributed/planner/insert_select_planner.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 148ccb507..83df11c42 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -3040,6 +3040,8 @@ AvailableExtensionVersionColumnar(void) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not found"))); + + return NULL; /* keep compiler happy */ } diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 2fa97bbbe..3451415d1 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -2521,6 +2521,8 @@ AvailableExtensionVersion(void) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not found"))); + + return NULL; /* keep compiler happy */ } diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index 178ea235d..ca0c74f8f 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -1810,6 +1810,8 @@ CastExpr(Expr *expr, Oid sourceType, Oid targetType, Oid targetCollation, ereport(ERROR, (errmsg("could not find a conversion path from type %d to %d", sourceType, targetType))); } + + return NULL; /* keep compiler happy */ } From a19e18021254a934a45c2c1aebf818cd09af1e10 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 23 Dec 2024 17:01:53 +0300 Subject: [PATCH 128/155] Avoid re-assigning the global pid for client backends and bg workers when the application_name changes (#7791) DESCRIPTION: Fixes a crash that happens because of unsafe catalog access when re-assigning the global pid after application_name changes. When application_name changes, we don't actually need to try re-assigning the global pid for external client backends because application_name doesn't affect the global pid for such backends. Plus, trying to re-assign the global pid for external client backends would unnecessarily cause performing a catalog access when the cached local node id is invalidated. However, accessing to the catalog tables is dangerous in certain situations like when we're not in a transaction block. And for the other types of backends, i.e., the Citus internal backends, we need to re-assign the global pid when the application_name changes because for such backends we simply extract the global pid inherited from the originating backend from the application_name -that's specified by originating backend when openning that connection- and this doesn't require catalog access. (cherry picked from commit 73411915a47d514cb17c0aaff0ce25ba35ae06d2) --- src/backend/distributed/shared_library_init.c | 31 +++++++++++----- .../test/run_from_same_connection.c | 4 +++ .../distributed/transaction/backend_data.c | 17 +++++++++ src/include/distributed/backend_data.h | 1 + .../regress/expected/remove_coordinator.out | 31 ++++++++++++++++ src/test/regress/sql/remove_coordinator.sql | 36 +++++++++++++++++++ 6 files changed, 111 insertions(+), 9 deletions(-) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index d004b1989..21ae6b4b5 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -2853,14 +2853,27 @@ ApplicationNameAssignHook(const char *newval, void *extra) DetermineCitusBackendType(newval); /* - * AssignGlobalPID might read from catalog tables to get the the local - * nodeid. But ApplicationNameAssignHook might be called before catalog - * access is available to the backend (such as in early stages of - * authentication). We use StartupCitusBackend to initialize the global pid - * after catalogs are available. After that happens this hook becomes - * responsible to update the global pid on later application_name changes. - * So we set the FinishedStartupCitusBackend flag in StartupCitusBackend to - * indicate when this responsibility handoff has happened. + * We use StartupCitusBackend to initialize the global pid after catalogs + * are available. After that happens this hook becomes responsible to update + * the global pid on later application_name changes. So we set the + * FinishedStartupCitusBackend flag in StartupCitusBackend to indicate when + * this responsibility handoff has happened. + * + * Also note that when application_name changes, we don't actually need to + * try re-assigning the global pid for external client backends and + * background workers because application_name doesn't affect the global + * pid for such backends - note that !IsExternalClientBackend() check covers + * both types of backends. Plus, + * trying to re-assign the global pid for such backends would unnecessarily + * cause performing a catalog access when the cached local node id is + * invalidated. However, accessing to the catalog tables is dangerous in + * certain situations like when we're not in a transaction block. And for + * the other types of backends, i.e., the Citus internal backends, we need + * to re-assign the global pid when the application_name changes because for + * such backends we simply extract the global pid inherited from the + * originating backend from the application_name -that's specified by + * originating backend when openning that connection- and this doesn't require + * catalog access. * * Another solution to the catalog table acccess problem would be to update * global pid lazily, like we do for HideShards. But that's not possible @@ -2870,7 +2883,7 @@ ApplicationNameAssignHook(const char *newval, void *extra) * as reasonably possible, which is also why we extract global pids in the * AuthHook already (extracting doesn't require catalog access). */ - if (FinishedStartupCitusBackend) + if (FinishedStartupCitusBackend && !IsExternalClientBackend()) { AssignGlobalPID(newval); } diff --git a/src/backend/distributed/test/run_from_same_connection.c b/src/backend/distributed/test/run_from_same_connection.c index 5663a42fa..9e664b03d 100644 --- a/src/backend/distributed/test/run_from_same_connection.c +++ b/src/backend/distributed/test/run_from_same_connection.c @@ -190,6 +190,9 @@ run_commands_on_session_level_connection_to_node(PG_FUNCTION_ARGS) /* * override_backend_data_gpid is a wrapper around SetBackendDataGpid(). + * Also sets distributedCommandOriginator to true since the only caller of + * this method calls this function actually wants this backend to + * be treated as a distributed command originator with the given global pid. */ Datum override_backend_data_gpid(PG_FUNCTION_ARGS) @@ -199,6 +202,7 @@ override_backend_data_gpid(PG_FUNCTION_ARGS) uint64 gpid = PG_GETARG_INT64(0); SetBackendDataGlobalPID(gpid); + SetBackendDataDistributedCommandOriginator(true); PG_RETURN_VOID(); } diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index e2afd18f7..384954b81 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -964,6 +964,23 @@ SetBackendDataGlobalPID(uint64 gpid) } +/* + * SetBackendDataDistributedCommandOriginator sets the distributedCommandOriginator + * field on MyBackendData. + */ +void +SetBackendDataDistributedCommandOriginator(bool distributedCommandOriginator) +{ + if (!MyBackendData) + { + return; + } + SpinLockAcquire(&MyBackendData->mutex); + MyBackendData->distributedCommandOriginator = distributedCommandOriginator; + SpinLockRelease(&MyBackendData->mutex); +} + + /* * GetGlobalPID returns the global process id of the current backend. */ diff --git a/src/include/distributed/backend_data.h b/src/include/distributed/backend_data.h index 8014fe5a6..5b3fcf2ac 100644 --- a/src/include/distributed/backend_data.h +++ b/src/include/distributed/backend_data.h @@ -61,6 +61,7 @@ extern void AssignGlobalPID(const char *applicationName); extern uint64 GetGlobalPID(void); extern void SetBackendDataDatabaseId(void); extern void SetBackendDataGlobalPID(uint64 gpid); +extern void SetBackendDataDistributedCommandOriginator(bool distributedCommandOriginator); extern uint64 ExtractGlobalPID(const char *applicationName); extern int ExtractNodeIdFromGlobalPID(uint64 globalPID, bool missingOk); extern int ExtractProcessIdFromGlobalPID(uint64 globalPID); diff --git a/src/test/regress/expected/remove_coordinator.out b/src/test/regress/expected/remove_coordinator.out index 0226a7cd0..e2fd5df02 100644 --- a/src/test/regress/expected/remove_coordinator.out +++ b/src/test/regress/expected/remove_coordinator.out @@ -5,6 +5,37 @@ SELECT master_remove_node('localhost', :master_port); (1 row) +-- to silence -potentially flaky- "could not establish connection after" warnings in below test +SET client_min_messages TO ERROR; +-- to fail fast when the hostname is not resolvable, as it will be the case below +SET citus.node_connection_timeout to '1s'; +BEGIN; + SET application_name TO 'new_app_name'; + -- that should fail because of bad hostname & port + SELECT citus_add_node('200.200.200.200', 1, 200); +ERROR: connection to the remote node postgres@200.200.200.200:1 failed + -- Since above command failed, now Postgres will need to revert the + -- application_name change made in this transaction and this will + -- happen within abort-transaction callback, so we won't be in a + -- transaction block while Postgres does that. + -- + -- And when the application_name changes, Citus tries to re-assign + -- the global pid but it does so only for Citus internal backends, + -- and doing so for Citus internal backends doesn't require being + -- in a transaction block and is safe. + -- + -- However, for the client external backends (like us here), Citus + -- doesn't re-assign the global pid because it's not needed and it's + -- not safe to do so outside of a transaction block. This is because, + -- it would require performing a catalog access to retrive the local + -- node id when the cached local node is invalidated like what just + -- happened here because of the failed citus_add_node() call made + -- above. + -- + -- So by failing here (rather than crashing), we ensure this behavior. +ROLLBACK; +RESET client_min_messages; +RESET citus.node_connection_timeout; -- restore coordinator for the rest of the tests SELECT citus_set_coordinator_host('localhost', :master_port); citus_set_coordinator_host diff --git a/src/test/regress/sql/remove_coordinator.sql b/src/test/regress/sql/remove_coordinator.sql index b0df327d1..35a8a5718 100644 --- a/src/test/regress/sql/remove_coordinator.sql +++ b/src/test/regress/sql/remove_coordinator.sql @@ -1,5 +1,41 @@ -- removing coordinator from pg_dist_node should update pg_dist_colocation SELECT master_remove_node('localhost', :master_port); +-- to silence -potentially flaky- "could not establish connection after" warnings in below test +SET client_min_messages TO ERROR; + +-- to fail fast when the hostname is not resolvable, as it will be the case below +SET citus.node_connection_timeout to '1s'; + +BEGIN; + SET application_name TO 'new_app_name'; + + -- that should fail because of bad hostname & port + SELECT citus_add_node('200.200.200.200', 1, 200); + + -- Since above command failed, now Postgres will need to revert the + -- application_name change made in this transaction and this will + -- happen within abort-transaction callback, so we won't be in a + -- transaction block while Postgres does that. + -- + -- And when the application_name changes, Citus tries to re-assign + -- the global pid but it does so only for Citus internal backends, + -- and doing so for Citus internal backends doesn't require being + -- in a transaction block and is safe. + -- + -- However, for the client external backends (like us here), Citus + -- doesn't re-assign the global pid because it's not needed and it's + -- not safe to do so outside of a transaction block. This is because, + -- it would require performing a catalog access to retrive the local + -- node id when the cached local node is invalidated like what just + -- happened here because of the failed citus_add_node() call made + -- above. + -- + -- So by failing here (rather than crashing), we ensure this behavior. +ROLLBACK; + +RESET client_min_messages; +RESET citus.node_connection_timeout; + -- restore coordinator for the rest of the tests SELECT citus_set_coordinator_host('localhost', :master_port); From d2ca63fb8cf49202b3f43da8d9350d73948d5db3 Mon Sep 17 00:00:00 2001 From: Teja Mupparti Date: Tue, 24 Dec 2024 14:42:15 -0800 Subject: [PATCH 129/155] For scenarios, such as, Bug 3697586: Server crashes when assigning distributed transaction: Raise an ERROR instead of a crash (cherry picked from commit ab7c13beb5ec7415dc881c9b72ac0881b9daf4e5) --- src/backend/distributed/transaction/backend_data.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index 384954b81..866b18fd2 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -855,6 +855,16 @@ GetCurrentDistributedTransactionId(void) void AssignDistributedTransactionId(void) { + /* + * MyBackendData should always be available. However, we observed some + * crashes where certain hooks were not executed. + * Bug 3697586: Server crashes when assigning distributed transaction + */ + if (!MyBackendData) + { + ereport(ERROR, (errmsg("backend is not ready for distributed transactions"))); + } + pg_atomic_uint64 *transactionNumberSequence = &backendManagementShmemData->nextTransactionNumber; From 7e316c90c407f6f8ea97c9ba9453b502a6b4ef74 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Tue, 26 Sep 2023 13:47:50 +0200 Subject: [PATCH 130/155] Shard moves/isolate report LSN's in lsn format (#7227) DESCRIPTION: Shard moves/isolate report LSN's in lsn format While investigating an issue with our catchup mechanism on certain postgres versions we noticed we print LSN's in the format of the native long type. This is an uncommon representation for LSN's in postgres logs. This patch changes the output of our log message to go from the long type representation to the native LSN type representation. Making it easier for postgres users to recognize and compare LSN's with other related reports. example of new output: ``` 2023-09-25 17:28:47.544 CEST [11345] LOG: The LSN of the target subscriptions on node localhost:9701 have increased from 0/0 to 0/E1ED20F8 at 2023-09-25 17:28:47.544165+02 where the source LSN is 1/415DCAD0 ``` (cherry picked from commit b87fbcbf792c46c789b4e77ef15646e91e4390fd) --- .../replication/multi_logical_replication.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backend/distributed/replication/multi_logical_replication.c b/src/backend/distributed/replication/multi_logical_replication.c index cd34a787a..2dbf8daa8 100644 --- a/src/backend/distributed/replication/multi_logical_replication.c +++ b/src/backend/distributed/replication/multi_logical_replication.c @@ -1882,14 +1882,15 @@ WaitForGroupedLogicalRepTargetsToCatchUp(XLogRecPtr sourcePosition, GetCurrentTimestamp(), logicalReplicationProgressReportTimeout)) { - ereport(LOG, (errmsg( - "The LSN of the target subscriptions on node %s:%d have " - "increased from %ld to %ld at %s where the source LSN is %ld ", - superuserConnection->hostname, - superuserConnection->port, previousTargetBeforeThisLoop, - targetPosition, - timestamptz_to_str(previousLSNIncrementTime), - sourcePosition))); + ereport(LOG, (errmsg("The LSN of the target subscriptions on node %s:%d " + "has increased from %X/%X to %X/%X at %s where the " + "source LSN is %X/%X ", + superuserConnection->hostname, + superuserConnection->port, + LSN_FORMAT_ARGS(previousTargetBeforeThisLoop), + LSN_FORMAT_ARGS(targetPosition), + timestamptz_to_str(previousLSNIncrementTime), + LSN_FORMAT_ARGS(sourcePosition)))); previousReportTime = GetCurrentTimestamp(); } From c55bc8c669071b73af69e1136e183b640d84b23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emel=20=C5=9Eim=C5=9Fek?= Date: Wed, 13 Nov 2024 14:21:08 +0300 Subject: [PATCH 131/155] Propagates SECURITY LABEL ON ROLE stmt (#7304) (#7735) Propagates SECURITY LABEL ON ROLE stmt (https://github.com/citusdata/citus/pull/7304) We propagate `SECURITY LABEL [for provider] ON ROLE rolename IS labelname` to the worker nodes. We also make sure to run the relevant `SecLabelStmt` commands on a newly added node by looking at roles found in `pg_shseclabel`. See official docs for explanation on how this command works: https://www.postgresql.org/docs/current/sql-security-label.html This command stores the role label in the `pg_shseclabel` catalog table. This commit also fixes the regex string in `check_gucs_are_alphabetically_sorted.sh` script such that it escapes the dot. Previously it was looking for all strings starting with "citus" instead of "citus." as it should. To test this feature, I currently make use of a special GUC to control label provider registration in PG_init when creating the Citus extension. (cherry picked from commit 0d1f18862be68350a2f784bcbe98344ddcad8a6d) Co-authored-by: Naisila Puka <37271756+naisila@users.noreply.github.com> (cherry picked from commit 686d2b46cadf8f8257797506c677d24c5598ba52) --- ci/check_gucs_are_alphabetically_sorted.sh | 2 +- gucs.out | 133 ++++++++++++++ .../commands/distribute_object_ops.c | 14 ++ src/backend/distributed/commands/role.c | 71 ++++++- src/backend/distributed/commands/seclabel.c | 125 +++++++++++++ .../deparser/deparse_seclabel_stmts.c | 79 ++++++++ .../distributed/operations/shard_rebalancer.c | 2 +- .../replication/multi_logical_replication.c | 4 +- src/backend/distributed/shared_library_init.c | 18 +- src/include/distributed/commands.h | 5 + src/include/distributed/deparser.h | 3 + src/include/distributed/shard_rebalancer.h | 2 +- .../regress/expected/multi_test_helpers.out | 30 +++ src/test/regress/expected/seclabel.out | 173 ++++++++++++++++++ src/test/regress/multi_1_schedule | 1 + src/test/regress/pg_regress_multi.pl | 7 +- src/test/regress/sql/multi_test_helpers.sql | 30 +++ src/test/regress/sql/seclabel.sql | 87 +++++++++ 18 files changed, 774 insertions(+), 12 deletions(-) create mode 100644 gucs.out create mode 100644 src/backend/distributed/commands/seclabel.c create mode 100644 src/backend/distributed/deparser/deparse_seclabel_stmts.c create mode 100644 src/test/regress/expected/seclabel.out create mode 100644 src/test/regress/sql/seclabel.sql diff --git a/ci/check_gucs_are_alphabetically_sorted.sh b/ci/check_gucs_are_alphabetically_sorted.sh index a769ae4fb..763b5305f 100755 --- a/ci/check_gucs_are_alphabetically_sorted.sh +++ b/ci/check_gucs_are_alphabetically_sorted.sh @@ -5,6 +5,6 @@ set -euo pipefail source ci/ci_helpers.sh # extract citus gucs in the form of "citus.X" -grep -o -E "(\.*\"citus.\w+\")," src/backend/distributed/shared_library_init.c > gucs.out +grep -o -E "(\.*\"citus\.\w+\")," src/backend/distributed/shared_library_init.c > gucs.out sort -c gucs.out rm gucs.out diff --git a/gucs.out b/gucs.out new file mode 100644 index 000000000..8501e6c1f --- /dev/null +++ b/gucs.out @@ -0,0 +1,133 @@ +"citus.all_modifications_commutative", +"citus.allow_modifications_from_workers_to_replicated_tables", +"citus.allow_nested_distributed_execution", +"citus.allow_unsafe_constraints", +"citus.allow_unsafe_locks_from_workers", +"citus.background_task_queue_interval", +"citus.check_available_space_before_move", +"citus.cluster_name", +"citus.coordinator_aggregation_strategy", +"citus.copy_switchover_threshold", +"citus.count_distinct_error_rate", +"citus.cpu_priority", +"citus.cpu_priority_for_logical_replication_senders", +"citus.create_object_propagation", +"citus.defer_drop_after_shard_move", +"citus.defer_drop_after_shard_split", +"citus.defer_shard_delete_interval", +"citus.desired_percent_disk_available_after_move", +"citus.distributed_deadlock_detection_factor", +"citus.enable_alter_database_owner", +"citus.enable_alter_role_propagation", +"citus.enable_alter_role_set_propagation", +"citus.enable_binary_protocol", +"citus.enable_change_data_capture", +"citus.enable_cluster_clock", +"citus.enable_cost_based_connection_establishment", +"citus.enable_create_role_propagation", +"citus.enable_create_type_propagation", +"citus.enable_ddl_propagation", +"citus.enable_deadlock_prevention", +"citus.enable_fast_path_router_planner", +"citus.enable_local_execution", +"citus.enable_local_reference_table_foreign_keys", +"citus.enable_manual_changes_to_shards", +"citus.enable_manual_metadata_changes_for_user", +"citus.enable_metadata_sync", +"citus.enable_non_colocated_router_query_pushdown", +"citus.enable_repartition_joins", +"citus.enable_repartitioned_insert_select", +"citus.enable_router_execution", +"citus.enable_schema_based_sharding", +"citus.enable_single_hash_repartition_joins", +"citus.enable_statistics_collection", +"citus.enable_unique_job_ids", +"citus.enable_unsafe_triggers", +"citus.enable_unsupported_feature_messages", +"citus.enable_version_checks", +"citus.enforce_foreign_key_restrictions", +"citus.enforce_object_restrictions_for_local_objects", +"citus.executor_slow_start_interval", +"citus.explain_all_tasks", +"citus.explain_analyze_sort_method", +"citus.explain_distributed_queries", +"citus.force_max_query_parallelization", +"citus.function_opens_transaction_block", +"citus.grep_remote_commands", +"citus.hide_citus_dependent_objects", +"citus.hide_shards_from_app_name_prefixes", +"citus.isolation_test_session_process_id", +"citus.isolation_test_session_remote_process_id", +"citus.limit_clause_row_fetch_count", +"citus.local_copy_flush_threshold", +"citus.local_hostname", +"citus.local_shared_pool_size", +"citus.local_table_join_policy", +"citus.log_distributed_deadlock_detection", +"citus.log_intermediate_results", +"citus.log_local_commands", +"citus.log_multi_join_order", +"citus.log_remote_commands", +"citus.logical_replication_timeout", +"citus.main_db", +"citus.max_adaptive_executor_pool_size", +"citus.max_background_task_executors", +"citus.max_background_task_executors_per_node", +"citus.max_cached_connection_lifetime", +"citus.max_cached_conns_per_worker", +"citus.max_client_connections", +"citus.max_high_priority_background_processes", +"citus.max_intermediate_result_size", +"citus.max_matview_size_to_auto_recreate", +"citus.max_rebalancer_logged_ignored_moves", +"citus.max_shared_pool_size", +"citus.max_worker_nodes_tracked", +"citus.metadata_sync_interval", +"citus.metadata_sync_mode", +"citus.metadata_sync_retry_interval", +"citus.mitmfifo", +"citus.multi_shard_modify_mode", +"citus.multi_task_query_log_level", +"citus.next_cleanup_record_id", +"citus.next_operation_id", +"citus.next_placement_id", +"citus.next_shard_id", +"citus.node_connection_timeout", +"citus.node_conninfo", +"citus.override_table_visibility", +"citus.prevent_incomplete_connection_establishment", +"citus.propagate_session_settings_for_loopback_connection", +"citus.propagate_set_commands", +"citus.rebalancer_by_disk_size_base_cost", +"citus.recover_2pc_interval", +"citus.remote_copy_flush_threshold", +"citus.remote_task_check_interval", +"citus.repartition_join_bucket_count_per_node", +"citus.replicate_reference_tables_on_activate", +"citus.replication_model", +"citus.running_under_citus_test_suite", +"citus.select_opens_transaction_block", +"citus.shard_count", +"citus.shard_replication_factor", +"citus.show_shards_for_app_name_prefixes", +"citus.skip_advisory_lock_permission_checks", +"citus.skip_constraint_validation", +"citus.skip_jsonb_validation_in_copy", +"citus.sort_returning", +"citus.stat_statements_max", +"citus.stat_statements_purge_interval", +"citus.stat_statements_track", +"citus.stat_tenants_limit", +"citus.stat_tenants_log_level", +"citus.stat_tenants_period", +"citus.stat_tenants_track", +"citus.stat_tenants_untracked_sample_rate", +"citus.subquery_pushdown", +"citus.task_assignment_policy", +"citus.task_executor_type", +"citus.use_citus_managed_tables", +"citus.use_secondary_nodes", +"citus.values_materialization_threshold", +"citus.version", +"citus.worker_min_messages", +"citus.writable_standby_coordinator", diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 16f6507df..0b476e496 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -364,6 +364,15 @@ static DistributeObjectOps Any_Rename = { .address = NULL, .markDistributed = false, }; +static DistributeObjectOps Any_SecLabel = { + .deparse = DeparseSecLabelStmt, + .qualify = NULL, + .preprocess = NULL, + .postprocess = PostprocessSecLabelStmt, + .operationType = DIST_OPS_ALTER, + .address = SecLabelStmtObjectAddress, + .markDistributed = false, +}; static DistributeObjectOps Attribute_Rename = { .deparse = DeparseRenameAttributeStmt, .qualify = QualifyRenameAttributeStmt, @@ -1991,6 +2000,11 @@ GetDistributeObjectOps(Node *node) return &Vacuum_Analyze; } + case T_SecLabelStmt: + { + return &Any_SecLabel; + } + case T_RenameStmt: { RenameStmt *stmt = castNode(RenameStmt, node); diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 208539018..6a42211a6 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -22,6 +22,7 @@ #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_db_role_setting.h" +#include "catalog/pg_shseclabel.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "nodes/makefuncs.h" @@ -65,6 +66,7 @@ static DefElem * makeDefElemBool(char *name, bool value); static List * GenerateRoleOptionsList(HeapTuple tuple); static List * GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options); static List * GenerateGrantRoleStmtsOfRole(Oid roleid); +static List * GenerateSecLabelOnRoleStmts(Oid roleid, char *rolename); static void EnsureSequentialModeForRoleDDL(void); static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, @@ -516,13 +518,14 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) { HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); + char *rolename = pstrdup(NameStr(role->rolname)); CreateRoleStmt *createRoleStmt = NULL; if (EnableCreateRolePropagation) { createRoleStmt = makeNode(CreateRoleStmt); createRoleStmt->stmt_type = ROLESTMT_ROLE; - createRoleStmt->role = pstrdup(NameStr(role->rolname)); + createRoleStmt->role = rolename; createRoleStmt->options = GenerateRoleOptionsList(roleTuple); } @@ -533,7 +536,7 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) alterRoleStmt->role = makeNode(RoleSpec); alterRoleStmt->role->roletype = ROLESPEC_CSTRING; alterRoleStmt->role->location = -1; - alterRoleStmt->role->rolename = pstrdup(NameStr(role->rolname)); + alterRoleStmt->role->rolename = rolename; alterRoleStmt->action = 1; alterRoleStmt->options = GenerateRoleOptionsList(roleTuple); } @@ -545,7 +548,7 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) { /* add a worker_create_or_alter_role command if any of them are set */ char *createOrAlterRoleQuery = CreateCreateOrAlterRoleCommand( - pstrdup(NameStr(role->rolname)), + rolename, createRoleStmt, alterRoleStmt); @@ -567,6 +570,20 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) { completeRoleList = lappend(completeRoleList, DeparseTreeNode(stmt)); } + + /* + * append SECURITY LABEL ON ROLE commands for this specific user + * When we propagate user creation, we also want to make sure that we propagate + * all the security labels it has been given. For this, we check pg_shseclabel + * for the ROLE entry corresponding to roleOid, and generate the relevant + * SecLabel stmts to be run in the new node. + */ + List *secLabelOnRoleStmts = GenerateSecLabelOnRoleStmts(roleOid, rolename); + stmt = NULL; + foreach_declared_ptr(stmt, secLabelOnRoleStmts) + { + completeRoleList = lappend(completeRoleList, DeparseTreeNode(stmt)); + } } return completeRoleList; @@ -896,6 +913,54 @@ GenerateGrantRoleStmtsOfRole(Oid roleid) } +/* + * GenerateSecLabelOnRoleStmts generates the SecLabelStmts for the role + * whose oid is roleid. + */ +static List * +GenerateSecLabelOnRoleStmts(Oid roleid, char *rolename) +{ + List *secLabelStmts = NIL; + + /* + * Note that roles are shared database objects, therefore their + * security labels are stored in pg_shseclabel instead of pg_seclabel. + */ + Relation pg_shseclabel = table_open(SharedSecLabelRelationId, AccessShareLock); + ScanKeyData skey[1]; + ScanKeyInit(&skey[0], Anum_pg_shseclabel_objoid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(roleid)); + SysScanDesc scan = systable_beginscan(pg_shseclabel, SharedSecLabelObjectIndexId, + true, NULL, 1, &skey[0]); + + HeapTuple tuple = NULL; + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + SecLabelStmt *secLabelStmt = makeNode(SecLabelStmt); + secLabelStmt->objtype = OBJECT_ROLE; + secLabelStmt->object = (Node *) makeString(pstrdup(rolename)); + + Datum datumArray[Natts_pg_shseclabel]; + bool isNullArray[Natts_pg_shseclabel]; + + heap_deform_tuple(tuple, RelationGetDescr(pg_shseclabel), datumArray, + isNullArray); + + secLabelStmt->provider = TextDatumGetCString( + datumArray[Anum_pg_shseclabel_provider - 1]); + secLabelStmt->label = TextDatumGetCString( + datumArray[Anum_pg_shseclabel_label - 1]); + + secLabelStmts = lappend(secLabelStmts, secLabelStmt); + } + + systable_endscan(scan); + table_close(pg_shseclabel, AccessShareLock); + + return secLabelStmts; +} + + /* * PreprocessCreateRoleStmt creates a worker_create_or_alter_role query for the * role that is being created. With that query we can create the role in the diff --git a/src/backend/distributed/commands/seclabel.c b/src/backend/distributed/commands/seclabel.c new file mode 100644 index 000000000..3e1847dc9 --- /dev/null +++ b/src/backend/distributed/commands/seclabel.c @@ -0,0 +1,125 @@ +/*------------------------------------------------------------------------- + * + * seclabel.c + * + * This file contains the logic of SECURITY LABEL statement propagation. + * + * Copyright (c) Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/log_utils.h" +#include "distributed/metadata/distobject.h" +#include "distributed/metadata_sync.h" + + +/* + * PostprocessSecLabelStmt prepares the commands that need to be run on all workers to assign + * security labels on distributed objects, currently supporting just Role objects. + * It also ensures that all object dependencies exist on all + * nodes for the object in the SecLabelStmt. + */ +List * +PostprocessSecLabelStmt(Node *node, const char *queryString) +{ + if (!ShouldPropagate()) + { + return NIL; + } + + SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); + + List *objectAddresses = GetObjectAddressListFromParseTree(node, false, true); + if (!IsAnyObjectDistributed(objectAddresses)) + { + return NIL; + } + + if (secLabelStmt->objtype != OBJECT_ROLE) + { + /* + * If we are not in the coordinator, we don't want to interrupt the security + * label command with notices, the user expects that from the worker node + * the command will not be propagated + */ + if (EnableUnsupportedFeatureMessages && IsCoordinator()) + { + ereport(NOTICE, (errmsg("not propagating SECURITY LABEL commands whose " + "object type is not role"), + errhint("Connect to worker nodes directly to manually " + "run the same SECURITY LABEL command."))); + } + return NIL; + } + + if (!EnableCreateRolePropagation) + { + return NIL; + } + + EnsureCoordinator(); + EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); + + const char *sql = DeparseTreeNode((Node *) secLabelStmt); + + List *commandList = list_make3(DISABLE_DDL_PROPAGATION, + (void *) sql, + ENABLE_DDL_PROPAGATION); + + return NodeDDLTaskList(NON_COORDINATOR_NODES, commandList); +} + + +/* + * SecLabelStmtObjectAddress returns the object address of the object on + * which this statement operates (secLabelStmt->object). Note that it has no limitation + * on the object type being OBJECT_ROLE. This is intentionally implemented like this + * since it is fairly simple to implement and we might extend SECURITY LABEL propagation + * in the future to include more object types. + */ +List * +SecLabelStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) +{ + SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); + + Relation rel = NULL; + ObjectAddress address = get_object_address(secLabelStmt->objtype, + secLabelStmt->object, &rel, + AccessShareLock, missing_ok); + if (rel != NULL) + { + relation_close(rel, AccessShareLock); + } + + ObjectAddress *addressPtr = palloc0(sizeof(ObjectAddress)); + *addressPtr = address; + return list_make1(addressPtr); +} + + +/* + * citus_test_object_relabel is a dummy function for check_object_relabel_type hook. + * It is meant to be used in tests combined with citus_test_register_label_provider + */ +void +citus_test_object_relabel(const ObjectAddress *object, const char *seclabel) +{ + if (seclabel == NULL || + strcmp(seclabel, "citus_unclassified") == 0 || + strcmp(seclabel, "citus_classified") == 0 || + strcmp(seclabel, "citus '!unclassified") == 0) + { + return; + } + + ereport(ERROR, + (errcode(ERRCODE_INVALID_NAME), + errmsg("'%s' is not a valid security label for Citus tests.", seclabel))); +} diff --git a/src/backend/distributed/deparser/deparse_seclabel_stmts.c b/src/backend/distributed/deparser/deparse_seclabel_stmts.c new file mode 100644 index 000000000..ffe775b76 --- /dev/null +++ b/src/backend/distributed/deparser/deparse_seclabel_stmts.c @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------- + * + * deparse_seclabel_stmts.c + * All routines to deparse SECURITY LABEL statements. + * + * Copyright (c), Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#include "distributed/deparser.h" + +static void AppendSecLabelStmt(StringInfo buf, SecLabelStmt *stmt); + +/* + * DeparseSecLabelStmt builds and returns a string representing of the + * SecLabelStmt for application on a remote server. + */ +char * +DeparseSecLabelStmt(Node *node) +{ + SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); + StringInfoData buf = { 0 }; + initStringInfo(&buf); + + AppendSecLabelStmt(&buf, secLabelStmt); + + return buf.data; +} + + +/* + * AppendSecLabelStmt generates the string representation of the + * SecLabelStmt and appends it to the buffer. + */ +static void +AppendSecLabelStmt(StringInfo buf, SecLabelStmt *stmt) +{ + appendStringInfoString(buf, "SECURITY LABEL "); + + if (stmt->provider != NULL) + { + appendStringInfo(buf, "FOR %s ", quote_identifier(stmt->provider)); + } + + appendStringInfoString(buf, "ON "); + + switch (stmt->objtype) + { + case OBJECT_ROLE: + { + appendStringInfo(buf, "ROLE %s ", quote_identifier(strVal(stmt->object))); + break; + } + + /* normally, we shouldn't reach this */ + default: + { + ereport(ERROR, (errmsg("unsupported security label statement for" + " deparsing"))); + } + } + + appendStringInfoString(buf, "IS "); + + if (stmt->label != NULL) + { + appendStringInfo(buf, "%s", quote_literal_cstr(stmt->label)); + } + else + { + appendStringInfoString(buf, "NULL"); + } +} diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index a4c7364be..074f1bed0 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -319,7 +319,7 @@ PG_FUNCTION_INFO_V1(citus_rebalance_start); PG_FUNCTION_INFO_V1(citus_rebalance_stop); PG_FUNCTION_INFO_V1(citus_rebalance_wait); -bool RunningUnderIsolationTest = false; +bool RunningUnderCitusTestSuite = false; int MaxRebalancerLoggedIgnoredMoves = 5; int RebalancerByDiskSizeBaseCost = 100 * 1024 * 1024; bool PropagateSessionSettingsForLoopbackConnection = false; diff --git a/src/backend/distributed/replication/multi_logical_replication.c b/src/backend/distributed/replication/multi_logical_replication.c index 2dbf8daa8..9f8bb6a54 100644 --- a/src/backend/distributed/replication/multi_logical_replication.c +++ b/src/backend/distributed/replication/multi_logical_replication.c @@ -1143,7 +1143,7 @@ ConflictWithIsolationTestingBeforeCopy(void) const bool sessionLock = false; const bool dontWait = false; - if (RunningUnderIsolationTest) + if (RunningUnderCitusTestSuite) { SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, SHARD_MOVE_ADVISORY_LOCK_SECOND_KEY, @@ -1177,7 +1177,7 @@ ConflictWithIsolationTestingAfterCopy(void) const bool sessionLock = false; const bool dontWait = false; - if (RunningUnderIsolationTest) + if (RunningUnderCitusTestSuite) { SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, SHARD_MOVE_ADVISORY_LOCK_FIRST_KEY, diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 21ae6b4b5..d7fa094ed 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -27,6 +27,7 @@ #include "catalog/pg_extension.h" #include "commands/explain.h" #include "commands/extension.h" +#include "commands/seclabel.h" #include "common/string.h" #include "executor/executor.h" #include "libpq/auth.h" @@ -572,6 +573,16 @@ _PG_init(void) INIT_COLUMNAR_SYMBOL(PGFunction, columnar_storage_info); INIT_COLUMNAR_SYMBOL(PGFunction, columnar_store_memory_stats); INIT_COLUMNAR_SYMBOL(PGFunction, test_columnar_storage_write_new_page); + + /* + * This part is only for SECURITY LABEL tests + * mimicking what an actual security label provider would do + */ + if (RunningUnderCitusTestSuite) + { + register_label_provider("citus '!tests_label_provider", + citus_test_object_relabel); + } } @@ -2293,13 +2304,14 @@ RegisterCitusConfigVariables(void) WarnIfReplicationModelIsSet, NULL, NULL); DefineCustomBoolVariable( - "citus.running_under_isolation_test", + "citus.running_under_citus_test_suite", gettext_noop( "Only useful for testing purposes, when set to true, Citus does some " - "tricks to implement useful isolation tests with rebalancing. Should " + "tricks to implement useful isolation tests with rebalancing. It also " + "registers a dummy label provider for SECURITY LABEL tests. Should " "never be set to true on production systems "), gettext_noop("for details of the tricks implemented, refer to the source code"), - &RunningUnderIsolationTest, + &RunningUnderCitusTestSuite, false, PGC_SUSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 32bb38100..caf40ff95 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -510,6 +510,11 @@ extern List * AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, extern List * AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +/* seclabel.c - forward declarations*/ +extern List * PostprocessSecLabelStmt(Node *node, const char *queryString); +extern List * SecLabelStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern void citus_test_object_relabel(const ObjectAddress *object, const char *seclabel); + /* sequence.c - forward declarations */ extern List * PreprocessAlterSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 04a96e4be..8c4c58c88 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -259,6 +259,9 @@ extern void QualifyRenameTextSearchDictionaryStmt(Node *node); extern void QualifyTextSearchConfigurationCommentStmt(Node *node); extern void QualifyTextSearchDictionaryCommentStmt(Node *node); +/* forward declarations for deparse_seclabel_stmts.c */ +extern char * DeparseSecLabelStmt(Node *node); + /* forward declarations for deparse_sequence_stmts.c */ extern char * DeparseDropSequenceStmt(Node *node); extern char * DeparseRenameSequenceStmt(Node *node); diff --git a/src/include/distributed/shard_rebalancer.h b/src/include/distributed/shard_rebalancer.h index 8e47ac1e5..79414eb3c 100644 --- a/src/include/distributed/shard_rebalancer.h +++ b/src/include/distributed/shard_rebalancer.h @@ -191,7 +191,7 @@ typedef struct RebalancePlanFunctions extern char *VariablesToBePassedToNewConnections; extern int MaxRebalancerLoggedIgnoredMoves; extern int RebalancerByDiskSizeBaseCost; -extern bool RunningUnderIsolationTest; +extern bool RunningUnderCitusTestSuite; extern bool PropagateSessionSettingsForLoopbackConnection; extern int MaxBackgroundTaskExecutorsPerNode; diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index f3a0f2bba..d97c0579c 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -631,3 +631,33 @@ begin end loop; end; $$; +-- Returns pg_seclabels entries from all nodes in the cluster for which +-- the object name is the input. +CREATE OR REPLACE FUNCTION get_citus_tests_label_provider_labels(object_name text, + master_port INTEGER DEFAULT 57636, + worker_1_port INTEGER DEFAULT 57637, + worker_2_port INTEGER DEFAULT 57638) +RETURNS TABLE ( + node_type text, + result text +) +AS $func$ +DECLARE + pg_seclabels_cmd TEXT := 'SELECT to_jsonb(q.*) FROM (' || + 'SELECT provider, objtype, label FROM pg_seclabels ' || + 'WHERE objname = ''' || object_name || ''') q'; +BEGIN + RETURN QUERY + SELECT + CASE + WHEN nodeport = master_port THEN 'coordinator' + WHEN nodeport = worker_1_port THEN 'worker_1' + WHEN nodeport = worker_2_port THEN 'worker_2' + ELSE 'unexpected_node' + END AS node_type, + a.result + FROM run_command_on_all_nodes(pg_seclabels_cmd) a + JOIN pg_dist_node USING (nodeid) + ORDER BY node_type; +END; +$func$ LANGUAGE plpgsql; diff --git a/src/test/regress/expected/seclabel.out b/src/test/regress/expected/seclabel.out new file mode 100644 index 000000000..c1a799451 --- /dev/null +++ b/src/test/regress/expected/seclabel.out @@ -0,0 +1,173 @@ +-- +-- SECLABEL +-- +-- Test suite for SECURITY LABEL ON ROLE statements +-- +-- first we remove one of the worker nodes to be able to test +-- citus_add_node later +SELECT citus_remove_node('localhost', :worker_2_port); + citus_remove_node +--------------------------------------------------------------------- + +(1 row) + +-- create two roles, one with characters that need escaping +CREATE ROLE user1; +CREATE ROLE "user 2"; +-- check an invalid label for our current dummy hook citus_test_object_relabel +SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'invalid_label'; +ERROR: 'invalid_label' is not a valid security label for Citus tests. +-- if we disable metadata_sync, the command will not be propagated +SET citus.enable_metadata_sync TO off; +SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'citus_unclassified'; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_1 | +(2 rows) + +RESET citus.enable_metadata_sync; +-- check that we only support propagating for roles +SET citus.shard_replication_factor to 1; +-- distributed table +CREATE TABLE a (a int); +SELECT create_distributed_table('a', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- distributed view +CREATE VIEW v_dist AS SELECT * FROM a; +-- distributed function +CREATE FUNCTION notice(text) RETURNS void LANGUAGE plpgsql AS $$ + BEGIN RAISE NOTICE '%', $1; END; $$; +SECURITY LABEL ON TABLE a IS 'citus_classified'; +NOTICE: not propagating SECURITY LABEL commands whose object type is not role +HINT: Connect to worker nodes directly to manually run the same SECURITY LABEL command. +SECURITY LABEL ON FUNCTION notice IS 'citus_unclassified'; +NOTICE: not propagating SECURITY LABEL commands whose object type is not role +HINT: Connect to worker nodes directly to manually run the same SECURITY LABEL command. +SECURITY LABEL ON VIEW v_dist IS 'citus_classified'; +NOTICE: not propagating SECURITY LABEL commands whose object type is not role +HINT: Connect to worker nodes directly to manually run the same SECURITY LABEL command. +SELECT node_type, result FROM get_citus_tests_label_provider_labels('a') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_classified", "objtype": "table", "provider": "citus '!tests_label_provider"} + worker_1 | +(2 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('notice(text)') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_unclassified", "objtype": "function", "provider": "citus '!tests_label_provider"} + worker_1 | +(2 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('v_dist') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_classified", "objtype": "view", "provider": "citus '!tests_label_provider"} + worker_1 | +(2 rows) + +\c - - - :worker_1_port +SECURITY LABEL ON TABLE a IS 'citus_classified'; +SECURITY LABEL ON FUNCTION notice IS 'citus_unclassified'; +SECURITY LABEL ON VIEW v_dist IS 'citus_classified'; +\c - - - :master_port +SELECT node_type, result FROM get_citus_tests_label_provider_labels('a') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_classified", "objtype": "table", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus_classified", "objtype": "table", "provider": "citus '!tests_label_provider"} +(2 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('notice(text)') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_unclassified", "objtype": "function", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus_unclassified", "objtype": "function", "provider": "citus '!tests_label_provider"} +(2 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('v_dist') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_classified", "objtype": "view", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus_classified", "objtype": "view", "provider": "citus '!tests_label_provider"} +(2 rows) + +DROP TABLE a CASCADE; +NOTICE: drop cascades to view v_dist +DROP FUNCTION notice; +-- test that SECURITY LABEL statement is actually propagated for ROLES +SET citus.log_remote_commands TO on; +SET citus.grep_remote_commands = '%SECURITY LABEL%'; +-- we have exactly one provider loaded, so we may not include the provider in the command +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE user1 IS 'citus_classified'; +NOTICE: issuing SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'citus_classified' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SECURITY LABEL ON ROLE user1 IS NULL; +NOTICE: issuing SECURITY LABEL ON ROLE user1 IS NULL +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SECURITY LABEL ON ROLE user1 IS 'citus_unclassified'; +NOTICE: issuing SECURITY LABEL ON ROLE user1 IS 'citus_unclassified' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE "user 2" IS 'citus ''!unclassified'; +NOTICE: issuing SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE "user 2" IS 'citus ''!unclassified' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +\c - - - :worker_1_port +-- command not allowed from worker node +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE user1 IS 'citus ''!unclassified'; +ERROR: operation is not allowed on this node +HINT: Connect to the coordinator and run it again. +\c - - - :master_port +RESET citus.log_remote_commands; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} +(2 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('"user 2"') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus '!unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus '!unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} +(2 rows) + +-- add a new node and check that it also propagates the SECURITY LABEL statement to the new node +SET citus.log_remote_commands TO on; +SET citus.grep_remote_commands = '%SECURITY LABEL%'; +SELECT 1 FROM citus_add_node('localhost', :worker_2_port); +NOTICE: issuing SELECT worker_create_or_alter_role('user1', 'CREATE ROLE user1 NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN NOREPLICATION NOBYPASSRLS CONNECTION LIMIT -1 PASSWORD NULL', 'ALTER ROLE user1 NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN NOREPLICATION NOBYPASSRLS CONNECTION LIMIT -1 PASSWORD NULL');SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'citus_unclassified' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_create_or_alter_role('user 2', 'CREATE ROLE "user 2" NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN NOREPLICATION NOBYPASSRLS CONNECTION LIMIT -1 PASSWORD NULL', 'ALTER ROLE "user 2" NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOLOGIN NOREPLICATION NOBYPASSRLS CONNECTION LIMIT -1 PASSWORD NULL');SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE "user 2" IS 'citus ''!unclassified' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_2 | {"label": "citus_unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} +(3 rows) + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('"user 2"') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator | {"label": "citus '!unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_1 | {"label": "citus '!unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} + worker_2 | {"label": "citus '!unclassified", "objtype": "role", "provider": "citus '!tests_label_provider"} +(3 rows) + +-- cleanup +RESET citus.log_remote_commands; +DROP ROLE user1, "user 2"; diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 4dead5be3..c996b5c02 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -31,6 +31,7 @@ test: propagate_extension_commands test: escape_extension_name test: ref_citus_local_fkeys test: alter_database_owner +test: seclabel test: distributed_triggers test: create_single_shard_table # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index adcb431e1..66016f7c6 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -512,6 +512,12 @@ if($vanillatest) # we disable some restrictions for local objects like local views to not break postgres vanilla test behaviour. push(@pgOptions, "citus.enforce_object_restrictions_for_local_objects=false"); } +else +{ + # We currently need this config for isolation tests and security label tests + # this option loads a security label provider, which we don't want in vanilla tests + push(@pgOptions, "citus.running_under_citus_test_suite=true"); +} if ($useMitmproxy) { @@ -562,7 +568,6 @@ if($isolationtester) push(@pgOptions, "citus.metadata_sync_interval=1000"); push(@pgOptions, "citus.metadata_sync_retry_interval=100"); push(@pgOptions, "client_min_messages='warning'"); # pg12 introduced notice showing during isolation tests - push(@pgOptions, "citus.running_under_isolation_test=true"); # Disable all features of the maintenance daemon. Otherwise queries might # randomly show temporarily as "waiting..." because they are waiting for the diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 57abf15a3..17fcf58bb 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -660,3 +660,33 @@ begin end loop; end; $$; +-- Returns pg_seclabels entries from all nodes in the cluster for which +-- the object name is the input. +CREATE OR REPLACE FUNCTION get_citus_tests_label_provider_labels(object_name text, + master_port INTEGER DEFAULT 57636, + worker_1_port INTEGER DEFAULT 57637, + worker_2_port INTEGER DEFAULT 57638) +RETURNS TABLE ( + node_type text, + result text +) +AS $func$ +DECLARE + pg_seclabels_cmd TEXT := 'SELECT to_jsonb(q.*) FROM (' || + 'SELECT provider, objtype, label FROM pg_seclabels ' || + 'WHERE objname = ''' || object_name || ''') q'; +BEGIN + RETURN QUERY + SELECT + CASE + WHEN nodeport = master_port THEN 'coordinator' + WHEN nodeport = worker_1_port THEN 'worker_1' + WHEN nodeport = worker_2_port THEN 'worker_2' + ELSE 'unexpected_node' + END AS node_type, + a.result + FROM run_command_on_all_nodes(pg_seclabels_cmd) a + JOIN pg_dist_node USING (nodeid) + ORDER BY node_type; +END; +$func$ LANGUAGE plpgsql; diff --git a/src/test/regress/sql/seclabel.sql b/src/test/regress/sql/seclabel.sql new file mode 100644 index 000000000..e523fc1da --- /dev/null +++ b/src/test/regress/sql/seclabel.sql @@ -0,0 +1,87 @@ +-- +-- SECLABEL +-- +-- Test suite for SECURITY LABEL ON ROLE statements +-- + +-- first we remove one of the worker nodes to be able to test +-- citus_add_node later +SELECT citus_remove_node('localhost', :worker_2_port); + +-- create two roles, one with characters that need escaping +CREATE ROLE user1; +CREATE ROLE "user 2"; + +-- check an invalid label for our current dummy hook citus_test_object_relabel +SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'invalid_label'; + +-- if we disable metadata_sync, the command will not be propagated +SET citus.enable_metadata_sync TO off; +SECURITY LABEL FOR "citus '!tests_label_provider" ON ROLE user1 IS 'citus_unclassified'; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; + +RESET citus.enable_metadata_sync; + +-- check that we only support propagating for roles +SET citus.shard_replication_factor to 1; +-- distributed table +CREATE TABLE a (a int); +SELECT create_distributed_table('a', 'a'); +-- distributed view +CREATE VIEW v_dist AS SELECT * FROM a; +-- distributed function +CREATE FUNCTION notice(text) RETURNS void LANGUAGE plpgsql AS $$ + BEGIN RAISE NOTICE '%', $1; END; $$; + +SECURITY LABEL ON TABLE a IS 'citus_classified'; +SECURITY LABEL ON FUNCTION notice IS 'citus_unclassified'; +SECURITY LABEL ON VIEW v_dist IS 'citus_classified'; + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('a') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('notice(text)') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('v_dist') ORDER BY node_type; + +\c - - - :worker_1_port +SECURITY LABEL ON TABLE a IS 'citus_classified'; +SECURITY LABEL ON FUNCTION notice IS 'citus_unclassified'; +SECURITY LABEL ON VIEW v_dist IS 'citus_classified'; + +\c - - - :master_port +SELECT node_type, result FROM get_citus_tests_label_provider_labels('a') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('notice(text)') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('v_dist') ORDER BY node_type; + +DROP TABLE a CASCADE; +DROP FUNCTION notice; + +-- test that SECURITY LABEL statement is actually propagated for ROLES +SET citus.log_remote_commands TO on; +SET citus.grep_remote_commands = '%SECURITY LABEL%'; + +-- we have exactly one provider loaded, so we may not include the provider in the command +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE user1 IS 'citus_classified'; +SECURITY LABEL ON ROLE user1 IS NULL; +SECURITY LABEL ON ROLE user1 IS 'citus_unclassified'; +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE "user 2" IS 'citus ''!unclassified'; + +\c - - - :worker_1_port +-- command not allowed from worker node +SECURITY LABEL for "citus '!tests_label_provider" ON ROLE user1 IS 'citus ''!unclassified'; + +\c - - - :master_port +RESET citus.log_remote_commands; + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('"user 2"') ORDER BY node_type; + +-- add a new node and check that it also propagates the SECURITY LABEL statement to the new node +SET citus.log_remote_commands TO on; +SET citus.grep_remote_commands = '%SECURITY LABEL%'; +SELECT 1 FROM citus_add_node('localhost', :worker_2_port); + +SELECT node_type, result FROM get_citus_tests_label_provider_labels('user1') ORDER BY node_type; +SELECT node_type, result FROM get_citus_tests_label_provider_labels('"user 2"') ORDER BY node_type; + +-- cleanup +RESET citus.log_remote_commands; +DROP ROLE user1, "user 2"; From fa8e8676625c6ee98c3e77f31f66fd5880ae5970 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:35:11 +0300 Subject: [PATCH 132/155] Bump to latest PG minors 17.2, 16.6, 15.10, 14.15 (#7843) Similar to https://github.com/citusdata/citus/commit/5ef2cd67edef2d05f69e3d0f8c9795b5d538e3fa, we use the commit sha of a local build of the images, pushed. --- .devcontainer/Dockerfile | 10 +++++----- .github/workflows/build_and_test.yml | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 23a94f1d6..187df5e42 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -68,7 +68,7 @@ USER citus # build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions FROM base AS pg14 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.14 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.15 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -80,7 +80,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg15 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.9 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.10 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -92,7 +92,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg16 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.5 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.6 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -104,7 +104,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg17 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 17.1 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 17.2 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -223,7 +223,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 17.1 +RUN pgenv switch 17.2 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 50f25fe3c..3d9b57c37 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -26,13 +26,13 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.18" - sql_snapshot_pg_version: "17.1" - image_suffix: "-v84c0cf8" - pg14_version: '{ "major": "14", "full": "14.14" }' - pg15_version: '{ "major": "15", "full": "15.9" }' - pg16_version: '{ "major": "16", "full": "16.5" }' - pg17_version: '{ "major": "17", "full": "17.1" }' - upgrade_pg_versions: "14.14-15.9-16.5-17.1" + sql_snapshot_pg_version: "17.2" + image_suffix: "-v889e4c1" + pg14_version: '{ "major": "14", "full": "14.15" }' + pg15_version: '{ "major": "15", "full": "15.10" }' + pg16_version: '{ "major": "16", "full": "16.6" }' + pg17_version: '{ "major": "17", "full": "17.2" }' + upgrade_pg_versions: "14.15-15.10-16.6-17.2" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters From c2bc7aca4ae0f9683f8ac3995b07b9d627a4beff Mon Sep 17 00:00:00 2001 From: Colm Date: Mon, 20 Jan 2025 22:00:33 +0000 Subject: [PATCH 133/155] Update tdigest_aggregate_support output for PG15+ (#7849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regress test tdigest_aggregate_support has been failing since at least Citus 12.0, when tdigest extension is installed in Postgres. This appears to be because of an omission by commit 03832f3 and a change in the implementation of Postgres random() function (pg commit [d4f109e4a](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=d4f109e4a)). To reproduce the test diff: - Checkout [tdigest ](https://github.com/tvondra/tdigest)and run `make; make install` - In citus regress directory run `make check-multi` or `./citus_tests/run_test.py tdigest_aggregate_support` There are two parts to this commit: 1. Revert `Output: xxxxx` in EXPLAIN VERBOSE. Citus commit fe4ac51 normalized EXPLAIN VERBOSE output because of a change between pg12 and pg13. When pg12 support was no longer required, the rule was removed from normalize.sed and `Output: xxxx` was reverted in the impacted regress output files (03832f3), but `tdigest_aggregate_support` was omitted. 2. Adjust the query results; the tdigest_aggregate_support test file has a comment _verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges_ but the result values in this commit are consistent across citus 12.0 (pg 15), citus 12.1 (pg 16) and citus 13.0 (pg 17), or since the Postgres changed their [implementation of random](https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=d4f109e4a), so proposing to go with these results. --- .../expected/tdigest_aggregate_support.out | 206 +++--- .../expected/tdigest_aggregate_support_1.out | 651 ++++++++++++++++++ 2 files changed, 754 insertions(+), 103 deletions(-) create mode 100644 src/test/regress/expected/tdigest_aggregate_support_1.out diff --git a/src/test/regress/expected/tdigest_aggregate_support.out b/src/test/regress/expected/tdigest_aggregate_support.out index 7ad937ce5..5227d2426 100644 --- a/src/test/regress/expected/tdigest_aggregate_support.out +++ b/src/test/regress/expected/tdigest_aggregate_support.out @@ -42,18 +42,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest(remote_scan.tdigest) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(latency, 100) -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest(value, compression) @@ -64,17 +64,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest(latency, 100) Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest(value, compression) @@ -85,20 +85,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: xxxxxx + Output: remote_scan.b, tdigest(remote_scan.tdigest) Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.b, remote_scan.tdigest Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: b, tdigest(latency, 100) Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantile) @@ -108,18 +108,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(latency, 100) -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantile) @@ -130,17 +130,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(latency, 100, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile(latency, 100, '0.99'::double precision) Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantile) @@ -151,20 +151,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: xxxxxx + Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.b, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: b, tdigest(latency, 100) Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -174,18 +174,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(latency, 100) -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -196,17 +196,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -217,20 +217,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: xxxxxx + Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.b, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: b, tdigest(latency, 100) Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -240,18 +240,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(latency, 100) -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -262,17 +262,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(latency, 100, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile_of(latency, 100, '9000'::double precision) Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -283,20 +283,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: xxxxxx + Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.b, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: b, tdigest(latency, 100) Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -306,18 +306,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(latency, 100) -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -328,17 +328,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -349,51 +349,51 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: xxxxxx + Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.b, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: b, tdigest(latency, 100) Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: xxxxxx + Output: a, b, latency (15 rows) -- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges SELECT tdigest(latency, 100) FROM latencies; - tdigest + tdigest --------------------------------------------------------------------- - flags 0 count 10000 compression 100 centroids 46 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (12.707263, 2) (25.302479, 3) (43.435063, 4) (77.987860, 5) (269.478664, 10) (509.417419, 13) (1227.158879, 22) (3408.256171, 35) (7772.721988, 55) (13840.275516, 65) (32937.127607, 108) (64476.403332, 148) (118260.230644, 199) (239584.293240, 292) (562119.836766, 463) (944722.686313, 547) (1751089.620493, 749) (3751264.745959, 1128) (5877270.108576, 1300) (6224557.402567, 1104) (5804999.258033, 874) (5632316.697114, 755) (4648651.050740, 573) (3460055.227950, 402) (2820271.404686, 314) (2676501.012955, 288) (1649845.166017, 173) (1269335.942008, 131) (813964.853243, 83) (484144.878702, 49) (337179.763016, 34) (198775.241901, 20) (149353.499704, 15) (109688.319223, 11) (79855.926155, 8) (49937.731689, 5) (29971.046175, 3) (19982.538737, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1) + flags 1 count 10000 compression 100 centroids 46 (2.846051, 1) (3.323773, 1) (4.406495, 1) (4.532352, 1) (4.993616, 2) (7.673358, 3) (13.555084, 5) (18.776503, 7) (27.990526, 11) (37.903465, 17) (56.272069, 21) (91.011574, 34) (127.790676, 51) (190.655158, 70) (275.723291, 94) (407.151014, 135) (584.186017, 219) (827.310117, 287) (1121.971646, 345) (1605.113973, 609) (2278.067230, 751) (3126.852770, 1033) (4149.398030, 991) (5374.336553, 1354) (6470.439272, 939) (7319.715295, 777) (8095.598975, 715) (8667.524977, 456) (9077.609863, 374) (9385.068110, 203) (9571.304536, 150) (9702.936696, 118) (9806.254527, 75) (9873.753103, 50) (9918.059273, 33) (9945.081993, 22) (9962.407748, 16) (9974.769012, 9) (9979.796549, 6) (9984.017888, 5) (9985.809833, 3) (9989.863888, 2) (9995.910553, 1) (9995.979459, 1) (9997.355013, 1) (9997.761058, 1) (1 row) SELECT tdigest_percentile(latency, 100, 0.99) FROM latencies; tdigest_percentile --------------------------------------------------------------------- - 9904.28342426494 + 9900.44356712993 (1 row) SELECT tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) FROM latencies; tdigest_percentile --------------------------------------------------------------------- - {9904.28342426494,9485.49009399385} + {9900.44356712993,9484.78109695984} (1 row) SELECT tdigest_percentile_of(latency, 100, 9000) FROM latencies; tdigest_percentile_of --------------------------------------------------------------------- - 0.903462047211138 + 0.903845993581303 (1 row) SELECT tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) FROM latencies; - tdigest_percentile_of + tdigest_percentile_of --------------------------------------------------------------------- - {0.903462047211138,0.95137481812975} + {0.903845993581303,0.951492325994142} (1 row) CREATE TABLE latencies_rollup (a int, tdigest tdigest); @@ -413,18 +413,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest(remote_scan.tdigest) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(tdigest) -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest(tdigest) @@ -435,17 +435,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest(tdigest) Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(tdigest, quantile) @@ -455,18 +455,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(tdigest) -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(tdigest, quantile) @@ -477,17 +477,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(tdigest, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile(tdigest, '0.99'::double precision) Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -497,18 +497,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(tdigest) -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -519,17 +519,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -539,18 +539,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(tdigest) -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -561,17 +561,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(tdigest, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile_of(tdigest, '9000'::double precision) Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -581,18 +581,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: xxxxxx + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) -> Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: xxxxxx + Output: tdigest(tdigest) -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -603,48 +603,48 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: xxxxxx + Output: remote_scan.a, remote_scan.tdigest_percentile_of Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: xxxxxx + Output: a, tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: xxxxxx + Output: a, tdigest (12 rows) -- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges SELECT tdigest(tdigest) FROM latencies_rollup; - tdigest + tdigest --------------------------------------------------------------------- - flags 0 count 10000 compression 100 centroids 47 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (12.707263, 2) (25.302479, 3) (43.435063, 4) (77.987860, 5) (241.681030, 9) (402.696604, 11) (999.675875, 20) (2310.848640, 27) (4374.387978, 37) (9722.896547, 56) (21713.805492, 87) (39735.065966, 112) (87335.860853, 177) (182744.906162, 262) (336766.886786, 338) (661263.339724, 464) (1228663.222377, 623) (2146097.038498, 805) (2854487.701653, 827) (5292830.156590, 1195) (6168185.834602, 1104) (6399734.303813, 966) (5778088.854724, 773) (5213381.984997, 637) (3763042.148296, 431) (3036786.646485, 333) (1948238.134602, 207) (1456568.605821, 152) (999888.715345, 103) (715935.892988, 73) (543464.906535, 55) (327339.982973, 33) (198853.838033, 20) (159362.743852, 16) (79807.827301, 8) (69877.414438, 7) (49937.731689, 5) (29971.046175, 3) (19982.538737, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1) + flags 1 count 10000 compression 100 centroids 47 (2.846051, 1) (3.323773, 1) (4.406495, 1) (4.532352, 1) (4.993616, 2) (6.530962, 2) (11.814064, 4) (16.758727, 6) (24.422807, 7) (29.634013, 9) (38.230799, 17) (63.206944, 26) (93.000466, 33) (133.739050, 52) (203.937122, 73) (296.554766, 104) (433.471165, 156) (621.440087, 231) (848.382844, 249) (1169.617895, 397) (1644.617827, 549) (2300.974970, 785) (3167.406918, 921) (4133.429028, 1070) (5250.107199, 1221) (6441.973764, 1145) (7456.776221, 841) (8182.469509, 600) (8686.514890, 423) (9061.061505, 310) (9331.300632, 234) (9538.562658, 162) (9684.285952, 114) (9777.125865, 75) (9847.735791, 61) (9903.652590, 39) (9936.297173, 28) (9959.245853, 17) (9971.565967, 11) (9979.422819, 8) (9984.033648, 5) (9985.809833, 3) (9989.863888, 2) (9995.910553, 1) (9995.979459, 1) (9997.355013, 1) (9997.761058, 1) (1 row) SELECT tdigest_percentile(tdigest, 0.99) FROM latencies_rollup; tdigest_percentile --------------------------------------------------------------------- - 9903.76070790358 + 9900.8567500684 (1 row) SELECT tdigest_percentile(tdigest, ARRAY[0.99, 0.95]) FROM latencies_rollup; tdigest_percentile --------------------------------------------------------------------- - {9903.76070790358,9492.7106302226} + {9900.8567500684,9484.13020701958} (1 row) SELECT tdigest_percentile_of(tdigest, 9000) FROM latencies_rollup; tdigest_percentile_of --------------------------------------------------------------------- - 0.902852659582396 + 0.902275031655923 (1 row) SELECT tdigest_percentile_of(tdigest, ARRAY[9000, 9500]) FROM latencies_rollup; tdigest_percentile_of --------------------------------------------------------------------- - {0.902852659582396,0.950865574659141} + {0.902275031655923,0.951516061128103} (1 row) SET client_min_messages TO WARNING; -- suppress cascade messages diff --git a/src/test/regress/expected/tdigest_aggregate_support_1.out b/src/test/regress/expected/tdigest_aggregate_support_1.out new file mode 100644 index 000000000..8758f87c9 --- /dev/null +++ b/src/test/regress/expected/tdigest_aggregate_support_1.out @@ -0,0 +1,651 @@ +-- +-- TDIGEST_AGGREGATE_SUPPORT +-- test the integration of github.com/tvondra/tdigest aggregates into the citus planner +-- for push down parts of the aggregate to use parallelized execution and reduced data +-- transfer sizes for aggregates not grouped by the distribution column +-- +SET citus.next_shard_id TO 20070000; +CREATE SCHEMA tdigest_aggregate_support; +SET search_path TO tdigest_aggregate_support, public; +-- create the tdigest extension when installed +SELECT CASE WHEN COUNT(*) > 0 + THEN 'CREATE EXTENSION tdigest WITH SCHEMA public' + ELSE 'SELECT false AS tdigest_present' END +AS create_cmd FROM pg_available_extensions() +WHERE name = 'tdigest' +\gset +:create_cmd; +SET citus.shard_count TO 4; +SET citus.coordinator_aggregation_strategy TO 'disabled'; -- prevent aggregate execution when the aggregate can't be pushed down +CREATE TABLE latencies (a int, b int, latency double precision); +SELECT create_distributed_table('latencies', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT setseed(0.42); -- make the random data inserted deterministic + setseed +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO latencies +SELECT (random()*20)::int AS a, + (random()*20)::int AS b, + random()*10000.0 AS latency +FROM generate_series(1, 10000); +-- explain no grouping to verify partially pushed down for tdigest(value, compression) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest(latency, 100) +FROM latencies; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest(remote_scan.tdigest) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(latency, 100) + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest(value, compression) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest(latency, 100) +FROM latencies +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest(latency, 100) + Group Key: latencies.a + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(12 rows) + +-- explain grouping by non-distribution column is partially pushed down for tdigest(value, compression) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT b, tdigest(latency, 100) +FROM latencies +GROUP BY b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.b, tdigest(remote_scan.tdigest) + Group Key: remote_scan.b + -> Custom Scan (Citus Adaptive) + Output: remote_scan.b, remote_scan.tdigest + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT b, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: b, tdigest(latency, 100) + Group Key: latencies.b + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(15 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantile) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile(latency, 100, 0.99) +FROM latencies; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(latency, 100) + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantile) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile(latency, 100, 0.99) +FROM latencies +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile(latency, 100, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile(latency, 100, '0.99'::double precision) + Group Key: latencies.a + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(12 rows) + +-- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantile) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT b, tdigest_percentile(latency, 100, 0.99) +FROM latencies +GROUP BY b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + Group Key: remote_scan.b + -> Custom Scan (Citus Adaptive) + Output: remote_scan.b, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: b, tdigest(latency, 100) + Group Key: latencies.b + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(15 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) +FROM latencies; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(latency, 100) + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) +FROM latencies +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) + Group Key: latencies.a + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(12 rows) + +-- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantiles[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT b, tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) +FROM latencies +GROUP BY b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + Group Key: remote_scan.b + -> Custom Scan (Citus Adaptive) + Output: remote_scan.b, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: b, tdigest(latency, 100) + Group Key: latencies.b + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(15 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile_of(latency, 100, 9000) +FROM latencies; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(latency, 100) + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile_of(latency, 100, 9000) +FROM latencies +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile_of(latency, 100, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile_of(latency, 100, '9000'::double precision) + Group Key: latencies.a + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(12 rows) + +-- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT b, tdigest_percentile_of(latency, 100, 9000) +FROM latencies +GROUP BY b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + Group Key: remote_scan.b + -> Custom Scan (Citus Adaptive) + Output: remote_scan.b, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: b, tdigest(latency, 100) + Group Key: latencies.b + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(15 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) +FROM latencies; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(latency, 100) + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) +FROM latencies +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) + Group Key: latencies.a + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(12 rows) + +-- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT b, tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) +FROM latencies +GROUP BY b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + Group Key: remote_scan.b + -> Custom Scan (Citus Adaptive) + Output: remote_scan.b, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: b, tdigest(latency, 100) + Group Key: latencies.b + -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies + Output: a, b, latency +(15 rows) + +-- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges +SELECT tdigest(latency, 100) FROM latencies; + tdigest +--------------------------------------------------------------------- + flags 1 count 10000 compression 100 centroids 46 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (6.353631, 2) (8.434160, 3) (10.858766, 4) (15.597572, 5) (26.947867, 10) (39.185955, 13) (55.779949, 22) (97.378748, 35) (141.322218, 55) (212.927316, 65) (304.973404, 108) (435.651374, 148) (594.272516, 199) (820.494155, 292) (1214.081721, 463) (1727.098147, 547) (2337.903365, 749) (3325.589314, 1128) (4520.977007, 1300) (5638.186053, 1104) (6641.875581, 874) (7460.022115, 755) (8112.829059, 573) (8607.102557, 402) (8981.756066, 314) (9293.406295, 288) (9536.677260, 173) (9689.587344, 131) (9806.805461, 83) (9880.507729, 49) (9917.051853, 34) (9938.762095, 20) (9956.899980, 15) (9971.665384, 11) (9981.990769, 8) (9987.546338, 5) (9990.348725, 3) (9991.269368, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1) +(1 row) + +SELECT tdigest_percentile(latency, 100, 0.99) FROM latencies; + tdigest_percentile +--------------------------------------------------------------------- + 9903.84313359954 +(1 row) + +SELECT tdigest_percentile(latency, 100, ARRAY[0.99, 0.95]) FROM latencies; + tdigest_percentile +--------------------------------------------------------------------- + {9903.84313359954,9485.4900939991} +(1 row) + +SELECT tdigest_percentile_of(latency, 100, 9000) FROM latencies; + tdigest_percentile_of +--------------------------------------------------------------------- + 0.903462047211093 +(1 row) + +SELECT tdigest_percentile_of(latency, 100, ARRAY[9000, 9500]) FROM latencies; + tdigest_percentile_of +--------------------------------------------------------------------- + {0.903462047211093,0.951374818129208} +(1 row) + +CREATE TABLE latencies_rollup (a int, tdigest tdigest); +SELECT create_distributed_table('latencies_rollup', 'a', colocate_with => 'latencies'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO latencies_rollup +SELECT a, tdigest(latency, 100) +FROM latencies +GROUP BY a; +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest(tdigest) +FROM latencies_rollup; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest(remote_scan.tdigest) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(tdigest) + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest(tdigest) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest(tdigest) +FROM latencies_rollup +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest(tdigest) + Group Key: latencies_rollup.a + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(12 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile(tdigest, quantile) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile(tdigest, 0.99) +FROM latencies_rollup; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(tdigest) + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile(tdigest, quantile) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile(tdigest, 0.99) +FROM latencies_rollup +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile(tdigest, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile(tdigest, '0.99'::double precision) + Group Key: latencies_rollup.a + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(12 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile(tdigest, ARRAY[0.99, 0.95]) +FROM latencies_rollup; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(tdigest) + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile(tdigest, ARRAY[0.99, 0.95]) +FROM latencies_rollup +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) + Group Key: latencies_rollup.a + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(12 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile_of(tdigest, 9000) +FROM latencies_rollup; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(tdigest) + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile_of(tdigest, 9000) +FROM latencies_rollup +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile_of(tdigest, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile_of(tdigest, '9000'::double precision) + Group Key: latencies_rollup.a + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(12 rows) + +-- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT tdigest_percentile_of(tdigest, ARRAY[9000, 9500]) +FROM latencies_rollup; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: tdigest(tdigest) + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(13 rows) + +-- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) +EXPLAIN (COSTS OFF, VERBOSE) +SELECT a, tdigest_percentile_of(tdigest, ARRAY[9000, 9500]) +FROM latencies_rollup +GROUP BY a; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.a, remote_scan.tdigest_percentile_of + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT a, public.tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: a, tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) + Group Key: latencies_rollup.a + -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup + Output: a, tdigest +(12 rows) + +-- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges +SELECT tdigest(tdigest) FROM latencies_rollup; + tdigest +--------------------------------------------------------------------- + flags 1 count 10000 compression 100 centroids 47 (0.287235, 1) (1.025106, 1) (2.058216, 1) (5.335597, 1) (6.353631, 2) (8.434160, 3) (10.858766, 4) (15.597572, 5) (26.853448, 9) (36.608782, 11) (49.983794, 20) (85.586987, 27) (118.226702, 37) (173.623153, 56) (249.583971, 87) (354.777375, 112) (493.422943, 177) (697.499642, 262) (996.351736, 338) (1425.136508, 464) (1972.172107, 623) (2665.959054, 805) (3451.617535, 827) (4429.146575, 1195) (5587.124850, 1104) (6624.983751, 966) (7474.888557, 773) (8184.273132, 637) (8730.956261, 431) (9119.479419, 333) (9411.778428, 207) (9582.688196, 152) (9707.657431, 103) (9807.341000, 73) (9881.180119, 55) (9919.393423, 33) (9942.691902, 20) (9960.171491, 16) (9975.978413, 8) (9982.487777, 7) (9987.546338, 5) (9990.348725, 3) (9991.269368, 2) (9991.467422, 1) (9992.337047, 1) (9995.578357, 1) (9999.700339, 1) +(1 row) + +SELECT tdigest_percentile(tdigest, 0.99) FROM latencies_rollup; + tdigest_percentile +--------------------------------------------------------------------- + 9903.32646582201 +(1 row) + +SELECT tdigest_percentile(tdigest, ARRAY[0.99, 0.95]) FROM latencies_rollup; + tdigest_percentile +--------------------------------------------------------------------- + {9903.32646582201,9492.23455844906} +(1 row) + +SELECT tdigest_percentile_of(tdigest, 9000) FROM latencies_rollup; + tdigest_percentile_of +--------------------------------------------------------------------- + 0.902852659582613 +(1 row) + +SELECT tdigest_percentile_of(tdigest, ARRAY[9000, 9500]) FROM latencies_rollup; + tdigest_percentile_of +--------------------------------------------------------------------- + {0.902852659582613,0.950865574658712} +(1 row) + +SET client_min_messages TO WARNING; -- suppress cascade messages +DROP SCHEMA tdigest_aggregate_support CASCADE; From ab7c3b7804c1ec96be6cc88625060294ec2b2a58 Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Tue, 21 Jan 2025 17:48:06 +0300 Subject: [PATCH 134/155] PG17 Compatibility - Fix crash when pg_class is used in MERGE (#7853) This pull request addresses Issue #7846, where specific MERGE queries on non-distributed and distributed tables can result in crashes in certain scenarios. The issue stems from the usage of `pg_class` catalog table, and the `FilterShardsFromPgclass` function in Citus. This function goes through the query's jointree to hide the shards. However, in PG17, MERGE's join quals are in a separate structure called `mergeJoinCondition`. Therefore FilterShardsFromPgclass was not filtering correctly in a `MERGE` command that involves `pg_class`. To fix the issue, we handle `mergeJoinCondition` separately in PG17. Relevant PG commit: https://github.com/postgres/postgres/commit/0294df2f1f842dfb0eed79007b21016f486a3c6c **Non-Distributed Tables:** A MERGE query involving a non-distributed table using `pg_catalog.pg_class` as the source may execute successfully but needs testing to ensure stability. **Distributed Tables:** Performing a MERGE on a distributed table using `pg_catalog.pg_class` as the source raises an error: `ERROR: MERGE INTO a distributed table from Postgres table is not yet supported` However, in some cases, this can lead to a server crash if the unsupported operation is not properly handled. This is the test output from the same test conducted prior to the code changes being implemented. ``` -- Issue #7846: Test crash scenarios with MERGE on non-distributed and distributed tables -- Step 1: Connect to a worker node to verify shard visibility \c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql SET search_path TO pg17; -- Step 2: Create and test a non-distributed table CREATE TABLE non_dist_table_12345 (id INTEGER); -- Test MERGE on the non-distributed table MERGE INTO non_dist_table_12345 AS target_0 USING pg_catalog.pg_class AS ref_0 ON target_0.id = ref_0.relpages WHEN NOT MATCHED THEN DO NOTHING; SSL SYSCALL error: EOF detected connection to server was lost ``` --- citus-tools | 1 + .../worker/worker_shard_visibility.c | 33 ++++++++++++++---- src/test/regress/expected/pg17.out | 33 ++++++++++++++++++ src/test/regress/sql/pg17.sql | 34 +++++++++++++++++++ 4 files changed, 94 insertions(+), 7 deletions(-) create mode 160000 citus-tools diff --git a/citus-tools b/citus-tools new file mode 160000 index 000000000..3376bd684 --- /dev/null +++ b/citus-tools @@ -0,0 +1 @@ +Subproject commit 3376bd6845f0614908ed304f5033bd644c82d3bf diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index f783d514d..e80d66f58 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -441,7 +441,7 @@ FilterShardsFromPgclass(Node *node, void *context) /* * We process the whole rtable rather than visiting individual RangeTblEntry's * in the walker, since we need to know the varno to generate the right - * fiter. + * filter. */ int varno = 0; RangeTblEntry *rangeTableEntry = NULL; @@ -471,20 +471,39 @@ FilterShardsFromPgclass(Node *node, void *context) /* make sure the expression is in the right memory context */ MemoryContext originalContext = MemoryContextSwitchTo(queryContext); - /* add relation_is_a_known_shard(oid) IS NOT TRUE to the quals of the query */ Node *newQual = CreateRelationIsAKnownShardFilter(varno); - Node *oldQuals = query->jointree->quals; - if (oldQuals) + +#if PG_VERSION_NUM >= PG_VERSION_17 + + /* + * In PG17, MERGE queries introduce a new struct `mergeJoinCondition`. + * We need to handle this condition safely. + */ + if (query->mergeJoinCondition != NULL) { - query->jointree->quals = (Node *) makeBoolExpr( + /* Add the filter to mergeJoinCondition */ + query->mergeJoinCondition = (Node *) makeBoolExpr( AND_EXPR, - list_make2(oldQuals, newQual), + list_make2(query->mergeJoinCondition, newQual), -1); } else +#endif { - query->jointree->quals = newQual; + /* Handle older versions or queries without mergeJoinCondition */ + Node *oldQuals = query->jointree->quals; + if (oldQuals) + { + query->jointree->quals = (Node *) makeBoolExpr( + AND_EXPR, + list_make2(oldQuals, newQual), + -1); + } + else + { + query->jointree->quals = newQual; + } } MemoryContextSwitchTo(originalContext); diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 83507bb15..5e4460be1 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -2690,6 +2690,39 @@ SELECT * FROM sensor_readings ORDER BY 1; (4 rows) -- End of MERGE ... WHEN NOT MATCHED BY SOURCE tests +-- Issue #7846: Test crash scenarios with MERGE on non-distributed and distributed tables +-- Step 1: Connect to a worker node to verify shard visibility +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql +SET search_path TO pg17; +-- Step 2: Create and test a non-distributed table +CREATE TABLE non_dist_table_12345 (id INTEGER); +-- Test MERGE on the non-distributed table +MERGE INTO non_dist_table_12345 AS target_0 +USING pg_catalog.pg_class AS ref_0 +ON target_0.id = ref_0.relpages +WHEN NOT MATCHED THEN DO NOTHING; +-- Step 3: Switch back to the coordinator for distributed table operations +\c postgresql://postgres@localhost::master_port/regression?application_name=psql +SET search_path TO pg17; +-- Step 4: Create and test a distributed table +CREATE TABLE dist_table_67890 (id INTEGER); +SELECT create_distributed_table('dist_table_67890', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Test MERGE on the distributed table +MERGE INTO dist_table_67890 AS target_0 +USING pg_catalog.pg_class AS ref_0 +ON target_0.id = ref_0.relpages +WHEN NOT MATCHED THEN DO NOTHING; +ERROR: MERGE INTO an distributed table from Postgres table is not yet supported +-- Step 5: Cleanup +DROP TABLE non_dist_table_12345; +ERROR: table "non_dist_table_12345" does not exist +DROP TABLE dist_table_67890 CASCADE; +-- End of Issue #7846 \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index e4843db44..ef7371551 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -1451,6 +1451,40 @@ SELECT * FROM sensor_readings ORDER BY 1; -- End of MERGE ... WHEN NOT MATCHED BY SOURCE tests +-- Issue #7846: Test crash scenarios with MERGE on non-distributed and distributed tables +-- Step 1: Connect to a worker node to verify shard visibility +\c postgresql://postgres@localhost::worker_1_port/regression?application_name=psql +SET search_path TO pg17; + +-- Step 2: Create and test a non-distributed table +CREATE TABLE non_dist_table_12345 (id INTEGER); + +-- Test MERGE on the non-distributed table +MERGE INTO non_dist_table_12345 AS target_0 +USING pg_catalog.pg_class AS ref_0 +ON target_0.id = ref_0.relpages +WHEN NOT MATCHED THEN DO NOTHING; + +-- Step 3: Switch back to the coordinator for distributed table operations +\c postgresql://postgres@localhost::master_port/regression?application_name=psql +SET search_path TO pg17; + +-- Step 4: Create and test a distributed table +CREATE TABLE dist_table_67890 (id INTEGER); +SELECT create_distributed_table('dist_table_67890', 'id'); + +-- Test MERGE on the distributed table +MERGE INTO dist_table_67890 AS target_0 +USING pg_catalog.pg_class AS ref_0 +ON target_0.id = ref_0.relpages +WHEN NOT MATCHED THEN DO NOTHING; + +-- Step 5: Cleanup +DROP TABLE non_dist_table_12345; +DROP TABLE dist_table_67890 CASCADE; + +-- End of Issue #7846 + \set VERBOSITY terse SET client_min_messages TO WARNING; DROP SCHEMA pg17 CASCADE; From 23d52077018e64a3538d814e264c0ee36df1c4c1 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:54:52 +0300 Subject: [PATCH 135/155] Fix pg17 test (#7857) error merged in https://github.com/citusdata/citus/commit/ab7c3b7804c1ec96be6cc88625060294ec2b2a58 --- citus-tools | 1 - src/test/regress/expected/pg17.out | 5 +---- src/test/regress/sql/pg17.sql | 5 +---- 3 files changed, 2 insertions(+), 9 deletions(-) delete mode 160000 citus-tools diff --git a/citus-tools b/citus-tools deleted file mode 160000 index 3376bd684..000000000 --- a/citus-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3376bd6845f0614908ed304f5033bd644c82d3bf diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index 5e4460be1..c6deb41aa 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -2718,10 +2718,7 @@ USING pg_catalog.pg_class AS ref_0 ON target_0.id = ref_0.relpages WHEN NOT MATCHED THEN DO NOTHING; ERROR: MERGE INTO an distributed table from Postgres table is not yet supported --- Step 5: Cleanup -DROP TABLE non_dist_table_12345; -ERROR: table "non_dist_table_12345" does not exist -DROP TABLE dist_table_67890 CASCADE; +\c - - - :master_port -- End of Issue #7846 \set VERBOSITY terse SET client_min_messages TO WARNING; diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index ef7371551..f55d50d17 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -1479,10 +1479,7 @@ USING pg_catalog.pg_class AS ref_0 ON target_0.id = ref_0.relpages WHEN NOT MATCHED THEN DO NOTHING; --- Step 5: Cleanup -DROP TABLE non_dist_table_12345; -DROP TABLE dist_table_67890 CASCADE; - +\c - - - :master_port -- End of Issue #7846 \set VERBOSITY terse From cba8e57737d3770b70832b84305286121540b1cf Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:17:35 +0300 Subject: [PATCH 136/155] Changelog entries for 13.0.0 (#7858) --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a226fe2e7..54dd3bc05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ +### citus v13.0.0 (January 17, 2025) ### + +* Adds support for PostgreSQL 17 (#7699, #7661) + +* Adds `JSON_TABLE()` support in distributed queries (#7816) + +* Propagates `MERGE ... WHEN NOT MATCHED BY SOURCE` (#7807) + +* Propagates `MEMORY` and `SERIALIZE` options of `EXPLAIN` (#7802) + +* Adds support for identity columns in distributed partitioned tables (#7785) + +* Allows specifying an access method for distributed partitioned tables (#7818) + +* Allows exclusion constraints on distributed partitioned tables (#7733) + +* Allows configuring sslnegotiation using `citus.node_conn_info` (#7821) + +* Avoids wal receiver timeouts during large shard splits (#7229) + +* Fixes a bug causing incorrect writing of data to target `MERGE` repartition + command (#7659) + +* Fixes a crash that happens because of unsafe catalog access when re-assigning + the global pid after `application_name` changes (#7791) + +* Fixes incorrect `VALID UNTIL` setting assumption made for roles when syncing + them to new nodes (#7534) + +* Fixes segfault when calling distributed procedure with a parameterized + distribution argument (#7242) + +* Fixes server crash when trying to execute `activate_node_snapshot()` on a + single-node cluster (#7552) + +* Improves `citus_move_shard_placement()` to fail early if there is a new node + without reference tables yet (#7467) + ### citus v12.1.5 (July 17, 2024) ### * Adds support for MERGE commands with single shard distributed target tables From 548395fd77d2f73a026dfa8ed81aa8bae3b75e07 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:28:46 +0300 Subject: [PATCH 137/155] fix changelog date (#7859) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54dd3bc05..b4410d439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -### citus v13.0.0 (January 17, 2025) ### +### citus v13.0.0 (January 22, 2025) ### * Adds support for PostgreSQL 17 (#7699, #7661) From b886cfa22316e1047f5c725f0f6b96b08c607ab2 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:23:01 +0300 Subject: [PATCH 138/155] Upgrade upload-artifacts action to 4.6.0 --- .github/actions/save_logs_and_results/action.yml | 2 +- .github/actions/upload_coverage/action.yml | 2 +- .github/workflows/build_and_test.yml | 2 +- .github/workflows/flaky_test_debugging.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml index 0f238835d..b344c68f2 100644 --- a/.github/actions/save_logs_and_results/action.yml +++ b/.github/actions/save_logs_and_results/action.yml @@ -6,7 +6,7 @@ inputs: runs: using: composite steps: - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 name: Upload logs with: name: ${{ inputs.folder }} diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml index 0b5f581a6..abffa784a 100644 --- a/.github/actions/upload_coverage/action.yml +++ b/.github/actions/upload_coverage/action.yml @@ -21,7 +21,7 @@ runs: mkdir -p /tmp/codeclimate cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: path: "/tmp/codeclimate/*.json" name: codeclimate diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3d9b57c37..f76831099 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -122,7 +122,7 @@ jobs: - name: Build run: "./ci/build-citus.sh" shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml index a666c1cd5..574cb7813 100644 --- a/.github/workflows/flaky_test_debugging.yml +++ b/.github/workflows/flaky_test_debugging.yml @@ -34,7 +34,7 @@ jobs: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV ./ci/build-citus.sh shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- From cbe0de33a67772b21a9984b57f6133ca7316d2dc Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:39:55 +0300 Subject: [PATCH 139/155] Upgrade download-artifacts action to 4.1.8 --- .github/actions/setup_extension/action.yml | 2 +- .github/workflows/build_and_test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml index 96b408e7e..33129f17d 100644 --- a/.github/actions/setup_extension/action.yml +++ b/.github/actions/setup_extension/action.yml @@ -17,7 +17,7 @@ runs: echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV fi shell: bash - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v4.1.8 with: name: build-${{ env.PG_MAJOR }} - name: Install Extension diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f76831099..0703d1a7e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -424,7 +424,7 @@ jobs: - test-citus-upgrade - test-pg-upgrade steps: - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v4.1.8 with: name: "codeclimate" path: "codeclimate" @@ -525,7 +525,7 @@ jobs: matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} steps: - uses: actions/checkout@v3.5.0 - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v4.1.8 - uses: "./.github/actions/setup_extension" - name: Run minimal tests run: |- From 684b4c6b9666954bdf13c9eb597390eb71a66d48 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:56:33 +0300 Subject: [PATCH 140/155] Release RowExclusiveLock on pg_dist_transaction as soon as remote xacts are recovered As of this commit, after recovering the remote transactions, now we release the lock on pg_dist_transaction while closing it to avoid deadlocks that might occur because of trying to acquire a lock on pg_dist_authinfo while holding a lock on pg_dist_transaction. Such a scenario can only cause a deadlock if another transaction is trying to acquire a strong lock on pg_dist_transaction while holding a lock on pg_dist_authinfo. As of today, we (implicitly) acquire a strong lock on pg_dist_transaction only when upgrading Citus to 11.3-1 and this happens when creating a REPLICA IDENTITY on pg_dist_transaction. And regardless of the code-path we are in, it should be okay to release the lock there because all we do after that point is to abort the prepared transactions that are not part of an in-progress distributed transaction and releasing the lock before doing so should be just fine. This also changes the blocking behavior between citus_create_restore_point and the transaction recovery code-path in the sense that now citus_create_restore_point doesn't until transaction recovery completes aborting the prepared transactions that are not part of an in-progress distributed transaction. However, this should be fine because even before this was possible, e.g., if transaction recovery fails to open a remote connection to a node. --- .../transaction/transaction_recovery.c | 18 +++++++++++++++++- .../isolation_create_restore_point.out | 7 +++---- .../spec/isolation_create_restore_point.spec | 5 ++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index f25823b30..c114a2ed3 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -347,7 +347,23 @@ RecoverWorkerTransactions(WorkerNode *workerNode) } systable_endscan(scanDescriptor); - table_close(pgDistTransaction, NoLock); + + /* + * Here we release the lock on pg_dist_transaction while closing it to avoid + * deadlocks that might occur because of trying to acquire a lock on + * pg_dist_authinfo while holding a lock on pg_dist_transaction. Such a scenario + * can only cause a deadlock if another transaction is trying to acquire a strong + * lock on pg_dist_transaction while holding a lock on pg_dist_authinfo. As of + * today, we (implicitly) acquire a strong lock on pg_dist_transaction only when + * upgrading Citus to 11.3-1 and this happens when creating a REPLICA IDENTITY on + * pg_dist_transaction. + * + * And reglardless of the code-path we are in, it should be okay to release the + * lock now because all we do after this point is to abort the prepared + * transactions that are not part of an in-progress distributed transaction and + * releasing the lock before doing so should be just fine. + */ + table_close(pgDistTransaction, RowExclusiveLock); if (!recoveryFailed) { diff --git a/src/test/regress/expected/isolation_create_restore_point.out b/src/test/regress/expected/isolation_create_restore_point.out index 3b1bdf9eb..dce15a35d 100644 --- a/src/test/regress/expected/isolation_create_restore_point.out +++ b/src/test/regress/expected/isolation_create_restore_point.out @@ -147,16 +147,15 @@ recover_prepared_transactions step s2-create-restore: SELECT 1 FROM citus_create_restore_point('citus-test'); - -step s1-commit: - COMMIT; -step s2-create-restore: <... completed> ?column? --------------------------------------------------------------------- 1 (1 row) +step s1-commit: + COMMIT; + starting permutation: s1-begin s1-drop s2-create-restore s1-commit create_reference_table diff --git a/src/test/regress/spec/isolation_create_restore_point.spec b/src/test/regress/spec/isolation_create_restore_point.spec index 2cdc66f85..c62a64a44 100644 --- a/src/test/regress/spec/isolation_create_restore_point.spec +++ b/src/test/regress/spec/isolation_create_restore_point.spec @@ -154,7 +154,10 @@ permutation "s1-begin" "s1-ddl" "s2-create-restore" "s1-commit" // verify that citus_create_restore_point is not blocked by concurrent COPY (only commit) permutation "s1-begin" "s1-copy" "s2-create-restore" "s1-commit" -// verify that citus_create_restore_point is blocked by concurrent recover_prepared_transactions +// verify that citus_create_restore_point is partially blocked by concurrent recover_prepared_transactions. +// In the test output, we won't be able to explicitly observe this since +// recover_prepared_transactions unblocks citus_create_restore_point after in-progress prepared transactions +// are recovered. permutation "s1-begin" "s1-recover" "s2-create-restore" "s1-commit" // verify that citus_create_restore_point is blocked by concurrent DROP TABLE From 24758c39a1dad2ed7f549d3013137deb94de3bf6 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:59:23 +0300 Subject: [PATCH 141/155] Fix flaky citus upgrade test --- .../upgrade_pg_dist_cleanup_after_0.out | 9 +++++++++ .../upgrade_pg_dist_cleanup_before_0.out | 17 +++++++++++++++++ .../sql/upgrade_pg_dist_cleanup_after.sql | 5 +++++ .../sql/upgrade_pg_dist_cleanup_before.sql | 10 ++++++++++ 4 files changed, 41 insertions(+) diff --git a/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out b/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out index d71fad887..168c64cca 100644 --- a/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out +++ b/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out @@ -28,3 +28,12 @@ SELECT * FROM pg_dist_cleanup; CALL citus_cleanup_orphaned_resources(); NOTICE: cleaned up 1 orphaned resources DROP TABLE table_with_orphaned_shards; +-- Re-enable automatic shard cleanup by maintenance daemon as +-- we have disabled it in upgrade_pg_dist_cleanup_before.sql +ALTER SYSTEM RESET citus.defer_shard_delete_interval; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + diff --git a/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out b/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out index a0cf9ceb1..dd6c8868e 100644 --- a/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out +++ b/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out @@ -30,6 +30,23 @@ SELECT COUNT(*) FROM pg_dist_placement WHERE shardstate = 1 AND shardid IN (SELE (1 row) -- create an orphaned placement based on an existing one +-- +-- But before doing that, first disable automatic shard cleanup +-- by maintenance daemon so that we can reliably test the cleanup +-- in upgrade_pg_dist_cleanup_after.sql. +ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + INSERT INTO pg_dist_placement(placementid, shardid, shardstate, shardlength, groupid) SELECT nextval('pg_dist_placement_placementid_seq'::regclass), shardid, 4, shardlength, 3-groupid FROM pg_dist_placement diff --git a/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql b/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql index e84c35b60..333ac60ca 100644 --- a/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql +++ b/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql @@ -13,3 +13,8 @@ SELECT COUNT(*) FROM pg_dist_placement WHERE shardid IN (SELECT shardid FROM pg_ SELECT * FROM pg_dist_cleanup; CALL citus_cleanup_orphaned_resources(); DROP TABLE table_with_orphaned_shards; + +-- Re-enable automatic shard cleanup by maintenance daemon as +-- we have disabled it in upgrade_pg_dist_cleanup_before.sql +ALTER SYSTEM RESET citus.defer_shard_delete_interval; +SELECT pg_reload_conf(); diff --git a/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql b/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql index 62ec8a1fb..ec0eef353 100644 --- a/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql +++ b/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql @@ -16,6 +16,16 @@ SELECT create_distributed_table('table_with_orphaned_shards', 'a'); -- show all 32 placements are active SELECT COUNT(*) FROM pg_dist_placement WHERE shardstate = 1 AND shardid IN (SELECT shardid FROM pg_dist_shard WHERE logicalrelid='table_with_orphaned_shards'::regclass); -- create an orphaned placement based on an existing one +-- +-- But before doing that, first disable automatic shard cleanup +-- by maintenance daemon so that we can reliably test the cleanup +-- in upgrade_pg_dist_cleanup_after.sql. + +ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1; +SELECT pg_reload_conf(); + +SELECT pg_sleep(0.1); + INSERT INTO pg_dist_placement(placementid, shardid, shardstate, shardlength, groupid) SELECT nextval('pg_dist_placement_placementid_seq'::regclass), shardid, 4, shardlength, 3-groupid FROM pg_dist_placement From 26f16a76542eca2caca7e25848663286c0c8c6d5 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 15:46:42 +0300 Subject: [PATCH 142/155] Avoid publishing artifacts with conflicting names .. as documented in actions/upload-artifact#480. --- .github/actions/upload_coverage/action.yml | 2 +- .github/workflows/build_and_test.yml | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml index abffa784a..ba80ba63a 100644 --- a/.github/actions/upload_coverage/action.yml +++ b/.github/actions/upload_coverage/action.yml @@ -24,4 +24,4 @@ runs: - uses: actions/upload-artifact@v4.6.0 with: path: "/tmp/codeclimate/*.json" - name: codeclimate + name: codeclimate-${{ inputs.flags }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0703d1a7e..fefcc5bcb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -303,10 +303,12 @@ jobs: check-arbitrary-configs parallel=4 CONFIGS=$TESTS - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} - uses: "./.github/actions/upload_coverage" if: always() with: - flags: ${{ env.pg_major }}_upgrade + flags: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} codecov_token: ${{ secrets.CODECOV_TOKEN }} test-pg-upgrade: name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade @@ -360,6 +362,8 @@ jobs: if: failure() - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: @@ -405,10 +409,12 @@ jobs: done; - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.PG_MAJOR }}_citus_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: - flags: ${{ env.pg_major }}_upgrade + flags: ${{ env.PG_MAJOR }}_citus_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} upload-coverage: if: always() @@ -426,8 +432,9 @@ jobs: steps: - uses: actions/download-artifact@v4.1.8 with: - name: "codeclimate" - path: "codeclimate" + pattern: codeclimate* + path: codeclimate + merge-multiple: true - name: Upload coverage results to Code Climate run: |- cc-test-reporter sum-coverage codeclimate/*.json -o total.json From 6b70724b31654b7498352273083cbf063f996772 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:49:18 +0300 Subject: [PATCH 143/155] Fix mixed Citus upgrade tests (#7218) (#7871) When testing rolling Citus upgrades, coordinator should not be upgraded until we upgrade all the workers. --------- Co-authored-by: Jelte Fennema-Nio (cherry picked from commit 27ac44eb2a6be1c12c06f193d8d8511509a54bca) Co-authored-by: Onur Tirtir --- .../after_citus_upgrade_coord_schedule | 1 + src/test/regress/after_pg_upgrade_schedule | 2 +- .../citus_tests/upgrade/citus_upgrade_test.py | 14 ++- .../regress/expected/upgrade_basic_after.out | 94 ------------------- .../upgrade_basic_after_non_mixed.out | 94 +++++++++++++++++++ src/test/regress/sql/upgrade_basic_after.sql | 42 --------- .../sql/upgrade_basic_after_non_mixed.sql | 42 +++++++++ 7 files changed, 147 insertions(+), 142 deletions(-) create mode 100644 src/test/regress/expected/upgrade_basic_after_non_mixed.out create mode 100644 src/test/regress/sql/upgrade_basic_after_non_mixed.sql diff --git a/src/test/regress/after_citus_upgrade_coord_schedule b/src/test/regress/after_citus_upgrade_coord_schedule index f4f6bb29f..83dd1d9eb 100644 --- a/src/test/regress/after_citus_upgrade_coord_schedule +++ b/src/test/regress/after_citus_upgrade_coord_schedule @@ -3,4 +3,5 @@ test: upgrade_citus_finish_citus_upgrade test: upgrade_pg_dist_cleanup_after test: upgrade_basic_after +test: upgrade_basic_after_non_mixed test: upgrade_post_11_after diff --git a/src/test/regress/after_pg_upgrade_schedule b/src/test/regress/after_pg_upgrade_schedule index b47763bdb..82e05cf3f 100644 --- a/src/test/regress/after_pg_upgrade_schedule +++ b/src/test/regress/after_pg_upgrade_schedule @@ -1,4 +1,4 @@ -test: upgrade_basic_after upgrade_ref2ref_after upgrade_type_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects upgrade_autoconverted_after upgrade_citus_stat_activity upgrade_citus_locks upgrade_single_shard_table_after upgrade_schema_based_sharding_after +test: upgrade_basic_after upgrade_ref2ref_after upgrade_type_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects upgrade_autoconverted_after upgrade_citus_stat_activity upgrade_citus_locks upgrade_single_shard_table_after upgrade_schema_based_sharding_after upgrade_basic_after_non_mixed # This test cannot be run with run_test.py currently due to its dependence on # the specific PG versions that we use to run upgrade tests. For now we leave diff --git a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py index 87b00c83c..1ab448031 100755 --- a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py +++ b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py @@ -115,9 +115,9 @@ def remove_tar_files(tar_path): def restart_databases(pg_path, rel_data_path, mixed_mode, config): for node_name in config.node_name_to_ports.keys(): - if ( - mixed_mode - and config.node_name_to_ports[node_name] == config.chosen_random_worker_port + if mixed_mode and config.node_name_to_ports[node_name] in ( + config.chosen_random_worker_port, + config.coordinator_port(), ): continue abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) @@ -148,7 +148,10 @@ def restart_database(pg_path, abs_data_path, node_name, node_ports, logfile_pref def run_alter_citus(pg_path, mixed_mode, config): for port in config.node_name_to_ports.values(): - if mixed_mode and port == config.chosen_random_worker_port: + if mixed_mode and port in ( + config.chosen_random_worker_port, + config.coordinator_port(), + ): continue utils.psql(pg_path, port, "ALTER EXTENSION citus UPDATE;") @@ -158,7 +161,8 @@ def verify_upgrade(config, mixed_mode, node_ports): actual_citus_version = get_actual_citus_version(config.bindir, port) expected_citus_version = MASTER_VERSION if expected_citus_version != actual_citus_version and not ( - mixed_mode and port == config.chosen_random_worker_port + mixed_mode + and port in (config.chosen_random_worker_port, config.coordinator_port()) ): print( "port: {} citus version {} expected {}".format( diff --git a/src/test/regress/expected/upgrade_basic_after.out b/src/test/regress/expected/upgrade_basic_after.out index 1bfbfc989..7c0ebfb29 100644 --- a/src/test/regress/expected/upgrade_basic_after.out +++ b/src/test/regress/expected/upgrade_basic_after.out @@ -9,100 +9,6 @@ SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' and tablename NOT LI upgrade_basic | tp | tp_pkey | | CREATE UNIQUE INDEX tp_pkey ON upgrade_basic.tp USING btree (a) (3 rows) -SELECT nextval('pg_dist_shardid_seq') > MAX(shardid) FROM pg_dist_shard; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_placement_placementid_seq') > MAX(placementid) FROM pg_dist_placement; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_groupid_seq') > MAX(groupid) FROM pg_dist_node; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_node_nodeid_seq') > MAX(nodeid) FROM pg_dist_node; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_colocationid_seq') > MAX(colocationid) FROM pg_dist_colocation; - ?column? ---------------------------------------------------------------------- - t -(1 row) - --- while testing sequences on pg_dist_cleanup, they return null in pg upgrade schedule --- but return a valid value in citus upgrade schedule --- that's why we accept both NULL and MAX()+1 here -SELECT - CASE WHEN MAX(operation_id) IS NULL - THEN true - ELSE nextval('pg_dist_operationid_seq') > MAX(operation_id) - END AS check_operationid - FROM pg_dist_cleanup; - check_operationid ---------------------------------------------------------------------- - t -(1 row) - -SELECT - CASE WHEN MAX(record_id) IS NULL - THEN true - ELSE nextval('pg_dist_cleanup_recordid_seq') > MAX(record_id) - END AS check_recordid - FROM pg_dist_cleanup; - check_recordid ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_background_job_job_id_seq') > COALESCE(MAX(job_id), 0) FROM pg_dist_background_job; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT nextval('pg_dist_background_task_task_id_seq') > COALESCE(MAX(task_id), 0) FROM pg_dist_background_task; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT last_value > 0 FROM pg_dist_clock_logical_seq; - ?column? ---------------------------------------------------------------------- - t -(1 row) - --- If this query gives output it means we've added a new sequence that should --- possibly be restored after upgrades. -SELECT sequence_name FROM information_schema.sequences - WHERE sequence_name LIKE 'pg_dist_%' - AND sequence_name NOT IN ( - -- these ones are restored above - 'pg_dist_shardid_seq', - 'pg_dist_placement_placementid_seq', - 'pg_dist_groupid_seq', - 'pg_dist_node_nodeid_seq', - 'pg_dist_colocationid_seq', - 'pg_dist_operationid_seq', - 'pg_dist_cleanup_recordid_seq', - 'pg_dist_background_job_job_id_seq', - 'pg_dist_background_task_task_id_seq', - 'pg_dist_clock_logical_seq' - ); - sequence_name ---------------------------------------------------------------------- -(0 rows) - SELECT logicalrelid FROM pg_dist_partition JOIN pg_depend ON logicalrelid=objid JOIN pg_catalog.pg_class ON logicalrelid=oid diff --git a/src/test/regress/expected/upgrade_basic_after_non_mixed.out b/src/test/regress/expected/upgrade_basic_after_non_mixed.out new file mode 100644 index 000000000..8dbc13bab --- /dev/null +++ b/src/test/regress/expected/upgrade_basic_after_non_mixed.out @@ -0,0 +1,94 @@ +SELECT nextval('pg_dist_shardid_seq') > MAX(shardid) FROM pg_dist_shard; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_placement_placementid_seq') > MAX(placementid) FROM pg_dist_placement; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_groupid_seq') > MAX(groupid) FROM pg_dist_node; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_node_nodeid_seq') > MAX(nodeid) FROM pg_dist_node; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_colocationid_seq') > MAX(colocationid) FROM pg_dist_colocation; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +-- while testing sequences on pg_dist_cleanup, they return null in pg upgrade schedule +-- but return a valid value in citus upgrade schedule +-- that's why we accept both NULL and MAX()+1 here +SELECT + CASE WHEN MAX(operation_id) IS NULL + THEN true + ELSE nextval('pg_dist_operationid_seq') > MAX(operation_id) + END AS check_operationid + FROM pg_dist_cleanup; + check_operationid +--------------------------------------------------------------------- + t +(1 row) + +SELECT + CASE WHEN MAX(record_id) IS NULL + THEN true + ELSE nextval('pg_dist_cleanup_recordid_seq') > MAX(record_id) + END AS check_recordid + FROM pg_dist_cleanup; + check_recordid +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_background_job_job_id_seq') > COALESCE(MAX(job_id), 0) FROM pg_dist_background_job; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT nextval('pg_dist_background_task_task_id_seq') > COALESCE(MAX(task_id), 0) FROM pg_dist_background_task; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +SELECT last_value > 0 FROM pg_dist_clock_logical_seq; + ?column? +--------------------------------------------------------------------- + t +(1 row) + +-- If this query gives output it means we've added a new sequence that should +-- possibly be restored after upgrades. +SELECT sequence_name FROM information_schema.sequences + WHERE sequence_name LIKE 'pg_dist_%' + AND sequence_name NOT IN ( + -- these ones are restored above + 'pg_dist_shardid_seq', + 'pg_dist_placement_placementid_seq', + 'pg_dist_groupid_seq', + 'pg_dist_node_nodeid_seq', + 'pg_dist_colocationid_seq', + 'pg_dist_operationid_seq', + 'pg_dist_cleanup_recordid_seq', + 'pg_dist_background_job_job_id_seq', + 'pg_dist_background_task_task_id_seq', + 'pg_dist_clock_logical_seq' + ); + sequence_name +--------------------------------------------------------------------- +(0 rows) + diff --git a/src/test/regress/sql/upgrade_basic_after.sql b/src/test/regress/sql/upgrade_basic_after.sql index b40501a1e..855c06008 100644 --- a/src/test/regress/sql/upgrade_basic_after.sql +++ b/src/test/regress/sql/upgrade_basic_after.sql @@ -3,48 +3,6 @@ BEGIN; -- We have the tablename filter to avoid adding an alternative output for when the coordinator is in metadata vs when not SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' and tablename NOT LIKE 'r_%' ORDER BY tablename; -SELECT nextval('pg_dist_shardid_seq') > MAX(shardid) FROM pg_dist_shard; -SELECT nextval('pg_dist_placement_placementid_seq') > MAX(placementid) FROM pg_dist_placement; -SELECT nextval('pg_dist_groupid_seq') > MAX(groupid) FROM pg_dist_node; -SELECT nextval('pg_dist_node_nodeid_seq') > MAX(nodeid) FROM pg_dist_node; -SELECT nextval('pg_dist_colocationid_seq') > MAX(colocationid) FROM pg_dist_colocation; --- while testing sequences on pg_dist_cleanup, they return null in pg upgrade schedule --- but return a valid value in citus upgrade schedule --- that's why we accept both NULL and MAX()+1 here -SELECT - CASE WHEN MAX(operation_id) IS NULL - THEN true - ELSE nextval('pg_dist_operationid_seq') > MAX(operation_id) - END AS check_operationid - FROM pg_dist_cleanup; -SELECT - CASE WHEN MAX(record_id) IS NULL - THEN true - ELSE nextval('pg_dist_cleanup_recordid_seq') > MAX(record_id) - END AS check_recordid - FROM pg_dist_cleanup; -SELECT nextval('pg_dist_background_job_job_id_seq') > COALESCE(MAX(job_id), 0) FROM pg_dist_background_job; -SELECT nextval('pg_dist_background_task_task_id_seq') > COALESCE(MAX(task_id), 0) FROM pg_dist_background_task; -SELECT last_value > 0 FROM pg_dist_clock_logical_seq; - --- If this query gives output it means we've added a new sequence that should --- possibly be restored after upgrades. -SELECT sequence_name FROM information_schema.sequences - WHERE sequence_name LIKE 'pg_dist_%' - AND sequence_name NOT IN ( - -- these ones are restored above - 'pg_dist_shardid_seq', - 'pg_dist_placement_placementid_seq', - 'pg_dist_groupid_seq', - 'pg_dist_node_nodeid_seq', - 'pg_dist_colocationid_seq', - 'pg_dist_operationid_seq', - 'pg_dist_cleanup_recordid_seq', - 'pg_dist_background_job_job_id_seq', - 'pg_dist_background_task_task_id_seq', - 'pg_dist_clock_logical_seq' - ); - SELECT logicalrelid FROM pg_dist_partition JOIN pg_depend ON logicalrelid=objid JOIN pg_catalog.pg_class ON logicalrelid=oid diff --git a/src/test/regress/sql/upgrade_basic_after_non_mixed.sql b/src/test/regress/sql/upgrade_basic_after_non_mixed.sql new file mode 100644 index 000000000..17b8367fb --- /dev/null +++ b/src/test/regress/sql/upgrade_basic_after_non_mixed.sql @@ -0,0 +1,42 @@ +SELECT nextval('pg_dist_shardid_seq') > MAX(shardid) FROM pg_dist_shard; +SELECT nextval('pg_dist_placement_placementid_seq') > MAX(placementid) FROM pg_dist_placement; +SELECT nextval('pg_dist_groupid_seq') > MAX(groupid) FROM pg_dist_node; +SELECT nextval('pg_dist_node_nodeid_seq') > MAX(nodeid) FROM pg_dist_node; +SELECT nextval('pg_dist_colocationid_seq') > MAX(colocationid) FROM pg_dist_colocation; + +-- while testing sequences on pg_dist_cleanup, they return null in pg upgrade schedule +-- but return a valid value in citus upgrade schedule +-- that's why we accept both NULL and MAX()+1 here +SELECT + CASE WHEN MAX(operation_id) IS NULL + THEN true + ELSE nextval('pg_dist_operationid_seq') > MAX(operation_id) + END AS check_operationid + FROM pg_dist_cleanup; +SELECT + CASE WHEN MAX(record_id) IS NULL + THEN true + ELSE nextval('pg_dist_cleanup_recordid_seq') > MAX(record_id) + END AS check_recordid + FROM pg_dist_cleanup; +SELECT nextval('pg_dist_background_job_job_id_seq') > COALESCE(MAX(job_id), 0) FROM pg_dist_background_job; +SELECT nextval('pg_dist_background_task_task_id_seq') > COALESCE(MAX(task_id), 0) FROM pg_dist_background_task; +SELECT last_value > 0 FROM pg_dist_clock_logical_seq; + +-- If this query gives output it means we've added a new sequence that should +-- possibly be restored after upgrades. +SELECT sequence_name FROM information_schema.sequences + WHERE sequence_name LIKE 'pg_dist_%' + AND sequence_name NOT IN ( + -- these ones are restored above + 'pg_dist_shardid_seq', + 'pg_dist_placement_placementid_seq', + 'pg_dist_groupid_seq', + 'pg_dist_node_nodeid_seq', + 'pg_dist_colocationid_seq', + 'pg_dist_operationid_seq', + 'pg_dist_cleanup_recordid_seq', + 'pg_dist_background_job_job_id_seq', + 'pg_dist_background_task_task_id_seq', + 'pg_dist_clock_logical_seq' + ); From 9a7f6d6c59e03063f6b3cf01ddbb2a99b3fe0f2a Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:13:40 +0300 Subject: [PATCH 144/155] Drops PG14 support (#7753) DESCRIPTION: Drops PG14 support 1. Remove "$version_num" != 'xx' from configure file 2. delete all PG_VERSION_NUM = PG_VERSION_XX references in the code 3. Look at pg_version_compat.h file, remove all _compat functions etc defined specifically for PGXX differences 4. delete all PG_VERSION_NUM >= PG_VERSION_(XX+1), PG_VERSION_NUM < PG_VERSION_(XX+1) ifs in the codebase 5. delete ruleutils_xx.c file 6. cleanup normalize.sed file from pg14 specific lines 7. delete all alternative output files for that particular PG version, server_version_ge variable helps here --- .gitattributes | 2 - .github/workflows/build_and_test.yml | 31 +- configure | 2 +- configure.ac | 2 +- src/backend/columnar/columnar_customscan.c | 3 - src/backend/columnar/columnar_metadata.c | 2 +- src/backend/columnar/columnar_tableam.c | 13 +- src/backend/distributed/clock/causal_clock.c | 11 - src/backend/distributed/commands/collation.c | 30 - .../commands/create_distributed_table.c | 9 - src/backend/distributed/commands/database.c | 5 - .../commands/distribute_object_ops.c | 10 +- .../distributed/commands/foreign_constraint.c | 2 - .../distributed/commands/publication.c | 30 - src/backend/distributed/commands/role.c | 15 - src/backend/distributed/commands/sequence.c | 5 - src/backend/distributed/commands/table.c | 15 +- src/backend/distributed/commands/trigger.c | 95 - .../connection/shared_connection_stats.c | 10 - .../distributed/deparser/citus_ruleutils.c | 4 - .../deparser/deparse_database_stmts.c | 4 - .../deparser/deparse_publication_stmts.c | 136 - .../deparser/deparse_sequence_stmts.c | 7 - .../deparser/deparse_table_stmts.c | 2 - .../deparser/qualify_publication_stmt.c | 34 - .../deparser/qualify_sequence_stmt.c | 5 - .../distributed/deparser/ruleutils_14.c | 8638 ----------------- .../distributed/executor/adaptive_executor.c | 25 - .../distributed/executor/query_stats.c | 13 - src/backend/distributed/metadata/dependency.c | 2 - .../distributed/metadata/metadata_sync.c | 43 - .../metadata/pg_get_object_address_13_14_15.c | 4 - .../planner/combine_query_planner.c | 3 - .../distributed/planner/distributed_planner.c | 5 - .../distributed/planner/merge_planner.c | 37 - .../distributed/planner/multi_explain.c | 4 +- .../shardsplit/shardsplit_decoder.c | 4 - src/backend/distributed/shared_library_init.c | 11 - src/backend/distributed/test/fake_am.c | 2 +- .../test/shared_connection_counters.c | 5 - .../distributed/transaction/backend_data.c | 9 - .../distributed/utils/background_jobs.c | 81 - .../distributed/utils/citus_stat_tenants.c | 11 +- .../columnar/columnar_version_compat.h | 8 - src/include/distributed/commands.h | 6 - src/include/distributed/deparser.h | 4 - src/include/distributed/distributed_planner.h | 5 - src/include/distributed/metadata_sync.h | 3 - src/include/pg_version_compat.h | 75 +- src/include/pg_version_constants.h | 1 - src/test/regress/bin/normalize.sed | 6 - .../expected/citus_local_tables_queries.out | 11 - .../expected/citus_local_tables_queries_0.out | 1168 --- src/test/regress/expected/columnar_pg15.out | 7 - src/test/regress/expected/columnar_pg15_0.out | 6 - .../expected/columnar_vacuum_vs_insert.out | 2 +- .../expected/coordinator_shouldhaveshards.out | 11 - .../coordinator_shouldhaveshards_0.out | 1190 --- src/test/regress/expected/cte_inline.out | 11 - src/test/regress/expected/cte_inline_0.out | 1489 --- .../regress/expected/detect_conn_close.out | 7 - .../regress/expected/detect_conn_close_0.out | 9 - .../expected/grant_on_schema_propagation.out | 10 - .../grant_on_schema_propagation_0.out | 400 - .../expected/insert_select_repartition.out | 11 - .../expected/insert_select_repartition_0.out | 1334 --- .../expected/intermediate_result_pruning.out | 11 - .../intermediate_result_pruning_0.out | 1077 -- src/test/regress/expected/issue_5248.out | 12 - src/test/regress/expected/issue_5248_0.out | 230 - .../expected/local_shard_execution.out | 11 - .../expected/local_shard_execution_0.out | 3302 ------- .../local_shard_execution_replicated.out | 11 - .../local_shard_execution_replicated_0.out | 2462 ----- src/test/regress/expected/merge.out | 7 - src/test/regress/expected/merge_0.out | 6 - src/test/regress/expected/merge_arbitrary.out | 7 - .../regress/expected/merge_arbitrary_0.out | 6 - .../expected/merge_arbitrary_create.out | 7 - .../expected/merge_arbitrary_create_0.out | 6 - .../expected/merge_partition_tables.out | 7 - .../expected/merge_partition_tables_0.out | 6 - .../regress/expected/merge_repartition1.out | 7 - .../regress/expected/merge_repartition1_0.out | 6 - .../regress/expected/merge_repartition2.out | 7 - .../regress/expected/merge_repartition2_0.out | 6 - .../expected/merge_schema_sharding.out | 7 - .../expected/merge_schema_sharding_0.out | 6 - src/test/regress/expected/merge_vcore.out | 7 - src/test/regress/expected/merge_vcore_0.out | 6 - ...ter_table_add_constraints_without_name.out | 5 - .../expected/multi_deparse_shard_query.out | 11 - .../expected/multi_deparse_shard_query_0.out | 423 - src/test/regress/expected/multi_extension.out | 8 +- .../regress/expected/multi_insert_select.out | 11 - .../expected/multi_insert_select_0.out | 3507 ------- .../expected/multi_insert_select_conflict.out | 11 - .../multi_insert_select_conflict_0.out | 600 -- .../regress/expected/multi_metadata_sync.out | 10 - .../expected/multi_metadata_sync_0.out | 2262 ----- .../multi_mx_insert_select_repartition.out | 11 - .../multi_mx_insert_select_repartition_0.out | 166 - .../mx_coordinator_shouldhaveshards.out | 11 - .../mx_coordinator_shouldhaveshards_0.out | 335 - src/test/regress/expected/pg15.out | 7 - src/test/regress/expected/pg15_0.out | 9 - src/test/regress/expected/pg15_jsonpath.out | 7 - src/test/regress/expected/pg15_jsonpath_0.out | 10 - src/test/regress/expected/pgmerge.out | 7 - src/test/regress/expected/pgmerge_0.out | 6 - src/test/regress/expected/publication.out | 11 - src/test/regress/expected/publication_0.out | 270 - src/test/regress/expected/single_node.out | 11 - src/test/regress/expected/single_node_0.out | 2581 ----- .../upgrade_citus_finish_citus_upgrade_1.out | 41 + .../sql/citus_local_tables_queries.sql | 6 - src/test/regress/sql/columnar_pg15.sql | 8 - .../sql/coordinator_shouldhaveshards.sql | 6 - src/test/regress/sql/cte_inline.sql | 6 - src/test/regress/sql/detect_conn_close.sql | 7 - .../sql/grant_on_schema_propagation.sql | 5 - .../regress/sql/insert_select_repartition.sql | 6 - .../sql/intermediate_result_pruning.sql | 6 - src/test/regress/sql/issue_5248.sql | 13 - .../regress/sql/local_shard_execution.sql | 6 - .../sql/local_shard_execution_replicated.sql | 6 - src/test/regress/sql/merge.sql | 8 - src/test/regress/sql/merge_arbitrary.sql | 8 - .../regress/sql/merge_arbitrary_create.sql | 8 - .../regress/sql/merge_partition_tables.sql | 9 - src/test/regress/sql/merge_repartition1.sql | 8 - src/test/regress/sql/merge_repartition2.sql | 9 - .../regress/sql/merge_schema_sharding.sql | 8 - src/test/regress/sql/merge_vcore.sql | 8 - ...ter_table_add_constraints_without_name.sql | 6 - .../regress/sql/multi_deparse_shard_query.sql | 6 - src/test/regress/sql/multi_extension.sql | 8 +- src/test/regress/sql/multi_insert_select.sql | 7 +- .../sql/multi_insert_select_conflict.sql | 6 - src/test/regress/sql/multi_metadata_sync.sql | 5 - .../multi_mx_insert_select_repartition.sql | 6 - .../sql/mx_coordinator_shouldhaveshards.sql | 6 - src/test/regress/sql/pg15.sql | 7 - src/test/regress/sql/pg15_jsonpath.sql | 8 - src/test/regress/sql/pgmerge.sql | 8 - src/test/regress/sql/publication.sql | 12 - src/test/regress/sql/single_node.sql | 6 - 147 files changed, 63 insertions(+), 32875 deletions(-) delete mode 100644 src/backend/distributed/deparser/ruleutils_14.c delete mode 100644 src/test/regress/expected/citus_local_tables_queries_0.out delete mode 100644 src/test/regress/expected/columnar_pg15_0.out delete mode 100644 src/test/regress/expected/coordinator_shouldhaveshards_0.out delete mode 100644 src/test/regress/expected/cte_inline_0.out delete mode 100644 src/test/regress/expected/detect_conn_close_0.out delete mode 100644 src/test/regress/expected/grant_on_schema_propagation_0.out delete mode 100644 src/test/regress/expected/insert_select_repartition_0.out delete mode 100644 src/test/regress/expected/intermediate_result_pruning_0.out delete mode 100644 src/test/regress/expected/issue_5248_0.out delete mode 100644 src/test/regress/expected/local_shard_execution_0.out delete mode 100644 src/test/regress/expected/local_shard_execution_replicated_0.out delete mode 100644 src/test/regress/expected/merge_0.out delete mode 100644 src/test/regress/expected/merge_arbitrary_0.out delete mode 100644 src/test/regress/expected/merge_arbitrary_create_0.out delete mode 100644 src/test/regress/expected/merge_partition_tables_0.out delete mode 100644 src/test/regress/expected/merge_repartition1_0.out delete mode 100644 src/test/regress/expected/merge_repartition2_0.out delete mode 100644 src/test/regress/expected/merge_schema_sharding_0.out delete mode 100644 src/test/regress/expected/merge_vcore_0.out delete mode 100644 src/test/regress/expected/multi_deparse_shard_query_0.out delete mode 100644 src/test/regress/expected/multi_insert_select_0.out delete mode 100644 src/test/regress/expected/multi_insert_select_conflict_0.out delete mode 100644 src/test/regress/expected/multi_metadata_sync_0.out delete mode 100644 src/test/regress/expected/multi_mx_insert_select_repartition_0.out delete mode 100644 src/test/regress/expected/mx_coordinator_shouldhaveshards_0.out delete mode 100644 src/test/regress/expected/pg15_0.out delete mode 100644 src/test/regress/expected/pg15_jsonpath_0.out delete mode 100644 src/test/regress/expected/pgmerge_0.out delete mode 100644 src/test/regress/expected/publication_0.out delete mode 100644 src/test/regress/expected/single_node_0.out create mode 100644 src/test/regress/expected/upgrade_citus_finish_citus_upgrade_1.out diff --git a/.gitattributes b/.gitattributes index c7c03e1ef..51520c243 100644 --- a/.gitattributes +++ b/.gitattributes @@ -25,8 +25,6 @@ configure -whitespace # except these exceptions... src/backend/distributed/utils/citus_outfuncs.c -citus-style -src/backend/distributed/deparser/ruleutils_13.c -citus-style -src/backend/distributed/deparser/ruleutils_14.c -citus-style src/backend/distributed/deparser/ruleutils_15.c -citus-style src/backend/distributed/deparser/ruleutils_16.c -citus-style src/backend/distributed/deparser/ruleutils_17.c -citus-style diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index fefcc5bcb..36ba97039 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -28,7 +28,7 @@ jobs: style_checker_tools_version: "0.8.18" sql_snapshot_pg_version: "17.2" image_suffix: "-v889e4c1" - pg14_version: '{ "major": "14", "full": "14.15" }' + image_suffix_citus_upgrade: "-dev-2ad1f90" pg15_version: '{ "major": "15", "full": "15.10" }' pg16_version: '{ "major": "16", "full": "16.6" }' pg17_version: '{ "major": "17", "full": "17.2" }' @@ -106,7 +106,6 @@ jobs: image_suffix: - ${{ needs.params.outputs.image_suffix}} pg_version: - - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} @@ -138,7 +137,6 @@ jobs: image_name: - ${{ needs.params.outputs.test_image_name }} pg_version: - - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} @@ -159,10 +157,6 @@ jobs: - check-enterprise-isolation-logicalrep-2 - check-enterprise-isolation-logicalrep-3 include: - - make: check-failure - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-failure pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress @@ -175,10 +169,6 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-enterprise-failure - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-enterprise-failure pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress @@ -191,10 +181,6 @@ jobs: pg_version: ${{ needs.params.outputs.pg17_version }} suite: regress image_name: ${{ needs.params.outputs.fail_test_image_name }} - - make: check-pytest - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-pytest pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress @@ -219,10 +205,6 @@ jobs: suite: cdc image_name: ${{ needs.params.outputs.test_image_name }} pg_version: ${{ needs.params.outputs.pg17_version }} - - make: check-query-generator - pg_version: ${{ needs.params.outputs.pg14_version }} - suite: regress - image_name: ${{ needs.params.outputs.fail_test_image_name }} - make: check-query-generator pg_version: ${{ needs.params.outputs.pg15_version }} suite: regress @@ -275,7 +257,6 @@ jobs: image_name: - ${{ needs.params.outputs.fail_test_image_name }} pg_version: - - ${{ needs.params.outputs.pg14_version }} - ${{ needs.params.outputs.pg15_version }} - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} @@ -323,18 +304,12 @@ jobs: fail-fast: false matrix: include: - - old_pg_major: 14 - new_pg_major: 15 - old_pg_major: 15 new_pg_major: 16 - - old_pg_major: 14 - new_pg_major: 16 - old_pg_major: 16 new_pg_major: 17 - old_pg_major: 15 new_pg_major: 17 - - old_pg_major: 14 - new_pg_major: 17 env: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} @@ -370,10 +345,10 @@ jobs: flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-upgrade: - name: PG${{ fromJson(needs.params.outputs.pg14_version).major }} - check-citus-upgrade + name: PG${{ fromJson(needs.params.outputs.pg15_version).major }} - check-citus-upgrade runs-on: ubuntu-20.04 container: - image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg14_version).full }}${{ needs.params.outputs.image_suffix }}" + image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(needs.params.outputs.pg15_version).full }}${{ needs.params.outputs.image_suffix_citus_upgrade }}" options: --user root needs: - params diff --git a/configure b/configure index 5240df4db..d2f4060a7 100755 --- a/configure +++ b/configure @@ -2588,7 +2588,7 @@ fi if test "$with_pg_version_check" = no; then { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;} -elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then +elif test "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.ac b/configure.ac index c7fde02de..0d79adce1 100644 --- a/configure.ac +++ b/configure.ac @@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check) if test "$with_pg_version_check" = no; then AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)]) -elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then +elif test "$version_num" != '15' -a "$version_num" != '16' -a "$version_num" != '17'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) diff --git a/src/backend/columnar/columnar_customscan.c b/src/backend/columnar/columnar_customscan.c index c78485c66..6b4e338c3 100644 --- a/src/backend/columnar/columnar_customscan.c +++ b/src/backend/columnar/columnar_customscan.c @@ -1312,11 +1312,8 @@ AddColumnarScanPath(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, cpath->methods = &ColumnarScanPathMethods; -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* necessary to avoid extra Result node in PG15 */ cpath->flags = CUSTOMPATH_SUPPORT_PROJECTION; -#endif /* * populate generic path information diff --git a/src/backend/columnar/columnar_metadata.c b/src/backend/columnar/columnar_metadata.c index f3035ba8d..3857f1f07 100644 --- a/src/backend/columnar/columnar_metadata.c +++ b/src/backend/columnar/columnar_metadata.c @@ -1686,7 +1686,7 @@ DeleteTupleAndEnforceConstraints(ModifyState *state, HeapTuple heapTuple) simple_heap_delete(state->rel, tid); /* execute AFTER ROW DELETE Triggers to enforce constraints */ - ExecARDeleteTriggers_compat(estate, resultRelInfo, tid, NULL, NULL, false); + ExecARDeleteTriggers(estate, resultRelInfo, tid, NULL, NULL, false); } diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 83df11c42..4b777364e 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -877,7 +877,7 @@ columnar_relation_set_new_filelocator(Relation rel, *freezeXid = RecentXmin; *minmulti = GetOldestMultiXactId(); - SMgrRelation srel = RelationCreateStorage_compat(*newrlocator, persistence, true); + SMgrRelation srel = RelationCreateStorage(*newrlocator, persistence, true); ColumnarStorageInit(srel, ColumnarMetadataNewStorageId()); InitColumnarOptions(rel->rd_id); @@ -2245,7 +2245,6 @@ ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions columnarRangeVar = alterTableStmt->relation; } } -#if PG_VERSION_NUM >= PG_VERSION_15 else if (alterTableCmd->subtype == AT_SetAccessMethod) { if (columnarRangeVar || *columnarOptions) @@ -2265,7 +2264,6 @@ ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions DeleteColumnarTableOptions(RelationGetRelid(rel), true); } } -#endif /* PG_VERSION_15 */ } relation_close(rel, NoLock); @@ -2649,21 +2647,12 @@ ColumnarCheckLogicalReplication(Relation rel) return; } -#if PG_VERSION_NUM >= PG_VERSION_15 { PublicationDesc pubdesc; RelationBuildPublicationDesc(rel, &pubdesc); pubActionInsert = pubdesc.pubactions.pubinsert; } -#else - if (rel->rd_pubactions == NULL) - { - GetRelationPublicationActions(rel); - Assert(rel->rd_pubactions != NULL); - } - pubActionInsert = rel->rd_pubactions->pubinsert; -#endif if (pubActionInsert) { diff --git a/src/backend/distributed/clock/causal_clock.c b/src/backend/distributed/clock/causal_clock.c index 230d4e733..1bc594693 100644 --- a/src/backend/distributed/clock/causal_clock.c +++ b/src/backend/distributed/clock/causal_clock.c @@ -145,17 +145,6 @@ LogicalClockShmemSize(void) void InitializeClusterClockMem(void) { - /* On PG 15 and above, we use shmem_request_hook_type */ - #if PG_VERSION_NUM < PG_VERSION_15 - - /* allocate shared memory for pre PG-15 versions */ - if (!IsUnderPostmaster) - { - RequestAddinShmemSpace(LogicalClockShmemSize()); - } - - #endif - prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = LogicalClockShmemInit; } diff --git a/src/backend/distributed/commands/collation.c b/src/backend/distributed/commands/collation.c index 4a47b5c18..268694034 100644 --- a/src/backend/distributed/commands/collation.c +++ b/src/backend/distributed/commands/collation.c @@ -68,8 +68,6 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati char *collcollate; char *collctype; -#if PG_VERSION_NUM >= PG_VERSION_15 - /* * In PG15, there is an added option to use ICU as global locale provider. * pg_collation has three locale-related fields: collcollate and collctype, @@ -112,16 +110,6 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati } Assert((collcollate && collctype) || colllocale); -#else - - /* - * In versions before 15, collcollate and collctype were type "name". Use - * pstrdup() to match the interface of 15 so that we consistently free the - * result later. - */ - collcollate = pstrdup(NameStr(collationForm->collcollate)); - collctype = pstrdup(NameStr(collationForm->collctype)); -#endif if (collowner != NULL) { @@ -147,7 +135,6 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati "CREATE COLLATION %s (provider = '%s'", *quotedCollationName, providerString); -#if PG_VERSION_NUM >= PG_VERSION_15 if (colllocale) { appendStringInfo(&collationNameDef, @@ -173,24 +160,7 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati pfree(collcollate); pfree(collctype); } -#else - if (strcmp(collcollate, collctype) == 0) - { - appendStringInfo(&collationNameDef, - ", locale = %s", - quote_literal_cstr(collcollate)); - } - else - { - appendStringInfo(&collationNameDef, - ", lc_collate = %s, lc_ctype = %s", - quote_literal_cstr(collcollate), - quote_literal_cstr(collctype)); - } - pfree(collcollate); - pfree(collctype); -#endif #if PG_VERSION_NUM >= PG_VERSION_16 char *collicurules = NULL; datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collicurules, &isnull); diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 7af6f2dd0..536f80291 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -170,12 +170,10 @@ static void EnsureDistributedSequencesHaveOneType(Oid relationId, static void CopyLocalDataIntoShards(Oid distributedTableId); static List * TupleDescColumnNameList(TupleDesc tupleDescriptor); -#if (PG_VERSION_NUM >= PG_VERSION_15) static bool DistributionColumnUsesNumericColumnNegativeScale(TupleDesc relationDesc, Var *distributionColumn); static int numeric_typmod_scale(int32 typmod); static bool is_valid_numeric_typmod(int32 typmod); -#endif static bool DistributionColumnUsesGeneratedStoredColumn(TupleDesc relationDesc, Var *distributionColumn); @@ -2114,8 +2112,6 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, "AS (...) STORED."))); } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* verify target relation is not distributed by a column of type numeric with negative scale */ if (distributionMethod != DISTRIBUTE_BY_NONE && DistributionColumnUsesNumericColumnNegativeScale(relationDesc, @@ -2126,7 +2122,6 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, errdetail("Distribution column must not use numeric type " "with negative scale"))); } -#endif /* check for support function needed by specified partition method */ if (distributionMethod == DISTRIBUTE_BY_HASH) @@ -2844,8 +2839,6 @@ TupleDescColumnNameList(TupleDesc tupleDescriptor) } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * is_valid_numeric_typmod checks if the typmod value is valid * @@ -2895,8 +2888,6 @@ DistributionColumnUsesNumericColumnNegativeScale(TupleDesc relationDesc, } -#endif - /* * DistributionColumnUsesGeneratedStoredColumn returns whether a given relation uses * GENERATED ALWAYS AS (...) STORED on distribution column diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index c92191774..15bc0e5bc 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -185,8 +185,6 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString, } -#if PG_VERSION_NUM >= PG_VERSION_15 - /* * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local * postgres instance. @@ -217,9 +215,6 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, } -#endif - - /* * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress * of the database. diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 0b476e496..d76bb44f4 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -465,7 +465,6 @@ static DistributeObjectOps Database_Alter = { .markDistributed = false, }; -#if PG_VERSION_NUM >= PG_VERSION_15 static DistributeObjectOps Database_RefreshColl = { .deparse = DeparseAlterDatabaseRefreshCollStmt, .qualify = NULL, @@ -476,7 +475,6 @@ static DistributeObjectOps Database_RefreshColl = { .address = NULL, .markDistributed = false, }; -#endif static DistributeObjectOps Domain_Alter = { .deparse = DeparseAlterDomainStmt, @@ -837,7 +835,6 @@ static DistributeObjectOps Sequence_AlterOwner = { .address = AlterSequenceOwnerStmtObjectAddress, .markDistributed = false, }; -#if (PG_VERSION_NUM >= PG_VERSION_15) static DistributeObjectOps Sequence_AlterPersistence = { .deparse = DeparseAlterSequencePersistenceStmt, .qualify = QualifyAlterSequencePersistenceStmt, @@ -847,7 +844,6 @@ static DistributeObjectOps Sequence_AlterPersistence = { .address = AlterSequencePersistenceStmtObjectAddress, .markDistributed = false, }; -#endif static DistributeObjectOps Sequence_Drop = { .deparse = DeparseDropSequenceStmt, .qualify = QualifyDropSequenceStmt, @@ -1299,7 +1295,7 @@ static DistributeObjectOps View_Rename = { static DistributeObjectOps Trigger_Rename = { .deparse = NULL, .qualify = NULL, - .preprocess = PreprocessAlterTriggerRenameStmt, + .preprocess = NULL, .operationType = DIST_OPS_ALTER, .postprocess = PostprocessAlterTriggerRenameStmt, .address = NULL, @@ -1321,13 +1317,11 @@ GetDistributeObjectOps(Node *node) return &Database_Alter; } -#if PG_VERSION_NUM >= PG_VERSION_15 case T_AlterDatabaseRefreshCollStmt: { return &Database_RefreshColl; } -#endif case T_AlterDomainStmt: { return &Domain_Alter; @@ -1612,7 +1606,6 @@ GetDistributeObjectOps(Node *node) case OBJECT_SEQUENCE: { -#if (PG_VERSION_NUM >= PG_VERSION_15) ListCell *cmdCell = NULL; foreach(cmdCell, stmt->cmds) { @@ -1640,7 +1633,6 @@ GetDistributeObjectOps(Node *node) } } } -#endif /* * Prior to PG15, the only Alter Table statement diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index b7162b1a4..bc12ccb4d 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -467,7 +467,6 @@ ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple) } List *onDeleteSetDefColumnList = NIL; -#if PG_VERSION_NUM >= PG_VERSION_15 Datum onDeleteSetDefColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple, Anum_pg_constraint_confdelsetcols, &isNull); @@ -482,7 +481,6 @@ ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple) onDeleteSetDefColumnList = IntegerArrayTypeToList(DatumGetArrayTypeP(onDeleteSetDefColumnsDatum)); } -#endif if (list_length(onDeleteSetDefColumnList) == 0) { diff --git a/src/backend/distributed/commands/publication.c b/src/backend/distributed/commands/publication.c index efb017977..91272ca68 100644 --- a/src/backend/distributed/commands/publication.c +++ b/src/backend/distributed/commands/publication.c @@ -33,11 +33,9 @@ static CreatePublicationStmt * BuildCreatePublicationStmt(Oid publicationId); -#if (PG_VERSION_NUM >= PG_VERSION_15) static PublicationObjSpec * BuildPublicationRelationObjSpec(Oid relationId, Oid publicationId, bool tableOnly); -#endif static void AppendPublishOptionList(StringInfo str, List *strings); static char * AlterPublicationOwnerCommand(Oid publicationId); static bool ShouldPropagateCreatePublication(CreatePublicationStmt *stmt); @@ -154,7 +152,6 @@ BuildCreatePublicationStmt(Oid publicationId) ReleaseSysCache(publicationTuple); -#if (PG_VERSION_NUM >= PG_VERSION_15) List *schemaIds = GetPublicationSchemas(publicationId); Oid schemaId = InvalidOid; @@ -170,7 +167,6 @@ BuildCreatePublicationStmt(Oid publicationId) createPubStmt->pubobjects = lappend(createPubStmt->pubobjects, publicationObject); } -#endif List *relationIds = GetPublicationRelations(publicationId, publicationForm->pubviaroot ? @@ -184,7 +180,6 @@ BuildCreatePublicationStmt(Oid publicationId) foreach_declared_oid(relationId, relationIds) { -#if (PG_VERSION_NUM >= PG_VERSION_15) bool tableOnly = false; /* since postgres 15, tables can have a column list and filter */ @@ -192,15 +187,6 @@ BuildCreatePublicationStmt(Oid publicationId) BuildPublicationRelationObjSpec(relationId, publicationId, tableOnly); createPubStmt->pubobjects = lappend(createPubStmt->pubobjects, publicationObject); -#else - - /* before postgres 15, only full tables are supported */ - char *schemaName = get_namespace_name(get_rel_namespace(relationId)); - char *tableName = get_rel_name(relationId); - RangeVar *rangeVar = makeRangeVar(schemaName, tableName, -1); - - createPubStmt->tables = lappend(createPubStmt->tables, rangeVar); -#endif if (IsCitusTable(relationId)) { @@ -276,8 +262,6 @@ AppendPublishOptionList(StringInfo str, List *options) } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * BuildPublicationRelationObjSpec returns a PublicationObjSpec that * can be included in a CREATE or ALTER PUBLICATION statement. @@ -357,9 +341,6 @@ BuildPublicationRelationObjSpec(Oid relationId, Oid publicationId, } -#endif - - /* * PreprocessAlterPublicationStmt handles ALTER PUBLICATION statements * in a way that is mostly similar to PreprocessAlterDistributedObjectStmt, @@ -458,7 +439,6 @@ GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, ReleaseSysCache(pubTuple); -#if (PG_VERSION_NUM >= PG_VERSION_15) bool tableOnly = !isAdd; /* since postgres 15, tables can have a column list and filter */ @@ -467,16 +447,6 @@ GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, alterPubStmt->pubobjects = lappend(alterPubStmt->pubobjects, publicationObject); alterPubStmt->action = isAdd ? AP_AddObjects : AP_DropObjects; -#else - - /* before postgres 15, only full tables are supported */ - char *schemaName = get_namespace_name(get_rel_namespace(relationId)); - char *tableName = get_rel_name(relationId); - RangeVar *rangeVar = makeRangeVar(schemaName, tableName, -1); - - alterPubStmt->tables = lappend(alterPubStmt->tables, rangeVar); - alterPubStmt->tableAction = isAdd ? DEFELEM_ADD : DEFELEM_DROP; -#endif /* we take the WHERE clause from the catalog where it is already transformed */ bool whereClauseNeedsTransform = false; diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 6a42211a6..ca76f0fc0 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -1027,13 +1027,8 @@ makeStringConst(char *str, int location) { A_Const *n = makeNode(A_Const); -#if PG_VERSION_NUM >= PG_VERSION_15 n->val.sval.type = T_String; n->val.sval.sval = str; -#else - n->val.type = T_String; - n->val.val.str = str; -#endif n->location = location; return (Node *) n; @@ -1053,13 +1048,8 @@ makeIntConst(int val, int location) { A_Const *n = makeNode(A_Const); -#if PG_VERSION_NUM >= PG_VERSION_15 n->val.ival.type = T_Integer; n->val.ival.ival = val; -#else - n->val.type = T_Integer; - n->val.val.ival = val; -#endif n->location = location; return (Node *) n; @@ -1076,13 +1066,8 @@ makeFloatConst(char *str, int location) { A_Const *n = makeNode(A_Const); -#if PG_VERSION_NUM >= PG_VERSION_15 n->val.fval.type = T_Float; n->val.fval.fval = str; -#else - n->val.type = T_Float; - n->val.val.str = str; -#endif n->location = location; return (Node *) n; diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index 4af4c4853..0dd544cc6 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -735,8 +735,6 @@ PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString) } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * PreprocessAlterSequencePersistenceStmt is called for change of persistence * of sequences before the persistence is changed on the local instance. @@ -847,9 +845,6 @@ PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, } -#endif - - /* * PreprocessGrantOnSequenceStmt is executed before the statement is applied to the local * postgres instance. diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 67b731a25..eaa8b1031 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -1153,7 +1153,6 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, { AlterTableStmt *stmtCopy = copyObject(alterTableStatement); stmtCopy->objtype = OBJECT_SEQUENCE; -#if (PG_VERSION_NUM >= PG_VERSION_15) /* * it must be ALTER TABLE .. OWNER TO .. @@ -1163,16 +1162,6 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, */ return PreprocessSequenceAlterTableStmt((Node *) stmtCopy, alterTableCommand, processUtilityContext); -#else - - /* - * it must be ALTER TABLE .. OWNER TO .. command - * since this is the only ALTER command of a sequence that - * passes through an AlterTableStmt - */ - return PreprocessAlterSequenceOwnerStmt((Node *) stmtCopy, alterTableCommand, - processUtilityContext); -#endif } else if (relKind == RELKIND_VIEW) { @@ -3673,9 +3662,8 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) "are currently unsupported."))); break; } - #endif -#if PG_VERSION_NUM >= PG_VERSION_15 + case AT_SetAccessMethod: { /* @@ -3695,7 +3683,6 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) break; } -#endif case AT_SetNotNull: case AT_ReplicaIdentity: case AT_ChangeOwner: diff --git a/src/backend/distributed/commands/trigger.c b/src/backend/distributed/commands/trigger.c index 01ee72d31..5b4d93584 100644 --- a/src/backend/distributed/commands/trigger.c +++ b/src/backend/distributed/commands/trigger.c @@ -57,9 +57,6 @@ static void ExtractDropStmtTriggerAndRelationName(DropStmt *dropTriggerStmt, static void ErrorIfDropStmtDropsMultipleTriggers(DropStmt *dropTriggerStmt); static char * GetTriggerNameById(Oid triggerId); static int16 GetTriggerTypeById(Oid triggerId); -#if (PG_VERSION_NUM < PG_VERSION_15) -static void ErrorOutIfCloneTrigger(Oid tgrelid, const char *tgname); -#endif /* GUC that overrides trigger checks for distributed tables and reference tables */ @@ -404,40 +401,6 @@ CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaNam } -/* - * PreprocessAlterTriggerRenameStmt is called before a ALTER TRIGGER RENAME - * command has been executed by standard process utility. This function errors - * out if we are trying to rename a child trigger on a partition of a distributed - * table. In PG15, this is not allowed anyway. - */ -List * -PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext) -{ -#if (PG_VERSION_NUM < PG_VERSION_15) - RenameStmt *renameTriggerStmt = castNode(RenameStmt, node); - Assert(renameTriggerStmt->renameType == OBJECT_TRIGGER); - - RangeVar *relation = renameTriggerStmt->relation; - - bool missingOk = false; - Oid relationId = RangeVarGetRelid(relation, ALTER_TRIGGER_LOCK_MODE, missingOk); - - if (!IsCitusTable(relationId)) - { - return NIL; - } - - EnsureCoordinator(); - ErrorOutForTriggerIfNotSupported(relationId); - - ErrorOutIfCloneTrigger(relationId, renameTriggerStmt->subname); -#endif - - return NIL; -} - - /* * PostprocessAlterTriggerRenameStmt is called after a ALTER TRIGGER RENAME * command has been executed by standard process utility. This function errors @@ -759,64 +722,6 @@ ErrorIfRelationHasUnsupportedTrigger(Oid relationId) } -#if (PG_VERSION_NUM < PG_VERSION_15) - -/* - * ErrorOutIfCloneTrigger is a helper function to error - * out if we are trying to rename a child trigger on a - * partition of a distributed table. - * A lot of this code is borrowed from PG15 because - * renaming clone triggers isn't allowed in PG15 anymore. - */ -static void -ErrorOutIfCloneTrigger(Oid tgrelid, const char *tgname) -{ - HeapTuple tuple; - ScanKeyData key[2]; - - Relation tgrel = table_open(TriggerRelationId, RowExclusiveLock); - - /* - * Search for the trigger to modify. - */ - ScanKeyInit(&key[0], - Anum_pg_trigger_tgrelid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(tgrelid)); - ScanKeyInit(&key[1], - Anum_pg_trigger_tgname, - BTEqualStrategyNumber, F_NAMEEQ, - CStringGetDatum(tgname)); - SysScanDesc tgscan = systable_beginscan(tgrel, TriggerRelidNameIndexId, true, - NULL, 2, key); - - if (HeapTupleIsValid(tuple = systable_getnext(tgscan))) - { - Form_pg_trigger trigform = (Form_pg_trigger) GETSTRUCT(tuple); - - /* - * If the trigger descends from a trigger on a parent partitioned - * table, reject the rename. - * Appended shard ids to find the trigger on the partition's shards - * are not correct. Hence we would fail to find the trigger on the - * partition's shard. - */ - if (OidIsValid(trigform->tgparentid)) - { - ereport(ERROR, ( - errmsg( - "cannot rename child triggers on distributed partitions"))); - } - } - - systable_endscan(tgscan); - table_close(tgrel, RowExclusiveLock); -} - - -#endif - - /* * GetDropTriggerStmtRelation takes a DropStmt for a trigger object and returns * RangeVar for the relation that owns the trigger. diff --git a/src/backend/distributed/connection/shared_connection_stats.c b/src/backend/distributed/connection/shared_connection_stats.c index 26598b465..027bb46a2 100644 --- a/src/backend/distributed/connection/shared_connection_stats.c +++ b/src/backend/distributed/connection/shared_connection_stats.c @@ -614,16 +614,6 @@ WaitForSharedConnection(void) void InitializeSharedConnectionStats(void) { -/* on PG 15, we use shmem_request_hook_type */ -#if PG_VERSION_NUM < PG_VERSION_15 - - /* allocate shared memory */ - if (!IsUnderPostmaster) - { - RequestAddinShmemSpace(SharedConnectionStatsShmemSize()); - } -#endif - prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = SharedConnectionStatsShmemInit; } diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index d138f8859..bdba2bf6e 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -258,10 +258,8 @@ pg_get_sequencedef_string(Oid sequenceRelationId) char *typeName = format_type_be(pgSequenceForm->seqtypid); char *sequenceDef = psprintf(CREATE_SEQUENCE_COMMAND, -#if (PG_VERSION_NUM >= PG_VERSION_15) get_rel_persistence(sequenceRelationId) == RELPERSISTENCE_UNLOGGED ? "UNLOGGED " : "", -#endif qualifiedSequenceName, typeName, pgSequenceForm->seqincrement, pgSequenceForm->seqmin, @@ -857,12 +855,10 @@ deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid, appendStringInfoString(buffer, ") "); } -#if PG_VERSION_NUM >= PG_VERSION_15 if (indexStmt->nulls_not_distinct) { appendStringInfoString(buffer, "NULLS NOT DISTINCT "); } -#endif /* PG_VERSION_15 */ if (indexStmt->options != NIL) { diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 596d10713..0e64e10ff 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -159,7 +159,6 @@ DeparseAlterDatabaseStmt(Node *node) } -#if PG_VERSION_NUM >= PG_VERSION_15 char * DeparseAlterDatabaseRefreshCollStmt(Node *node) { @@ -174,6 +173,3 @@ DeparseAlterDatabaseRefreshCollStmt(Node *node) return str.data; } - - -#endif diff --git a/src/backend/distributed/deparser/deparse_publication_stmts.c b/src/backend/distributed/deparser/deparse_publication_stmts.c index 35068266e..37346eb5e 100644 --- a/src/backend/distributed/deparser/deparse_publication_stmts.c +++ b/src/backend/distributed/deparser/deparse_publication_stmts.c @@ -32,7 +32,6 @@ static void AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables); -#if (PG_VERSION_NUM >= PG_VERSION_15) static bool AppendPublicationObjects(StringInfo buf, List *publicationObjects, bool whereClauseNeedsTransform, bool includeLocalTables); @@ -40,10 +39,6 @@ static void AppendWhereClauseExpression(StringInfo buf, RangeVar *tableName, Node *whereClause, bool whereClauseNeedsTransform); static void AppendAlterPublicationAction(StringInfo buf, AlterPublicationAction action); -#else -static bool AppendTables(StringInfo buf, List *tables, bool includeLocalTables); -static void AppendDefElemAction(StringInfo buf, DefElemAction action); -#endif static bool AppendAlterPublicationStmt(StringInfo buf, AlterPublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables); @@ -108,7 +103,6 @@ AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, { appendStringInfoString(buf, " FOR ALL TABLES"); } -#if (PG_VERSION_NUM >= PG_VERSION_15) else if (stmt->pubobjects != NIL) { bool hasObjects = false; @@ -146,32 +140,6 @@ AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, includeLocalTables); } } -#else - else if (stmt->tables != NIL) - { - bool hasTables = false; - RangeVar *rangeVar = NULL; - - /* - * Check whether there are tables to propagate, mainly to know whether - * we should include "FOR". - */ - foreach_declared_ptr(rangeVar, stmt->tables) - { - if (includeLocalTables || IsCitusTableRangeVar(rangeVar, NoLock, false)) - { - hasTables = true; - break; - } - } - - if (hasTables) - { - appendStringInfoString(buf, " FOR"); - AppendTables(buf, stmt->tables, includeLocalTables); - } - } -#endif if (stmt->options != NIL) { @@ -182,8 +150,6 @@ AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * AppendPublicationObjects appends a string representing a list of publication * objects to a buffer. @@ -320,57 +286,6 @@ AppendWhereClauseExpression(StringInfo buf, RangeVar *tableName, } -#else - -/* - * AppendPublicationObjects appends a string representing a list of publication - * objects to a buffer. - * - * For instance: TABLE users, departments - */ -static bool -AppendTables(StringInfo buf, List *tables, bool includeLocalTables) -{ - RangeVar *rangeVar = NULL; - bool appendedObject = false; - - foreach_declared_ptr(rangeVar, tables) - { - if (!includeLocalTables && - !IsCitusTableRangeVar(rangeVar, NoLock, false)) - { - /* do not propagate local tables */ - continue; - } - - char *schemaName = rangeVar->schemaname; - char *tableName = rangeVar->relname; - - if (schemaName != NULL) - { - /* qualified table name */ - appendStringInfo(buf, "%s %s", - appendedObject ? "," : " TABLE", - quote_qualified_identifier(schemaName, tableName)); - } - else - { - /* unqualified table name */ - appendStringInfo(buf, "%s %s", - appendedObject ? "," : " TABLE", - quote_identifier(tableName)); - } - - appendedObject = true; - } - - return appendedObject; -} - - -#endif - - /* * DeparseAlterPublicationSchemaStmt builds and returns a string representing * an AlterPublicationStmt. @@ -439,19 +354,12 @@ AppendAlterPublicationStmt(StringInfo buf, AlterPublicationStmt *stmt, return true; } -#if (PG_VERSION_NUM >= PG_VERSION_15) AppendAlterPublicationAction(buf, stmt->action); return AppendPublicationObjects(buf, stmt->pubobjects, whereClauseNeedsTransform, includeLocalTables); -#else - AppendDefElemAction(buf, stmt->tableAction); - return AppendTables(buf, stmt->tables, includeLocalTables); -#endif } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * AppendAlterPublicationAction appends a string representing an AlterPublicationAction * to a buffer. @@ -487,46 +395,6 @@ AppendAlterPublicationAction(StringInfo buf, AlterPublicationAction action) } -#else - -/* - * AppendDefElemAction appends a string representing a DefElemAction - * to a buffer. - */ -static void -AppendDefElemAction(StringInfo buf, DefElemAction action) -{ - switch (action) - { - case DEFELEM_ADD: - { - appendStringInfoString(buf, " ADD"); - break; - } - - case DEFELEM_DROP: - { - appendStringInfoString(buf, " DROP"); - break; - } - - case DEFELEM_SET: - { - appendStringInfoString(buf, " SET"); - break; - } - - default: - { - ereport(ERROR, (errmsg("unrecognized publication action: %d", action))); - } - } -} - - -#endif - - /* * DeparseDropPublicationStmt builds and returns a string representing the DropStmt */ @@ -651,11 +519,7 @@ AppendPublicationOptions(StringInfo stringBuffer, List *optionList) appendStringInfo(stringBuffer, "%s = ", quote_identifier(optionName)); -#if (PG_VERSION_NUM >= PG_VERSION_15) if (valueType == T_Integer || valueType == T_Float || valueType == T_Boolean) -#else - if (valueType == T_Integer || valueType == T_Float) -#endif { /* string escaping is unnecessary for numeric types and can cause issues */ appendStringInfo(stringBuffer, "%s", optionValue); diff --git a/src/backend/distributed/deparser/deparse_sequence_stmts.c b/src/backend/distributed/deparser/deparse_sequence_stmts.c index 9e5fab2c8..b16751d7f 100644 --- a/src/backend/distributed/deparser/deparse_sequence_stmts.c +++ b/src/backend/distributed/deparser/deparse_sequence_stmts.c @@ -28,9 +28,7 @@ static void AppendSequenceNameList(StringInfo buf, List *objects, ObjectType obj static void AppendRenameSequenceStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterSequenceSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterSequenceOwnerStmt(StringInfo buf, AlterTableStmt *stmt); -#if (PG_VERSION_NUM >= PG_VERSION_15) static void AppendAlterSequencePersistenceStmt(StringInfo buf, AlterTableStmt *stmt); -#endif static void AppendGrantOnSequenceStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnSequenceSequences(StringInfo buf, GrantStmt *stmt); @@ -262,8 +260,6 @@ AppendAlterSequenceOwnerStmt(StringInfo buf, AlterTableStmt *stmt) } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * DeparseAlterSequencePersistenceStmt builds and returns a string representing * the AlterTableStmt consisting of changing the persistence of a sequence @@ -349,9 +345,6 @@ AppendAlterSequencePersistenceStmt(StringInfo buf, AlterTableStmt *stmt) } -#endif - - /* * DeparseGrantOnSequenceStmt builds and returns a string representing the GrantOnSequenceStmt */ diff --git a/src/backend/distributed/deparser/deparse_table_stmts.c b/src/backend/distributed/deparser/deparse_table_stmts.c index 5d184fa66..d58fbabcc 100644 --- a/src/backend/distributed/deparser/deparse_table_stmts.c +++ b/src/backend/distributed/deparser/deparse_table_stmts.c @@ -193,12 +193,10 @@ AppendAlterTableCmdConstraint(StringInfo buf, Constraint *constraint, { appendStringInfoString(buf, " UNIQUE"); -#if (PG_VERSION_NUM >= PG_VERSION_15) if (constraint->nulls_not_distinct == true) { appendStringInfoString(buf, " NULLS NOT DISTINCT"); } -#endif } if (subtype == AT_AddConstraint) diff --git a/src/backend/distributed/deparser/qualify_publication_stmt.c b/src/backend/distributed/deparser/qualify_publication_stmt.c index c47f52e15..0790dc06b 100644 --- a/src/backend/distributed/deparser/qualify_publication_stmt.c +++ b/src/backend/distributed/deparser/qualify_publication_stmt.c @@ -19,11 +19,7 @@ #include "distributed/deparser.h" #include "distributed/listutils.h" -#if (PG_VERSION_NUM >= PG_VERSION_15) static void QualifyPublicationObjects(List *publicationObjects); -#else -static void QualifyTables(List *tables); -#endif static void QualifyPublicationRangeVar(RangeVar *publication); @@ -36,16 +32,10 @@ QualifyCreatePublicationStmt(Node *node) { CreatePublicationStmt *stmt = castNode(CreatePublicationStmt, node); -#if (PG_VERSION_NUM >= PG_VERSION_15) QualifyPublicationObjects(stmt->pubobjects); -#else - QualifyTables(stmt->tables); -#endif } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * QualifyPublicationObjects ensures all table names in a list of * publication objects are fully qualified. @@ -68,26 +58,6 @@ QualifyPublicationObjects(List *publicationObjects) } -#else - -/* - * QualifyTables ensures all table names in a list are fully qualified. - */ -static void -QualifyTables(List *tables) -{ - RangeVar *rangeVar = NULL; - - foreach_declared_ptr(rangeVar, tables) - { - QualifyPublicationRangeVar(rangeVar); - } -} - - -#endif - - /* * QualifyPublicationObjects ensures all table names in a list of * publication objects are fully qualified. @@ -97,11 +67,7 @@ QualifyAlterPublicationStmt(Node *node) { AlterPublicationStmt *stmt = castNode(AlterPublicationStmt, node); -#if (PG_VERSION_NUM >= PG_VERSION_15) QualifyPublicationObjects(stmt->pubobjects); -#else - QualifyTables(stmt->tables); -#endif } diff --git a/src/backend/distributed/deparser/qualify_sequence_stmt.c b/src/backend/distributed/deparser/qualify_sequence_stmt.c index c56d0fda0..402a661ce 100644 --- a/src/backend/distributed/deparser/qualify_sequence_stmt.c +++ b/src/backend/distributed/deparser/qualify_sequence_stmt.c @@ -52,8 +52,6 @@ QualifyAlterSequenceOwnerStmt(Node *node) } -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* * QualifyAlterSequencePersistenceStmt transforms a * ALTER SEQUENCE .. SET LOGGED/UNLOGGED @@ -80,9 +78,6 @@ QualifyAlterSequencePersistenceStmt(Node *node) } -#endif - - /* * QualifyAlterSequenceSchemaStmt transforms a * ALTER SEQUENCE .. SET SCHEMA .. diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c deleted file mode 100644 index 88948cff5..000000000 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ /dev/null @@ -1,8638 +0,0 @@ -/*------------------------------------------------------------------------- - * - * ruleutils_14.c - * Functions to convert stored expressions/querytrees back to - * source text - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * - * IDENTIFICATION - * src/backend/distributed/deparser/ruleutils_14.c - * - * This needs to be closely in sync with the core code. - *------------------------------------------------------------------------- - */ -#include "pg_version_constants.h" - -#include "pg_config.h" - -#if (PG_VERSION_NUM >= PG_VERSION_14) && (PG_VERSION_NUM < PG_VERSION_15) - -#include "postgres.h" - -#include -#include -#include - -#include "access/amapi.h" -#include "access/htup_details.h" -#include "access/relation.h" -#include "access/sysattr.h" -#include "access/table.h" -#include "catalog/pg_aggregate.h" -#include "catalog/pg_am.h" -#include "catalog/pg_authid.h" -#include "catalog/pg_collation.h" -#include "catalog/pg_constraint.h" -#include "catalog/pg_depend.h" -#include "catalog/pg_extension.h" -#include "catalog/pg_foreign_data_wrapper.h" -#include "catalog/pg_language.h" -#include "catalog/pg_opclass.h" -#include "catalog/pg_operator.h" -#include "catalog/pg_partitioned_table.h" -#include "catalog/pg_proc.h" -#include "catalog/pg_statistic_ext.h" -#include "catalog/pg_trigger.h" -#include "catalog/pg_type.h" -#include "commands/defrem.h" -#include "commands/extension.h" -#include "commands/tablespace.h" -#include "common/keywords.h" -#include "distributed/citus_nodefuncs.h" -#include "distributed/citus_ruleutils.h" -#include "distributed/namespace_utils.h" -#include "executor/spi.h" -#include "foreign/foreign.h" -#include "funcapi.h" -#include "mb/pg_wchar.h" -#include "miscadmin.h" -#include "nodes/makefuncs.h" -#include "nodes/nodeFuncs.h" -#include "nodes/pathnodes.h" -#include "optimizer/optimizer.h" -#include "parser/parse_node.h" -#include "parser/parse_agg.h" -#include "parser/parse_func.h" -#include "parser/parse_node.h" -#include "parser/parse_oper.h" -#include "parser/parser.h" -#include "parser/parsetree.h" -#include "rewrite/rewriteHandler.h" -#include "rewrite/rewriteManip.h" -#include "rewrite/rewriteSupport.h" -#include "utils/array.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/hsearch.h" -#include "utils/lsyscache.h" -#include "utils/rel.h" -#include "utils/ruleutils.h" -#include "utils/snapmgr.h" -#include "utils/syscache.h" -#include "utils/typcache.h" -#include "utils/varlena.h" -#include "utils/xml.h" - - -/* ---------- - * Pretty formatting constants - * ---------- - */ - -/* Indent counts */ -#define PRETTYINDENT_STD 8 -#define PRETTYINDENT_JOIN 4 -#define PRETTYINDENT_VAR 4 - -#define PRETTYINDENT_LIMIT 40 /* wrap limit */ - -/* Pretty flags */ -#define PRETTYFLAG_PAREN 0x0001 -#define PRETTYFLAG_INDENT 0x0002 - -/* Default line length for pretty-print wrapping: 0 means wrap always */ -#define WRAP_COLUMN_DEFAULT 0 - -/* macros to test if pretty action needed */ -#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) -#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) - - -/* ---------- - * Local data types - * ---------- - */ - -/* Context info needed for invoking a recursive querytree display routine */ -typedef struct -{ - StringInfo buf; /* output buffer to append to */ - List *namespaces; /* List of deparse_namespace nodes */ - List *windowClause; /* Current query level's WINDOW clause */ - List *windowTList; /* targetlist for resolving WINDOW clause */ - int prettyFlags; /* enabling of pretty-print functions */ - int wrapColumn; /* max line length, or -1 for no limit */ - int indentLevel; /* current indent level for prettyprint */ - bool varprefix; /* true to print prefixes on Vars */ - Oid distrelid; /* the distributed table being modified, if valid */ - int64 shardid; /* a distributed table's shardid, if positive */ - ParseExprKind special_exprkind; /* set only for exprkinds needing special - * handling */ - Bitmapset *appendparents; /* if not null, map child Vars of these relids - * back to the parent rel */ -} deparse_context; - -/* - * Each level of query context around a subtree needs a level of Var namespace. - * A Var having varlevelsup=N refers to the N'th item (counting from 0) in - * the current context's namespaces list. - * - * The rangetable is the list of actual RTEs from the query tree, and the - * cte list is the list of actual CTEs. - * - * rtable_names holds the alias name to be used for each RTE (either a C - * string, or NULL for nameless RTEs such as unnamed joins). - * rtable_columns holds the column alias names to be used for each RTE. - * - * In some cases we need to make names of merged JOIN USING columns unique - * across the whole query, not only per-RTE. If so, unique_using is true - * and using_names is a list of C strings representing names already assigned - * to USING columns. - * - * When deparsing plan trees, there is always just a single item in the - * deparse_namespace list (since a plan tree never contains Vars with - * varlevelsup > 0). We store the PlanState node that is the immediate - * parent of the expression to be deparsed, as well as a list of that - * PlanState's ancestors. In addition, we store its outer and inner subplan - * state nodes, as well as their plan nodes' targetlists, and the index tlist - * if the current plan node might contain INDEX_VAR Vars. (These fields could - * be derived on-the-fly from the current PlanState, but it seems notationally - * clearer to set them up as separate fields.) - */ -typedef struct -{ - List *rtable; /* List of RangeTblEntry nodes */ - List *rtable_names; /* Parallel list of names for RTEs */ - List *rtable_columns; /* Parallel list of deparse_columns structs */ - List *subplans; /* List of Plan trees for SubPlans */ - List *ctes; /* List of CommonTableExpr nodes */ - AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ - /* Workspace for column alias assignment: */ - bool unique_using; /* Are we making USING names globally unique */ - List *using_names; /* List of assigned names for USING columns */ - /* Remaining fields are used only when deparsing a Plan tree: */ - Plan *plan; /* immediate parent of current expression */ - List *ancestors; /* ancestors of planstate */ - Plan *outer_plan; /* outer subnode, or NULL if none */ - Plan *inner_plan; /* inner subnode, or NULL if none */ - List *outer_tlist; /* referent for OUTER_VAR Vars */ - List *inner_tlist; /* referent for INNER_VAR Vars */ - List *index_tlist; /* referent for INDEX_VAR Vars */ - /* Special namespace representing a function signature: */ - char *funcname; - int numargs; - char **argnames; -} deparse_namespace; - -/* Callback signature for resolve_special_varno() */ -typedef void (*rsv_callback) (Node *node, deparse_context *context, - void *callback_arg); - -/* - * Per-relation data about column alias names. - * - * Selecting aliases is unreasonably complicated because of the need to dump - * rules/views whose underlying tables may have had columns added, deleted, or - * renamed since the query was parsed. We must nonetheless print the rule/view - * in a form that can be reloaded and will produce the same results as before. - * - * For each RTE used in the query, we must assign column aliases that are - * unique within that RTE. SQL does not require this of the original query, - * but due to factors such as *-expansion we need to be able to uniquely - * reference every column in a decompiled query. As long as we qualify all - * column references, per-RTE uniqueness is sufficient for that. - * - * However, we can't ensure per-column name uniqueness for unnamed join RTEs, - * since they just inherit column names from their input RTEs, and we can't - * rename the columns at the join level. Most of the time this isn't an issue - * because we don't need to reference the join's output columns as such; we - * can reference the input columns instead. That approach can fail for merged - * JOIN USING columns, however, so when we have one of those in an unnamed - * join, we have to make that column's alias globally unique across the whole - * query to ensure it can be referenced unambiguously. - * - * Another problem is that a JOIN USING clause requires the columns to be - * merged to have the same aliases in both input RTEs, and that no other - * columns in those RTEs or their children conflict with the USING names. - * To handle that, we do USING-column alias assignment in a recursive - * traversal of the query's jointree. When descending through a JOIN with - * USING, we preassign the USING column names to the child columns, overriding - * other rules for column alias assignment. We also mark each RTE with a list - * of all USING column names selected for joins containing that RTE, so that - * when we assign other columns' aliases later, we can avoid conflicts. - * - * Another problem is that if a JOIN's input tables have had columns added or - * deleted since the query was parsed, we must generate a column alias list - * for the join that matches the current set of input columns --- otherwise, a - * change in the number of columns in the left input would throw off matching - * of aliases to columns of the right input. Thus, positions in the printable - * column alias list are not necessarily one-for-one with varattnos of the - * JOIN, so we need a separate new_colnames[] array for printing purposes. - */ -typedef struct -{ - /* - * colnames is an array containing column aliases to use for columns that - * existed when the query was parsed. Dropped columns have NULL entries. - * This array can be directly indexed by varattno to get a Var's name. - * - * Non-NULL entries are guaranteed unique within the RTE, *except* when - * this is for an unnamed JOIN RTE. In that case we merely copy up names - * from the two input RTEs. - * - * During the recursive descent in set_using_names(), forcible assignment - * of a child RTE's column name is represented by pre-setting that element - * of the child's colnames array. So at that stage, NULL entries in this - * array just mean that no name has been preassigned, not necessarily that - * the column is dropped. - */ - int num_cols; /* length of colnames[] array */ - char **colnames; /* array of C strings and NULLs */ - - /* - * new_colnames is an array containing column aliases to use for columns - * that would exist if the query was re-parsed against the current - * definitions of its base tables. This is what to print as the column - * alias list for the RTE. This array does not include dropped columns, - * but it will include columns added since original parsing. Indexes in - * it therefore have little to do with current varattno values. As above, - * entries are unique unless this is for an unnamed JOIN RTE. (In such an - * RTE, we never actually print this array, but we must compute it anyway - * for possible use in computing column names of upper joins.) The - * parallel array is_new_col marks which of these columns are new since - * original parsing. Entries with is_new_col false must match the - * non-NULL colnames entries one-for-one. - */ - int num_new_cols; /* length of new_colnames[] array */ - char **new_colnames; /* array of C strings */ - bool *is_new_col; /* array of bool flags */ - - /* This flag tells whether we should actually print a column alias list */ - bool printaliases; - - /* This list has all names used as USING names in joins above this RTE */ - List *parentUsing; /* names assigned to parent merged columns */ - - /* - * If this struct is for a JOIN RTE, we fill these fields during the - * set_using_names() pass to describe its relationship to its child RTEs. - * - * leftattnos and rightattnos are arrays with one entry per existing - * output column of the join (hence, indexable by join varattno). For a - * simple reference to a column of the left child, leftattnos[i] is the - * child RTE's attno and rightattnos[i] is zero; and conversely for a - * column of the right child. But for merged columns produced by JOIN - * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. - * Also, if the column has been dropped, both are zero. - * - * If it's a JOIN USING, usingNames holds the alias names selected for the - * merged columns (these might be different from the original USING list, - * if we had to modify names to achieve uniqueness). - */ - int leftrti; /* rangetable index of left child */ - int rightrti; /* rangetable index of right child */ - int *leftattnos; /* left-child varattnos of join cols, or 0 */ - int *rightattnos; /* right-child varattnos of join cols, or 0 */ - List *usingNames; /* names assigned to merged columns */ -} deparse_columns; - -/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ -#define deparse_columns_fetch(rangetable_index, dpns) \ - ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) - -/* - * Entry in set_rtable_names' hash table - */ -typedef struct -{ - char name[NAMEDATALEN]; /* Hash key --- must be first */ - int counter; /* Largest addition used so far for name */ -} NameHashEntry; - - -/* ---------- - * Local functions - * - * Most of these functions used to use fixed-size buffers to build their - * results. Now, they take an (already initialized) StringInfo object - * as a parameter, and append their text output to its contents. - * ---------- - */ -static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, - Bitmapset *rels_used); -static void set_deparse_for_query(deparse_namespace *dpns, Query *query, - List *parent_namespaces); -static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); -static void set_using_names(deparse_namespace *dpns, Node *jtnode, - List *parentUsing); -static void set_relation_column_names(deparse_namespace *dpns, - RangeTblEntry *rte, - deparse_columns *colinfo); -static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, - deparse_columns *colinfo); -static bool colname_is_unique(const char *colname, deparse_namespace *dpns, - deparse_columns *colinfo); -static char *make_colname_unique(char *colname, deparse_namespace *dpns, - deparse_columns *colinfo); -static void expand_colnames_array_to(deparse_columns *colinfo, int n); -static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, - deparse_columns *colinfo); -static char *get_rtable_name(int rtindex, deparse_context *context); -static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); -static void push_child_plan(deparse_namespace *dpns, Plan *plan, - deparse_namespace *save_dpns); -static void pop_child_plan(deparse_namespace *dpns, - deparse_namespace *save_dpns); -static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, - deparse_namespace *save_dpns); -static void pop_ancestor_plan(deparse_namespace *dpns, - deparse_namespace *save_dpns); -static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, - TupleDesc resultDesc, - int prettyFlags, int wrapColumn, int startIndent); -static void get_query_def_extended(Query *query, StringInfo buf, - List *parentnamespace, Oid distrelid, int64 shardid, - TupleDesc resultDesc, int prettyFlags, int wrapColumn, - int startIndent); -static void get_values_def(List *values_lists, deparse_context *context); -static void get_with_clause(Query *query, deparse_context *context); -static void get_select_query_def(Query *query, deparse_context *context, - TupleDesc resultDesc); -static void get_insert_query_def(Query *query, deparse_context *context); -static void get_update_query_def(Query *query, deparse_context *context); -static void get_update_query_targetlist_def(Query *query, List *targetList, - deparse_context *context, - RangeTblEntry *rte); -static void get_delete_query_def(Query *query, deparse_context *context); -static void get_utility_query_def(Query *query, deparse_context *context); -static void get_basic_select_query(Query *query, deparse_context *context, - TupleDesc resultDesc); -static void get_target_list(List *targetList, deparse_context *context, - TupleDesc resultDesc); -static void get_setop_query(Node *setOp, Query *query, - deparse_context *context, - TupleDesc resultDesc); -static Node *get_rule_sortgroupclause(Index ref, List *tlist, - bool force_colno, - deparse_context *context); -static void get_rule_groupingset(GroupingSet *gset, List *targetlist, - bool omit_parens, deparse_context *context); -static void get_rule_orderby(List *orderList, List *targetList, - bool force_colno, deparse_context *context); -static void get_rule_windowclause(Query *query, deparse_context *context); -static void get_rule_windowspec(WindowClause *wc, List *targetList, - deparse_context *context); -static char *get_variable(Var *var, int levelsup, bool istoplevel, - deparse_context *context); -static void get_special_variable(Node *node, deparse_context *context, - void *callback_arg); -static void resolve_special_varno(Node *node, deparse_context *context, - rsv_callback callback, void *callback_arg); -static Node *find_param_referent(Param *param, deparse_context *context, - deparse_namespace **dpns_p, ListCell **ancestor_cell_p); -static void get_parameter(Param *param, deparse_context *context); -static const char *get_simple_binary_op_name(OpExpr *expr); -static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); -static void appendContextKeyword(deparse_context *context, const char *str, - int indentBefore, int indentAfter, int indentPlus); -static void removeStringInfoSpaces(StringInfo str); -static void get_rule_expr(Node *node, deparse_context *context, - bool showimplicit); -static void get_rule_expr_toplevel(Node *node, deparse_context *context, - bool showimplicit); -static void get_rule_expr_funccall(Node *node, deparse_context *context, - bool showimplicit); -static bool looks_like_function(Node *node); -static void get_oper_expr(OpExpr *expr, deparse_context *context); -static void get_func_expr(FuncExpr *expr, deparse_context *context, - bool showimplicit); -static void get_proc_expr(CallStmt *stmt, deparse_context *context, - bool showimplicit); -static void get_agg_expr(Aggref *aggref, deparse_context *context, - Aggref *original_aggref); -static void get_agg_combine_expr(Node *node, deparse_context *context, - void *callback_arg); -static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); -static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); -static void get_coercion_expr(Node *arg, deparse_context *context, - Oid resulttype, int32 resulttypmod, - Node *parentNode); -static void get_const_expr(Const *constval, deparse_context *context, - int showtype); -static void get_const_collation(Const *constval, deparse_context *context); -static void simple_quote_literal(StringInfo buf, const char *val); -static void get_sublink_expr(SubLink *sublink, deparse_context *context); -static void get_tablefunc(TableFunc *tf, deparse_context *context, - bool showimplicit); -static void get_from_clause(Query *query, const char *prefix, - deparse_context *context); -static void get_from_clause_item(Node *jtnode, Query *query, - deparse_context *context); -static void get_column_alias_list(deparse_columns *colinfo, - deparse_context *context); -static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, - deparse_columns *colinfo, - deparse_context *context); -static void get_tablesample_def(TableSampleClause *tablesample, - deparse_context *context); -static void get_opclass_name(Oid opclass, Oid actual_datatype, - StringInfo buf); -static Node *processIndirection(Node *node, deparse_context *context); -static void printSubscripts(SubscriptingRef *aref, deparse_context *context); -static char *get_relation_name(Oid relid); -static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, - int64 shardid, List *namespaces); -static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); -static char *generate_fragment_name(char *schemaName, char *tableName); -static char *generate_function_name(Oid funcid, int nargs, - List *argnames, Oid *argtypes, - bool has_variadic, bool *use_variadic_p, - ParseExprKind special_exprkind); - -#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") - - - -/* - * pg_get_query_def parses back one query tree, and outputs the resulting query - * string into given buffer. - */ -void -pg_get_query_def(Query *query, StringInfo buffer) -{ - get_query_def(query, buffer, NIL, NULL, 0, WRAP_COLUMN_DEFAULT, 0); -} - -/* - * get_merged_argument_list merges both the IN and OUT arguments lists into one and - * also eliminates the INOUT duplicates(present in both the lists). After merging both - * the lists, it returns all the named-arguments in a list(mergedNamedArgList) along - * with their types(mergedNamedArgTypes), final argument list(mergedArgumentList), and - * the total number of arguments(totalArguments). - */ -bool -get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, - Oid **mergedNamedArgTypes, - List **mergedArgumentList, - int *totalArguments) -{ - - Oid functionOid = stmt->funcexpr->funcid; - List *namedArgList = NIL; - List *finalArgumentList = NIL; - Oid *finalArgTypes; - Oid *argTypes = NULL; - char *argModes = NULL; - char **argNames = NULL; - int argIndex = 0; - - HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); - if (!HeapTupleIsValid(proctup)) - { - elog(ERROR, "cache lookup failed for function %u", functionOid); - } - - int defArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); - ReleaseSysCache(proctup); - - if (argModes == NULL) - { - /* No OUT arguments */ - return false; - } - - /* - * Passed arguments Includes IN, OUT, INOUT (in both the lists) and VARIADIC arguments, - * which means INOUT arguments are double counted. - */ - int numberOfArgs = list_length(stmt->funcexpr->args) + list_length(stmt->outargs); - int totalInoutArgs = 0; - - /* Let's count INOUT arguments from the defined number of arguments */ - for (argIndex=0; argIndex < defArgs; ++argIndex) - { - if (argModes[argIndex] == PROARGMODE_INOUT) - totalInoutArgs++; - } - - /* Remove the duplicate INOUT counting */ - numberOfArgs = numberOfArgs - totalInoutArgs; - finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs); - - ListCell *inArgCell = list_head(stmt->funcexpr->args); - ListCell *outArgCell = list_head(stmt->outargs); - - for (argIndex=0; argIndex < numberOfArgs; ++argIndex) - { - switch (argModes[argIndex]) - { - case PROARGMODE_IN: - case PROARGMODE_VARIADIC: - { - Node *arg = (Node *) lfirst(inArgCell); - - if (IsA(arg, NamedArgExpr)) - namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); - finalArgTypes[argIndex] = exprType(arg); - finalArgumentList = lappend(finalArgumentList, arg); - inArgCell = lnext(stmt->funcexpr->args, inArgCell); - break; - } - - case PROARGMODE_OUT: - { - Node *arg = (Node *) lfirst(outArgCell); - - if (IsA(arg, NamedArgExpr)) - namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); - finalArgTypes[argIndex] = exprType(arg); - finalArgumentList = lappend(finalArgumentList, arg); - outArgCell = lnext(stmt->outargs, outArgCell); - break; - } - - case PROARGMODE_INOUT: - { - Node *arg = (Node *) lfirst(inArgCell); - - if (IsA(arg, NamedArgExpr)) - namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); - finalArgTypes[argIndex] = exprType(arg); - finalArgumentList = lappend(finalArgumentList, arg); - inArgCell = lnext(stmt->funcexpr->args, inArgCell); - outArgCell = lnext(stmt->outargs, outArgCell); - break; - } - - case PROARGMODE_TABLE: - default: - { - elog(ERROR, "Unhandled procedure argument mode[%d]", argModes[argIndex]); - break; - } - } - } - - /* - * After eliminating INOUT duplicates and merging OUT arguments, we now - * have the final list of arguments. - */ - if (defArgs != list_length(finalArgumentList)) - { - elog(ERROR, "Insufficient number of args passed[%d] for function[%s]", - list_length(finalArgumentList), - get_func_name(functionOid)); - } - - if (list_length(finalArgumentList) > FUNC_MAX_ARGS) - { - ereport(ERROR, - (errcode(ERRCODE_TOO_MANY_ARGUMENTS), - errmsg("too many arguments[%d] for function[%s]", - list_length(finalArgumentList), - get_func_name(functionOid)))); - } - - *mergedNamedArgList = namedArgList; - *mergedNamedArgTypes = finalArgTypes; - *mergedArgumentList = finalArgumentList; - *totalArguments = numberOfArgs; - - return true; -} -/* - * pg_get_rule_expr deparses an expression and returns the result as a string. - */ -char * -pg_get_rule_expr(Node *expression) -{ - bool showImplicitCasts = true; - deparse_context context; - StringInfo buffer = makeStringInfo(); - - /* - * Set search_path to NIL so that all objects outside of pg_catalog will be - * schema-prefixed. pg_catalog will be added automatically when we call - * PushEmptySearchPath(). - */ - int saveNestLevel = PushEmptySearchPath(); - - context.buf = buffer; - context.namespaces = NIL; - context.windowClause = NIL; - context.windowTList = NIL; - context.varprefix = false; - context.prettyFlags = 0; - context.wrapColumn = WRAP_COLUMN_DEFAULT; - context.indentLevel = 0; - context.special_exprkind = EXPR_KIND_NONE; - context.distrelid = InvalidOid; - context.shardid = INVALID_SHARD_ID; - - get_rule_expr(expression, &context, showImplicitCasts); - - /* revert back to original search_path */ - PopEmptySearchPath(saveNestLevel); - - return buffer->data; -} - - -/* - * set_rtable_names: select RTE aliases to be used in printing a query - * - * We fill in dpns->rtable_names with a list of names that is one-for-one with - * the already-filled dpns->rtable list. Each RTE name is unique among those - * in the new namespace plus any ancestor namespaces listed in - * parent_namespaces. - * - * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. - * - * Note that this function is only concerned with relation names, not column - * names. - */ -static void -set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, - Bitmapset *rels_used) -{ - HASHCTL hash_ctl; - HTAB *names_hash; - NameHashEntry *hentry; - bool found; - int rtindex; - ListCell *lc; - - dpns->rtable_names = NIL; - /* nothing more to do if empty rtable */ - if (dpns->rtable == NIL) - return; - - /* - * We use a hash table to hold known names, so that this process is O(N) - * not O(N^2) for N names. - */ - hash_ctl.keysize = NAMEDATALEN; - hash_ctl.entrysize = sizeof(NameHashEntry); - hash_ctl.hcxt = CurrentMemoryContext; - names_hash = hash_create("set_rtable_names names", - list_length(dpns->rtable), - &hash_ctl, - HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); - - /* Preload the hash table with names appearing in parent_namespaces */ - foreach(lc, parent_namespaces) - { - deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); - ListCell *lc2; - - foreach(lc2, olddpns->rtable_names) - { - char *oldname = (char *) lfirst(lc2); - - if (oldname == NULL) - continue; - hentry = (NameHashEntry *) hash_search(names_hash, - oldname, - HASH_ENTER, - &found); - /* we do not complain about duplicate names in parent namespaces */ - hentry->counter = 0; - } - } - - /* Now we can scan the rtable */ - rtindex = 1; - foreach(lc, dpns->rtable) - { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); - char *refname; - - /* Just in case this takes an unreasonable amount of time ... */ - CHECK_FOR_INTERRUPTS(); - - if (rels_used && !bms_is_member(rtindex, rels_used)) - { - /* Ignore unreferenced RTE */ - refname = NULL; - } - else if (rte->alias) - { - /* If RTE has a user-defined alias, prefer that */ - refname = rte->alias->aliasname; - } - else if (rte->rtekind == RTE_RELATION) - { - /* Use the current actual name of the relation */ - refname = get_rel_name(rte->relid); - } - else if (rte->rtekind == RTE_JOIN) - { - /* Unnamed join has no refname */ - refname = NULL; - } - else - { - /* Otherwise use whatever the parser assigned */ - refname = rte->eref->aliasname; - } - - /* - * If the selected name isn't unique, append digits to make it so, and - * make a new hash entry for it once we've got a unique name. For a - * very long input name, we might have to truncate to stay within - * NAMEDATALEN. - */ - if (refname) - { - hentry = (NameHashEntry *) hash_search(names_hash, - refname, - HASH_ENTER, - &found); - if (found) - { - /* Name already in use, must choose a new one */ - int refnamelen = strlen(refname); - char *modname = (char *) palloc(refnamelen + 16); - NameHashEntry *hentry2; - - do - { - hentry->counter++; - for (;;) - { - memcpy(modname, refname, refnamelen); - sprintf(modname + refnamelen, "_%d", hentry->counter); - if (strlen(modname) < NAMEDATALEN) - break; - /* drop chars from refname to keep all the digits */ - refnamelen = pg_mbcliplen(refname, refnamelen, - refnamelen - 1); - } - hentry2 = (NameHashEntry *) hash_search(names_hash, - modname, - HASH_ENTER, - &found); - } while (found); - hentry2->counter = 0; /* init new hash entry */ - refname = modname; - } - else - { - /* Name not previously used, need only initialize hentry */ - hentry->counter = 0; - } - } - - dpns->rtable_names = lappend(dpns->rtable_names, refname); - rtindex++; - } - - hash_destroy(names_hash); -} - -/* - * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree - * - * For convenience, this is defined to initialize the deparse_namespace struct - * from scratch. - */ -static void -set_deparse_for_query(deparse_namespace *dpns, Query *query, - List *parent_namespaces) -{ - ListCell *lc; - ListCell *lc2; - - /* Initialize *dpns and fill rtable/ctes links */ - memset(dpns, 0, sizeof(deparse_namespace)); - dpns->rtable = query->rtable; - dpns->subplans = NIL; - dpns->ctes = query->cteList; - dpns->appendrels = NULL; - - /* Assign a unique relation alias to each RTE */ - set_rtable_names(dpns, parent_namespaces, NULL); - - /* Initialize dpns->rtable_columns to contain zeroed structs */ - dpns->rtable_columns = NIL; - while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) - dpns->rtable_columns = lappend(dpns->rtable_columns, - palloc0(sizeof(deparse_columns))); - - /* If it's a utility query, it won't have a jointree */ - if (query->jointree) - { - /* Detect whether global uniqueness of USING names is needed */ - dpns->unique_using = - has_dangerous_join_using(dpns, (Node *) query->jointree); - - /* - * Select names for columns merged by USING, via a recursive pass over - * the query jointree. - */ - set_using_names(dpns, (Node *) query->jointree, NIL); - } - - /* - * Now assign remaining column aliases for each RTE. We do this in a - * linear scan of the rtable, so as to process RTEs whether or not they - * are in the jointree (we mustn't miss NEW.*, INSERT target relations, - * etc). JOIN RTEs must be processed after their children, but this is - * okay because they appear later in the rtable list than their children - * (cf Asserts in identify_join_columns()). - */ - forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) - { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); - deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); - - if (rte->rtekind == RTE_JOIN) - set_join_column_names(dpns, rte, colinfo); - else - set_relation_column_names(dpns, rte, colinfo); - } -} - -/* - * has_dangerous_join_using: search jointree for unnamed JOIN USING - * - * Merged columns of a JOIN USING may act differently from either of the input - * columns, either because they are merged with COALESCE (in a FULL JOIN) or - * because an implicit coercion of the underlying input column is required. - * In such a case the column must be referenced as a column of the JOIN not as - * a column of either input. And this is problematic if the join is unnamed - * (alias-less): we cannot qualify the column's name with an RTE name, since - * there is none. (Forcibly assigning an alias to the join is not a solution, - * since that will prevent legal references to tables below the join.) - * To ensure that every column in the query is unambiguously referenceable, - * we must assign such merged columns names that are globally unique across - * the whole query, aliasing other columns out of the way as necessary. - * - * Because the ensuing re-aliasing is fairly damaging to the readability of - * the query, we don't do this unless we have to. So, we must pre-scan - * the join tree to see if we have to, before starting set_using_names(). - */ -static bool -has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) -{ - if (IsA(jtnode, RangeTblRef)) - { - /* nothing to do here */ - } - else if (IsA(jtnode, FromExpr)) - { - FromExpr *f = (FromExpr *) jtnode; - ListCell *lc; - - foreach(lc, f->fromlist) - { - if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) - return true; - } - } - else if (IsA(jtnode, JoinExpr)) - { - JoinExpr *j = (JoinExpr *) jtnode; - - /* Is it an unnamed JOIN with USING? */ - if (j->alias == NULL && j->usingClause) - { - /* - * Yes, so check each join alias var to see if any of them are not - * simple references to underlying columns. If so, we have a - * dangerous situation and must pick unique aliases. - */ - RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); - - /* We need only examine the merged columns */ - for (int i = 0; i < jrte->joinmergedcols; i++) - { - Node *aliasvar = list_nth(jrte->joinaliasvars, i); - - if (!IsA(aliasvar, Var)) - return true; - } - } - - /* Nope, but inspect children */ - if (has_dangerous_join_using(dpns, j->larg)) - return true; - if (has_dangerous_join_using(dpns, j->rarg)) - return true; - } - else - elog(ERROR, "unrecognized node type: %d", - (int) nodeTag(jtnode)); - return false; -} - -/* - * set_using_names: select column aliases to be used for merged USING columns - * - * We do this during a recursive descent of the query jointree. - * dpns->unique_using must already be set to determine the global strategy. - * - * Column alias info is saved in the dpns->rtable_columns list, which is - * assumed to be filled with pre-zeroed deparse_columns structs. - * - * parentUsing is a list of all USING aliases assigned in parent joins of - * the current jointree node. (The passed-in list must not be modified.) - */ -static void -set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) -{ - if (IsA(jtnode, RangeTblRef)) - { - /* nothing to do now */ - } - else if (IsA(jtnode, FromExpr)) - { - FromExpr *f = (FromExpr *) jtnode; - ListCell *lc; - - foreach(lc, f->fromlist) - set_using_names(dpns, (Node *) lfirst(lc), parentUsing); - } - else if (IsA(jtnode, JoinExpr)) - { - JoinExpr *j = (JoinExpr *) jtnode; - RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); - deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); - int *leftattnos; - int *rightattnos; - deparse_columns *leftcolinfo; - deparse_columns *rightcolinfo; - int i; - ListCell *lc; - - /* Get info about the shape of the join */ - identify_join_columns(j, rte, colinfo); - leftattnos = colinfo->leftattnos; - rightattnos = colinfo->rightattnos; - - /* Look up the not-yet-filled-in child deparse_columns structs */ - leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); - rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); - - /* - * If this join is unnamed, then we cannot substitute new aliases at - * this level, so any name requirements pushed down to here must be - * pushed down again to the children. - */ - if (rte->alias == NULL) - { - for (i = 0; i < colinfo->num_cols; i++) - { - char *colname = colinfo->colnames[i]; - - if (colname == NULL) - continue; - - /* Push down to left column, unless it's a system column */ - if (leftattnos[i] > 0) - { - expand_colnames_array_to(leftcolinfo, leftattnos[i]); - leftcolinfo->colnames[leftattnos[i] - 1] = colname; - } - - /* Same on the righthand side */ - if (rightattnos[i] > 0) - { - expand_colnames_array_to(rightcolinfo, rightattnos[i]); - rightcolinfo->colnames[rightattnos[i] - 1] = colname; - } - } - } - - /* - * If there's a USING clause, select the USING column names and push - * those names down to the children. We have two strategies: - * - * If dpns->unique_using is true, we force all USING names to be - * unique across the whole query level. In principle we'd only need - * the names of dangerous USING columns to be globally unique, but to - * safely assign all USING names in a single pass, we have to enforce - * the same uniqueness rule for all of them. However, if a USING - * column's name has been pushed down from the parent, we should use - * it as-is rather than making a uniqueness adjustment. This is - * necessary when we're at an unnamed join, and it creates no risk of - * ambiguity. Also, if there's a user-written output alias for a - * merged column, we prefer to use that rather than the input name; - * this simplifies the logic and seems likely to lead to less aliasing - * overall. - * - * If dpns->unique_using is false, we only need USING names to be - * unique within their own join RTE. We still need to honor - * pushed-down names, though. - * - * Though significantly different in results, these two strategies are - * implemented by the same code, with only the difference of whether - * to put assigned names into dpns->using_names. - */ - if (j->usingClause) - { - /* Copy the input parentUsing list so we don't modify it */ - parentUsing = list_copy(parentUsing); - - /* USING names must correspond to the first join output columns */ - expand_colnames_array_to(colinfo, list_length(j->usingClause)); - i = 0; - foreach(lc, j->usingClause) - { - char *colname = strVal(lfirst(lc)); - - /* Assert it's a merged column */ - Assert(leftattnos[i] != 0 && rightattnos[i] != 0); - - /* Adopt passed-down name if any, else select unique name */ - if (colinfo->colnames[i] != NULL) - colname = colinfo->colnames[i]; - else - { - /* Prefer user-written output alias if any */ - if (rte->alias && i < list_length(rte->alias->colnames)) - colname = strVal(list_nth(rte->alias->colnames, i)); - /* Make it appropriately unique */ - colname = make_colname_unique(colname, dpns, colinfo); - if (dpns->unique_using) - dpns->using_names = lappend(dpns->using_names, - colname); - /* Save it as output column name, too */ - colinfo->colnames[i] = colname; - } - - /* Remember selected names for use later */ - colinfo->usingNames = lappend(colinfo->usingNames, colname); - parentUsing = lappend(parentUsing, colname); - - /* Push down to left column, unless it's a system column */ - if (leftattnos[i] > 0) - { - expand_colnames_array_to(leftcolinfo, leftattnos[i]); - leftcolinfo->colnames[leftattnos[i] - 1] = colname; - } - - /* Same on the righthand side */ - if (rightattnos[i] > 0) - { - expand_colnames_array_to(rightcolinfo, rightattnos[i]); - rightcolinfo->colnames[rightattnos[i] - 1] = colname; - } - - i++; - } - } - - /* Mark child deparse_columns structs with correct parentUsing info */ - leftcolinfo->parentUsing = parentUsing; - rightcolinfo->parentUsing = parentUsing; - - /* Now recursively assign USING column names in children */ - set_using_names(dpns, j->larg, parentUsing); - set_using_names(dpns, j->rarg, parentUsing); - } - else - elog(ERROR, "unrecognized node type: %d", - (int) nodeTag(jtnode)); -} - -/* - * set_relation_column_names: select column aliases for a non-join RTE - * - * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. - * If any colnames entries are already filled in, those override local - * choices. - */ -static void -set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, - deparse_columns *colinfo) -{ - int ncolumns; - char **real_colnames; - bool changed_any; - bool has_anonymous; - int noldcolumns; - int i; - int j; - - /* - * Extract the RTE's "real" column names. This is comparable to - * get_rte_attribute_name, except that it's important to disregard dropped - * columns. We put NULL into the array for a dropped column. - */ - if (rte->rtekind == RTE_RELATION || - GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - /* Relation --- look to the system catalogs for up-to-date info */ - Relation rel; - TupleDesc tupdesc; - - rel = relation_open(rte->relid, AccessShareLock); - tupdesc = RelationGetDescr(rel); - - ncolumns = tupdesc->natts; - real_colnames = (char **) palloc(ncolumns * sizeof(char *)); - - for (i = 0; i < ncolumns; i++) - { - Form_pg_attribute attr = TupleDescAttr(tupdesc, i); - - if (attr->attisdropped) - real_colnames[i] = NULL; - else - real_colnames[i] = pstrdup(NameStr(attr->attname)); - } - relation_close(rel, AccessShareLock); - } - else - { - /* Otherwise use the column names from eref */ - ListCell *lc; - - ncolumns = list_length(rte->eref->colnames); - real_colnames = (char **) palloc(ncolumns * sizeof(char *)); - - i = 0; - foreach(lc, rte->eref->colnames) - { - /* - * If the column name shown in eref is an empty string, then it's - * a column that was dropped at the time of parsing the query, so - * treat it as dropped. - */ - char *cname = strVal(lfirst(lc)); - - if (cname[0] == '\0') - cname = NULL; - real_colnames[i] = cname; - i++; - } - } - - /* - * Ensure colinfo->colnames has a slot for each column. (It could be long - * enough already, if we pushed down a name for the last column.) Note: - * it's possible that there are now more columns than there were when the - * query was parsed, ie colnames could be longer than rte->eref->colnames. - * We must assign unique aliases to the new columns too, else there could - * be unresolved conflicts when the view/rule is reloaded. - */ - expand_colnames_array_to(colinfo, ncolumns); - Assert(colinfo->num_cols == ncolumns); - - /* - * Make sufficiently large new_colnames and is_new_col arrays, too. - * - * Note: because we leave colinfo->num_new_cols zero until after the loop, - * colname_is_unique will not consult that array, which is fine because it - * would only be duplicate effort. - */ - colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); - colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); - - /* - * Scan the columns, select a unique alias for each one, and store it in - * colinfo->colnames and colinfo->new_colnames. The former array has NULL - * entries for dropped columns, the latter omits them. Also mark - * new_colnames entries as to whether they are new since parse time; this - * is the case for entries beyond the length of rte->eref->colnames. - */ - noldcolumns = list_length(rte->eref->colnames); - changed_any = false; - has_anonymous = false; - j = 0; - for (i = 0; i < ncolumns; i++) - { - char *real_colname = real_colnames[i]; - char *colname = colinfo->colnames[i]; - - /* Skip dropped columns */ - if (real_colname == NULL) - { - Assert(colname == NULL); /* colnames[i] is already NULL */ - continue; - } - - /* If alias already assigned, that's what to use */ - if (colname == NULL) - { - /* If user wrote an alias, prefer that over real column name */ - if (rte->alias && i < list_length(rte->alias->colnames)) - colname = strVal(list_nth(rte->alias->colnames, i)); - else - colname = real_colname; - - /* Unique-ify and insert into colinfo */ - colname = make_colname_unique(colname, dpns, colinfo); - - colinfo->colnames[i] = colname; - } - - /* Put names of non-dropped columns in new_colnames[] too */ - colinfo->new_colnames[j] = colname; - /* And mark them as new or not */ - colinfo->is_new_col[j] = (i >= noldcolumns); - j++; - - /* Remember if any assigned aliases differ from "real" name */ - if (!changed_any && strcmp(colname, real_colname) != 0) - changed_any = true; - - /* - * Remember if there is a reference to an anonymous column as named by - * char * FigureColname(Node *node) - */ - if (!has_anonymous && strcmp(real_colname, "?column?") == 0) - has_anonymous = true; - } - - /* - * Set correct length for new_colnames[] array. (Note: if columns have - * been added, colinfo->num_cols includes them, which is not really quite - * right but is harmless, since any new columns must be at the end where - * they won't affect varattnos of pre-existing columns.) - */ - colinfo->num_new_cols = j; - - /* - * For a relation RTE, we need only print the alias column names if any - * are different from the underlying "real" names. For a function RTE, - * always emit a complete column alias list; this is to protect against - * possible instability of the default column names (eg, from altering - * parameter names). For tablefunc RTEs, we never print aliases, because - * the column names are part of the clause itself. For other RTE types, - * print if we changed anything OR if there were user-written column - * aliases (since the latter would be part of the underlying "reality"). - */ - if (rte->rtekind == RTE_RELATION) - colinfo->printaliases = changed_any; - else if (rte->rtekind == RTE_FUNCTION) - colinfo->printaliases = true; - else if (rte->rtekind == RTE_TABLEFUNC) - colinfo->printaliases = false; - else if (rte->alias && rte->alias->colnames != NIL) - colinfo->printaliases = true; - else - colinfo->printaliases = changed_any || has_anonymous; -} - -/* - * set_join_column_names: select column aliases for a join RTE - * - * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. - * If any colnames entries are already filled in, those override local - * choices. Also, names for USING columns were already chosen by - * set_using_names(). We further expect that column alias selection has been - * completed for both input RTEs. - */ -static void -set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, - deparse_columns *colinfo) -{ - deparse_columns *leftcolinfo; - deparse_columns *rightcolinfo; - bool changed_any; - int noldcolumns; - int nnewcolumns; - Bitmapset *leftmerged = NULL; - Bitmapset *rightmerged = NULL; - int i; - int j; - int ic; - int jc; - - /* Look up the previously-filled-in child deparse_columns structs */ - leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); - rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); - - /* - * Ensure colinfo->colnames has a slot for each column. (It could be long - * enough already, if we pushed down a name for the last column.) Note: - * it's possible that one or both inputs now have more columns than there - * were when the query was parsed, but we'll deal with that below. We - * only need entries in colnames for pre-existing columns. - */ - noldcolumns = list_length(rte->eref->colnames); - expand_colnames_array_to(colinfo, noldcolumns); - Assert(colinfo->num_cols == noldcolumns); - - /* - * Scan the join output columns, select an alias for each one, and store - * it in colinfo->colnames. If there are USING columns, set_using_names() - * already selected their names, so we can start the loop at the first - * non-merged column. - */ - changed_any = false; - for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) - { - char *colname = colinfo->colnames[i]; - char *real_colname; - - /* Join column must refer to at least one input column */ - Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); - - /* Get the child column name */ - if (colinfo->leftattnos[i] > 0) - real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; - else if (colinfo->rightattnos[i] > 0) - real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; - else - { - /* We're joining system columns --- use eref name */ - real_colname = strVal(list_nth(rte->eref->colnames, i)); - } - /* If child col has been dropped, no need to assign a join colname */ - if (real_colname == NULL) - { - colinfo->colnames[i] = NULL; - continue; - } - - /* In an unnamed join, just report child column names as-is */ - if (rte->alias == NULL) - { - colinfo->colnames[i] = real_colname; - continue; - } - - /* If alias already assigned, that's what to use */ - if (colname == NULL) - { - /* If user wrote an alias, prefer that over real column name */ - if (rte->alias && i < list_length(rte->alias->colnames)) - colname = strVal(list_nth(rte->alias->colnames, i)); - else - colname = real_colname; - - /* Unique-ify and insert into colinfo */ - colname = make_colname_unique(colname, dpns, colinfo); - - colinfo->colnames[i] = colname; - } - - /* Remember if any assigned aliases differ from "real" name */ - if (!changed_any && strcmp(colname, real_colname) != 0) - changed_any = true; - } - - /* - * Calculate number of columns the join would have if it were re-parsed - * now, and create storage for the new_colnames and is_new_col arrays. - * - * Note: colname_is_unique will be consulting new_colnames[] during the - * loops below, so its not-yet-filled entries must be zeroes. - */ - nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - - list_length(colinfo->usingNames); - colinfo->num_new_cols = nnewcolumns; - colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); - colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); - - /* - * Generating the new_colnames array is a bit tricky since any new columns - * added since parse time must be inserted in the right places. This code - * must match the parser, which will order a join's columns as merged - * columns first (in USING-clause order), then non-merged columns from the - * left input (in attnum order), then non-merged columns from the right - * input (ditto). If one of the inputs is itself a join, its columns will - * be ordered according to the same rule, which means newly-added columns - * might not be at the end. We can figure out what's what by consulting - * the leftattnos and rightattnos arrays plus the input is_new_col arrays. - * - * In these loops, i indexes leftattnos/rightattnos (so it's join varattno - * less one), j indexes new_colnames/is_new_col, and ic/jc have similar - * meanings for the current child RTE. - */ - - /* Handle merged columns; they are first and can't be new */ - i = j = 0; - while (i < noldcolumns && - colinfo->leftattnos[i] != 0 && - colinfo->rightattnos[i] != 0) - { - /* column name is already determined and known unique */ - colinfo->new_colnames[j] = colinfo->colnames[i]; - colinfo->is_new_col[j] = false; - - /* build bitmapsets of child attnums of merged columns */ - if (colinfo->leftattnos[i] > 0) - leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); - if (colinfo->rightattnos[i] > 0) - rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); - - i++, j++; - } - - /* Handle non-merged left-child columns */ - ic = 0; - for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) - { - char *child_colname = leftcolinfo->new_colnames[jc]; - - if (!leftcolinfo->is_new_col[jc]) - { - /* Advance ic to next non-dropped old column of left child */ - while (ic < leftcolinfo->num_cols && - leftcolinfo->colnames[ic] == NULL) - ic++; - Assert(ic < leftcolinfo->num_cols); - ic++; - /* If it is a merged column, we already processed it */ - if (bms_is_member(ic, leftmerged)) - continue; - /* Else, advance i to the corresponding existing join column */ - while (i < colinfo->num_cols && - colinfo->colnames[i] == NULL) - i++; - Assert(i < colinfo->num_cols); - Assert(ic == colinfo->leftattnos[i]); - /* Use the already-assigned name of this column */ - colinfo->new_colnames[j] = colinfo->colnames[i]; - i++; - } - else - { - /* - * Unique-ify the new child column name and assign, unless we're - * in an unnamed join, in which case just copy - */ - if (rte->alias != NULL) - { - colinfo->new_colnames[j] = - make_colname_unique(child_colname, dpns, colinfo); - if (!changed_any && - strcmp(colinfo->new_colnames[j], child_colname) != 0) - changed_any = true; - } - else - colinfo->new_colnames[j] = child_colname; - } - - colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; - j++; - } - - /* Handle non-merged right-child columns in exactly the same way */ - ic = 0; - for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) - { - char *child_colname = rightcolinfo->new_colnames[jc]; - - if (!rightcolinfo->is_new_col[jc]) - { - /* Advance ic to next non-dropped old column of right child */ - while (ic < rightcolinfo->num_cols && - rightcolinfo->colnames[ic] == NULL) - ic++; - Assert(ic < rightcolinfo->num_cols); - ic++; - /* If it is a merged column, we already processed it */ - if (bms_is_member(ic, rightmerged)) - continue; - /* Else, advance i to the corresponding existing join column */ - while (i < colinfo->num_cols && - colinfo->colnames[i] == NULL) - i++; - Assert(i < colinfo->num_cols); - Assert(ic == colinfo->rightattnos[i]); - /* Use the already-assigned name of this column */ - colinfo->new_colnames[j] = colinfo->colnames[i]; - i++; - } - else - { - /* - * Unique-ify the new child column name and assign, unless we're - * in an unnamed join, in which case just copy - */ - if (rte->alias != NULL) - { - colinfo->new_colnames[j] = - make_colname_unique(child_colname, dpns, colinfo); - if (!changed_any && - strcmp(colinfo->new_colnames[j], child_colname) != 0) - changed_any = true; - } - else - colinfo->new_colnames[j] = child_colname; - } - - colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; - j++; - } - - /* Assert we processed the right number of columns */ -#ifdef USE_ASSERT_CHECKING - for (int col_index = 0; col_index < colinfo->num_cols; col_index++) - { - /* - * In the above processing-loops, "i" advances only if - * the column is not new, check if this is a new column. - */ - if (colinfo->is_new_col[col_index]) - i++; - } - Assert(i == colinfo->num_cols); - Assert(j == nnewcolumns); -#endif - - /* - * For a named join, print column aliases if we changed any from the child - * names. Unnamed joins cannot print aliases. - */ - if (rte->alias != NULL) - colinfo->printaliases = changed_any; - else - colinfo->printaliases = false; -} - -/* - * colname_is_unique: is colname distinct from already-chosen column names? - * - * dpns is query-wide info, colinfo is for the column's RTE - */ -static bool -colname_is_unique(const char *colname, deparse_namespace *dpns, - deparse_columns *colinfo) -{ - int i; - ListCell *lc; - - /* Check against already-assigned column aliases within RTE */ - for (i = 0; i < colinfo->num_cols; i++) - { - char *oldname = colinfo->colnames[i]; - - if (oldname && strcmp(oldname, colname) == 0) - return false; - } - - /* - * If we're building a new_colnames array, check that too (this will be - * partially but not completely redundant with the previous checks) - */ - for (i = 0; i < colinfo->num_new_cols; i++) - { - char *oldname = colinfo->new_colnames[i]; - - if (oldname && strcmp(oldname, colname) == 0) - return false; - } - - /* Also check against USING-column names that must be globally unique */ - foreach(lc, dpns->using_names) - { - char *oldname = (char *) lfirst(lc); - - if (strcmp(oldname, colname) == 0) - return false; - } - - /* Also check against names already assigned for parent-join USING cols */ - foreach(lc, colinfo->parentUsing) - { - char *oldname = (char *) lfirst(lc); - - if (strcmp(oldname, colname) == 0) - return false; - } - - return true; -} - -/* - * make_colname_unique: modify colname if necessary to make it unique - * - * dpns is query-wide info, colinfo is for the column's RTE - */ -static char * -make_colname_unique(char *colname, deparse_namespace *dpns, - deparse_columns *colinfo) -{ - /* - * If the selected name isn't unique, append digits to make it so. For a - * very long input name, we might have to truncate to stay within - * NAMEDATALEN. - */ - if (!colname_is_unique(colname, dpns, colinfo)) - { - int colnamelen = strlen(colname); - char *modname = (char *) palloc(colnamelen + 16); - int i = 0; - - do - { - i++; - for (;;) - { - memcpy(modname, colname, colnamelen); - sprintf(modname + colnamelen, "_%d", i); - if (strlen(modname) < NAMEDATALEN) - break; - /* drop chars from colname to keep all the digits */ - colnamelen = pg_mbcliplen(colname, colnamelen, - colnamelen - 1); - } - } while (!colname_is_unique(modname, dpns, colinfo)); - colname = modname; - } - return colname; -} - -/* - * expand_colnames_array_to: make colinfo->colnames at least n items long - * - * Any added array entries are initialized to zero. - */ -static void -expand_colnames_array_to(deparse_columns *colinfo, int n) -{ - if (n > colinfo->num_cols) - { - if (colinfo->colnames == NULL) - colinfo->colnames = (char **) palloc0(n * sizeof(char *)); - else - { - colinfo->colnames = (char **) repalloc(colinfo->colnames, - n * sizeof(char *)); - memset(colinfo->colnames + colinfo->num_cols, 0, - (n - colinfo->num_cols) * sizeof(char *)); - } - colinfo->num_cols = n; - } -} - -/* - * identify_join_columns: figure out where columns of a join come from - * - * Fills the join-specific fields of the colinfo struct, except for - * usingNames which is filled later. - */ -static void -identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, - deparse_columns *colinfo) -{ - int numjoincols; - int jcolno; - int rcolno; - ListCell *lc; - - /* Extract left/right child RT indexes */ - if (IsA(j->larg, RangeTblRef)) - colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; - else if (IsA(j->larg, JoinExpr)) - colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; - else - elog(ERROR, "unrecognized node type in jointree: %d", - (int) nodeTag(j->larg)); - if (IsA(j->rarg, RangeTblRef)) - colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; - else if (IsA(j->rarg, JoinExpr)) - colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; - else - elog(ERROR, "unrecognized node type in jointree: %d", - (int) nodeTag(j->rarg)); - - /* Assert children will be processed earlier than join in second pass */ - Assert(colinfo->leftrti < j->rtindex); - Assert(colinfo->rightrti < j->rtindex); - - /* Initialize result arrays with zeroes */ - numjoincols = list_length(jrte->joinaliasvars); - Assert(numjoincols == list_length(jrte->eref->colnames)); - colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); - colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); - - /* - * Deconstruct RTE's joinleftcols/joinrightcols into desired format. - * Recall that the column(s) merged due to USING are the first column(s) - * of the join output. We need not do anything special while scanning - * joinleftcols, but while scanning joinrightcols we must distinguish - * merged from unmerged columns. - */ - jcolno = 0; - foreach(lc, jrte->joinleftcols) - { - int leftattno = lfirst_int(lc); - - colinfo->leftattnos[jcolno++] = leftattno; - } - rcolno = 0; - foreach(lc, jrte->joinrightcols) - { - int rightattno = lfirst_int(lc); - - if (rcolno < jrte->joinmergedcols) /* merged column? */ - colinfo->rightattnos[rcolno] = rightattno; - else - colinfo->rightattnos[jcolno++] = rightattno; - rcolno++; - } - Assert(jcolno == numjoincols); -} - -/* - * get_rtable_name: convenience function to get a previously assigned RTE alias - * - * The RTE must belong to the topmost namespace level in "context". - */ -static char * -get_rtable_name(int rtindex, deparse_context *context) -{ - deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); - - Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); - return (char *) list_nth(dpns->rtable_names, rtindex - 1); -} - -/* - * set_deparse_plan: set up deparse_namespace to parse subexpressions - * of a given Plan node - * - * This sets the plan, outer_planstate, inner_planstate, outer_tlist, - * inner_tlist, and index_tlist fields. Caller is responsible for adjusting - * the ancestors list if necessary. Note that the rtable and ctes fields do - * not need to change when shifting attention to different plan nodes in a - * single plan tree. - */ -static void -set_deparse_plan(deparse_namespace *dpns, Plan *plan) -{ - dpns->plan = plan; - - /* - * We special-case Append and MergeAppend to pretend that the first child - * plan is the OUTER referent; we have to interpret OUTER Vars in their - * tlists according to one of the children, and the first one is the most - * natural choice. - */ - if (IsA(plan, Append)) - dpns->outer_plan = linitial(((Append *) plan)->appendplans); - else if (IsA(plan, MergeAppend)) - dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); - else - dpns->outer_plan = outerPlan(plan); - - if (dpns->outer_plan) - dpns->outer_tlist = dpns->outer_plan->targetlist; - else - dpns->outer_tlist = NIL; - - /* - * For a SubqueryScan, pretend the subplan is INNER referent. (We don't - * use OUTER because that could someday conflict with the normal meaning.) - * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. - * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the - * excluded expression's tlist. (Similar to the SubqueryScan we don't want - * to reuse OUTER, it's used for RETURNING in some modify table cases, - * although not INSERT .. CONFLICT). - */ - if (IsA(plan, SubqueryScan)) - dpns->inner_plan = ((SubqueryScan *) plan)->subplan; - else if (IsA(plan, CteScan)) - dpns->inner_plan = list_nth(dpns->subplans, - ((CteScan *) plan)->ctePlanId - 1); - else if (IsA(plan, ModifyTable)) - dpns->inner_plan = plan; - else - dpns->inner_plan = innerPlan(plan); - - if (IsA(plan, ModifyTable)) - dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; - else if (dpns->inner_plan) - dpns->inner_tlist = dpns->inner_plan->targetlist; - else - dpns->inner_tlist = NIL; - - /* Set up referent for INDEX_VAR Vars, if needed */ - if (IsA(plan, IndexOnlyScan)) - dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; - else if (IsA(plan, ForeignScan)) - dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; - else if (IsA(plan, CustomScan)) - dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; - else - dpns->index_tlist = NIL; -} - -/* - * push_child_plan: temporarily transfer deparsing attention to a child plan - * - * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the - * deparse context in case the referenced expression itself uses - * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid - * affecting levelsup issues (although in a Plan tree there really shouldn't - * be any). - * - * Caller must provide a local deparse_namespace variable to save the - * previous state for pop_child_plan. - */ -static void -push_child_plan(deparse_namespace *dpns, Plan *plan, - deparse_namespace *save_dpns) -{ - /* Save state for restoration later */ - *save_dpns = *dpns; - - /* Link current plan node into ancestors list */ - dpns->ancestors = lcons(dpns->plan, dpns->ancestors); - - /* Set attention on selected child */ - set_deparse_plan(dpns, plan); -} - -/* - * pop_child_plan: undo the effects of push_child_plan - */ -static void -pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) -{ - List *ancestors; - - /* Get rid of ancestors list cell added by push_child_plan */ - ancestors = list_delete_first(dpns->ancestors); - - /* Restore fields changed by push_child_plan */ - *dpns = *save_dpns; - - /* Make sure dpns->ancestors is right (may be unnecessary) */ - dpns->ancestors = ancestors; -} - -/* - * push_ancestor_plan: temporarily transfer deparsing attention to an - * ancestor plan - * - * When expanding a Param reference, we must adjust the deparse context - * to match the plan node that contains the expression being printed; - * otherwise we'd fail if that expression itself contains a Param or - * OUTER_VAR/INNER_VAR/INDEX_VAR variable. - * - * The target ancestor is conveniently identified by the ListCell holding it - * in dpns->ancestors. - * - * Caller must provide a local deparse_namespace variable to save the - * previous state for pop_ancestor_plan. - */ -static void -push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, - deparse_namespace *save_dpns) -{ - Plan *plan = (Plan *) lfirst(ancestor_cell); - - /* Save state for restoration later */ - *save_dpns = *dpns; - - /* Build a new ancestor list with just this node's ancestors */ - dpns->ancestors = - list_copy_tail(dpns->ancestors, - list_cell_number(dpns->ancestors, ancestor_cell) + 1); - - /* Set attention on selected ancestor */ - set_deparse_plan(dpns, plan); -} - -/* - * pop_ancestor_plan: undo the effects of push_ancestor_plan - */ -static void -pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) -{ - /* Free the ancestor list made in push_ancestor_plan */ - list_free(dpns->ancestors); - - /* Restore fields changed by push_ancestor_plan */ - *dpns = *save_dpns; -} - - -/* ---------- - * deparse_shard_query - Parse back a query for execution on a shard - * - * Builds an SQL string to perform the provided query on a specific shard and - * places this string into the provided buffer. - * ---------- - */ -void -deparse_shard_query(Query *query, Oid distrelid, int64 shardid, - StringInfo buffer) -{ - get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, 0, - WRAP_COLUMN_DEFAULT, 0); -} - - -/* ---------- - * get_query_def - Parse back one query parsetree - * - * If resultDesc is not NULL, then it is the output tuple descriptor for - * the view represented by a SELECT query. - * ---------- - */ -static void -get_query_def(Query *query, StringInfo buf, List *parentnamespace, - TupleDesc resultDesc, - int prettyFlags, int wrapColumn, int startIndent) -{ - get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, - prettyFlags, wrapColumn, startIndent); -} - - -/* ---------- - * get_query_def_extended - Parse back one query parsetree, optionally - * with extension using a shard identifier. - * - * If distrelid is valid and shardid is positive, the provided shardid is added - * any time the provided relid is deparsed, so that the query may be executed - * on a placement for the given shard. - * ---------- - */ -static void -get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, - Oid distrelid, int64 shardid, TupleDesc resultDesc, - int prettyFlags, int wrapColumn, int startIndent) -{ - deparse_context context; - deparse_namespace dpns; - - /* Guard against excessively long or deeply-nested queries */ - CHECK_FOR_INTERRUPTS(); - check_stack_depth(); - - /* - * Before we begin to examine the query, acquire locks on referenced - * relations, and fix up deleted columns in JOIN RTEs. This ensures - * consistent results. Note we assume it's OK to scribble on the passed - * querytree! - * - * We are only deparsing the query (we are not about to execute it), so we - * only need AccessShareLock on the relations it mentions. - */ - AcquireRewriteLocks(query, false, false); - - /* - * Set search_path to NIL so that all objects outside of pg_catalog will be - * schema-prefixed. pg_catalog will be added automatically when we call - * PushEmptySearchPath(). - */ - int saveNestLevel = PushEmptySearchPath(); - - context.buf = buf; - context.namespaces = lcons(&dpns, list_copy(parentnamespace)); - context.windowClause = NIL; - context.windowTList = NIL; - context.varprefix = (parentnamespace != NIL || - list_length(query->rtable) != 1); - context.prettyFlags = prettyFlags; - context.wrapColumn = wrapColumn; - context.indentLevel = startIndent; - context.special_exprkind = EXPR_KIND_NONE; - context.appendparents = NULL; - context.distrelid = distrelid; - context.shardid = shardid; - - set_deparse_for_query(&dpns, query, parentnamespace); - - switch (query->commandType) - { - case CMD_SELECT: - get_select_query_def(query, &context, resultDesc); - break; - - case CMD_UPDATE: - get_update_query_def(query, &context); - break; - - case CMD_INSERT: - get_insert_query_def(query, &context); - break; - - case CMD_DELETE: - get_delete_query_def(query, &context); - break; - - case CMD_NOTHING: - appendStringInfoString(buf, "NOTHING"); - break; - - case CMD_UTILITY: - get_utility_query_def(query, &context); - break; - - default: - elog(ERROR, "unrecognized query command type: %d", - query->commandType); - break; - } - - /* revert back to original search_path */ - PopEmptySearchPath(saveNestLevel); -} - -/* ---------- - * get_values_def - Parse back a VALUES list - * ---------- - */ -static void -get_values_def(List *values_lists, deparse_context *context) -{ - StringInfo buf = context->buf; - bool first_list = true; - ListCell *vtl; - - appendStringInfoString(buf, "VALUES "); - - foreach(vtl, values_lists) - { - List *sublist = (List *) lfirst(vtl); - bool first_col = true; - ListCell *lc; - - if (first_list) - first_list = false; - else - appendStringInfoString(buf, ", "); - - appendStringInfoChar(buf, '('); - foreach(lc, sublist) - { - Node *col = (Node *) lfirst(lc); - - if (first_col) - first_col = false; - else - appendStringInfoChar(buf, ','); - - /* - * Print the value. Whole-row Vars need special treatment. - */ - get_rule_expr_toplevel(col, context, false); - } - appendStringInfoChar(buf, ')'); - } -} - -/* ---------- - * get_with_clause - Parse back a WITH clause - * ---------- - */ -static void -get_with_clause(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - const char *sep; - ListCell *l; - - if (query->cteList == NIL) - return; - - if (PRETTY_INDENT(context)) - { - context->indentLevel += PRETTYINDENT_STD; - appendStringInfoChar(buf, ' '); - } - - if (query->hasRecursive) - sep = "WITH RECURSIVE "; - else - sep = "WITH "; - foreach(l, query->cteList) - { - CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); - - appendStringInfoString(buf, sep); - appendStringInfoString(buf, quote_identifier(cte->ctename)); - if (cte->aliascolnames) - { - bool first = true; - ListCell *col; - - appendStringInfoChar(buf, '('); - foreach(col, cte->aliascolnames) - { - if (first) - first = false; - else - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, - quote_identifier(strVal(lfirst(col)))); - } - appendStringInfoChar(buf, ')'); - } - appendStringInfoString(buf, " AS "); - switch (cte->ctematerialized) - { - case CTEMaterializeDefault: - break; - case CTEMaterializeAlways: - appendStringInfoString(buf, "MATERIALIZED "); - break; - case CTEMaterializeNever: - appendStringInfoString(buf, "NOT MATERIALIZED "); - break; - } - appendStringInfoChar(buf, '('); - if (PRETTY_INDENT(context)) - appendContextKeyword(context, "", 0, 0, 0); - get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, - context->prettyFlags, context->wrapColumn, - context->indentLevel); - if (PRETTY_INDENT(context)) - appendContextKeyword(context, "", 0, 0, 0); - appendStringInfoChar(buf, ')'); - - if (cte->search_clause) - { - bool first = true; - ListCell *lc; - - appendStringInfo(buf, " SEARCH %s FIRST BY ", - cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); - - foreach(lc, cte->search_clause->search_col_list) - { - if (first) - first = false; - else - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, - quote_identifier(strVal(lfirst(lc)))); - } - - appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); - } - - if (cte->cycle_clause) - { - bool first = true; - ListCell *lc; - - appendStringInfoString(buf, " CYCLE "); - - foreach(lc, cte->cycle_clause->cycle_col_list) - { - if (first) - first = false; - else - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, - quote_identifier(strVal(lfirst(lc)))); - } - - appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); - - { - Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); - Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); - - if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && - cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) - { - appendStringInfoString(buf, " TO "); - get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); - appendStringInfoString(buf, " DEFAULT "); - get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); - } - } - - appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); - } - - sep = ", "; - } - - if (PRETTY_INDENT(context)) - { - context->indentLevel -= PRETTYINDENT_STD; - appendContextKeyword(context, "", 0, 0, 0); - } - else - appendStringInfoChar(buf, ' '); -} - -/* ---------- - * get_select_query_def - Parse back a SELECT parsetree - * ---------- - */ -static void -get_select_query_def(Query *query, deparse_context *context, - TupleDesc resultDesc) -{ - StringInfo buf = context->buf; - List *save_windowclause; - List *save_windowtlist; - bool force_colno; - ListCell *l; - - /* Insert the WITH clause if given */ - get_with_clause(query, context); - - /* Set up context for possible window functions */ - save_windowclause = context->windowClause; - context->windowClause = query->windowClause; - save_windowtlist = context->windowTList; - context->windowTList = query->targetList; - - /* - * If the Query node has a setOperations tree, then it's the top level of - * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT - * fields are interesting in the top query itself. - */ - if (query->setOperations) - { - get_setop_query(query->setOperations, query, context, resultDesc); - /* ORDER BY clauses must be simple in this case */ - force_colno = true; - } - else - { - get_basic_select_query(query, context, resultDesc); - force_colno = false; - } - - /* Add the ORDER BY clause if given */ - if (query->sortClause != NIL) - { - appendContextKeyword(context, " ORDER BY ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_orderby(query->sortClause, query->targetList, - force_colno, context); - } - - /* - * Add the LIMIT/OFFSET clauses if given. If non-default options, use the - * standard spelling of LIMIT. - */ - if (query->limitOffset != NULL) - { - appendContextKeyword(context, " OFFSET ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - get_rule_expr(query->limitOffset, context, false); - } - if (query->limitCount != NULL) - { - if (query->limitOption == LIMIT_OPTION_WITH_TIES) - { - // had to add '(' and ')' here because it fails with casting - appendContextKeyword(context, " FETCH FIRST (", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - get_rule_expr(query->limitCount, context, false); - appendStringInfoString(buf, ") ROWS WITH TIES"); - } - else - { - appendContextKeyword(context, " LIMIT ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - if (IsA(query->limitCount, Const) && - ((Const *) query->limitCount)->constisnull) - appendStringInfoString(buf, "ALL"); - else - get_rule_expr(query->limitCount, context, false); - } - } - - /* Add FOR [KEY] UPDATE/SHARE clauses if present */ - if (query->hasForUpdate) - { - foreach(l, query->rowMarks) - { - RowMarkClause *rc = (RowMarkClause *) lfirst(l); - - /* don't print implicit clauses */ - if (rc->pushedDown) - continue; - - switch (rc->strength) - { - case LCS_NONE: - /* we intentionally throw an error for LCS_NONE */ - elog(ERROR, "unrecognized LockClauseStrength %d", - (int) rc->strength); - break; - case LCS_FORKEYSHARE: - appendContextKeyword(context, " FOR KEY SHARE", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - break; - case LCS_FORSHARE: - appendContextKeyword(context, " FOR SHARE", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - break; - case LCS_FORNOKEYUPDATE: - appendContextKeyword(context, " FOR NO KEY UPDATE", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - break; - case LCS_FORUPDATE: - appendContextKeyword(context, " FOR UPDATE", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - break; - } - - appendStringInfo(buf, " OF %s", - quote_identifier(get_rtable_name(rc->rti, - context))); - if (rc->waitPolicy == LockWaitError) - appendStringInfoString(buf, " NOWAIT"); - else if (rc->waitPolicy == LockWaitSkip) - appendStringInfoString(buf, " SKIP LOCKED"); - } - } - - context->windowClause = save_windowclause; - context->windowTList = save_windowtlist; -} - -/* - * Detect whether query looks like SELECT ... FROM VALUES(); - * if so, return the VALUES RTE. Otherwise return NULL. - */ -static RangeTblEntry * -get_simple_values_rte(Query *query, TupleDesc resultDesc) -{ - RangeTblEntry *result = NULL; - ListCell *lc; - int colno; - - /* - * We want to return true even if the Query also contains OLD or NEW rule - * RTEs. So the idea is to scan the rtable and see if there is only one - * inFromCl RTE that is a VALUES RTE. - */ - foreach(lc, query->rtable) - { - RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); - - if (rte->rtekind == RTE_VALUES && rte->inFromCl) - { - if (result) - return NULL; /* multiple VALUES (probably not possible) */ - result = rte; - } - else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) - continue; /* ignore rule entries */ - else - return NULL; /* something else -> not simple VALUES */ - } - - /* - * We don't need to check the targetlist in any great detail, because - * parser/analyze.c will never generate a "bare" VALUES RTE --- they only - * appear inside auto-generated sub-queries with very restricted - * structure. However, DefineView might have modified the tlist by - * injecting new column aliases; so compare tlist resnames against the - * RTE's names to detect that. - */ - if (result) - { - ListCell *lcn; - - if (list_length(query->targetList) != list_length(result->eref->colnames)) - return NULL; /* this probably cannot happen */ - colno = 0; - forboth(lc, query->targetList, lcn, result->eref->colnames) - { - TargetEntry *tle = (TargetEntry *) lfirst(lc); - char *cname = strVal(lfirst(lcn)); - char *colname; - - if (tle->resjunk) - return NULL; /* this probably cannot happen */ - /* compute name that get_target_list would use for column */ - colno++; - if (resultDesc && colno <= resultDesc->natts) - colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); - else - colname = tle->resname; - - /* does it match the VALUES RTE? */ - if (colname == NULL || strcmp(colname, cname) != 0) - return NULL; /* column name has been changed */ - } - } - - return result; -} - -static void -get_basic_select_query(Query *query, deparse_context *context, - TupleDesc resultDesc) -{ - StringInfo buf = context->buf; - RangeTblEntry *values_rte; - char *sep; - ListCell *l; - - if (PRETTY_INDENT(context)) - { - context->indentLevel += PRETTYINDENT_STD; - appendStringInfoChar(buf, ' '); - } - - /* - * If the query looks like SELECT * FROM (VALUES ...), then print just the - * VALUES part. This reverses what transformValuesClause() did at parse - * time. - */ - values_rte = get_simple_values_rte(query, resultDesc); - if (values_rte) - { - get_values_def(values_rte->values_lists, context); - return; - } - - /* - * Build up the query string - first we say SELECT - */ - if (query->isReturn) - appendStringInfoString(buf, "RETURN"); - else - appendStringInfoString(buf, "SELECT"); - - /* Add the DISTINCT clause if given */ - if (query->distinctClause != NIL) - { - if (query->hasDistinctOn) - { - appendStringInfoString(buf, " DISTINCT ON ("); - sep = ""; - foreach(l, query->distinctClause) - { - SortGroupClause *srt = (SortGroupClause *) lfirst(l); - - appendStringInfoString(buf, sep); - get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, - false, context); - sep = ", "; - } - appendStringInfoChar(buf, ')'); - } - else - appendStringInfoString(buf, " DISTINCT"); - } - - /* Then we tell what to select (the targetlist) */ - get_target_list(query->targetList, context, resultDesc); - - /* Add the FROM clause if needed */ - get_from_clause(query, " FROM ", context); - - /* Add the WHERE clause if given */ - if (query->jointree->quals != NULL) - { - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(query->jointree->quals, context, false); - } - - /* Add the GROUP BY clause if given */ - if (query->groupClause != NULL || query->groupingSets != NULL) - { - ParseExprKind save_exprkind; - - appendContextKeyword(context, " GROUP BY ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - if (query->groupDistinct) - appendStringInfoString(buf, "DISTINCT "); - - save_exprkind = context->special_exprkind; - context->special_exprkind = EXPR_KIND_GROUP_BY; - - if (query->groupingSets == NIL) - { - sep = ""; - foreach(l, query->groupClause) - { - SortGroupClause *grp = (SortGroupClause *) lfirst(l); - - appendStringInfoString(buf, sep); - get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, - false, context); - sep = ", "; - } - } - else - { - sep = ""; - foreach(l, query->groupingSets) - { - GroupingSet *grp = lfirst(l); - - appendStringInfoString(buf, sep); - get_rule_groupingset(grp, query->targetList, true, context); - sep = ", "; - } - } - - context->special_exprkind = save_exprkind; - } - - /* Add the HAVING clause if given */ - if (query->havingQual != NULL) - { - appendContextKeyword(context, " HAVING ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); - get_rule_expr(query->havingQual, context, false); - } - - /* Add the WINDOW clause if needed */ - if (query->windowClause != NIL) - get_rule_windowclause(query, context); -} - -/* ---------- - * get_target_list - Parse back a SELECT target list - * - * This is also used for RETURNING lists in INSERT/UPDATE/DELETE. - * ---------- - */ -static void -get_target_list(List *targetList, deparse_context *context, - TupleDesc resultDesc) -{ - StringInfo buf = context->buf; - StringInfoData targetbuf; - bool last_was_multiline = false; - char *sep; - int colno; - ListCell *l; - - /* we use targetbuf to hold each TLE's text temporarily */ - initStringInfo(&targetbuf); - - sep = " "; - colno = 0; - foreach(l, targetList) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - char *colname; - char *attname; - - if (tle->resjunk) - continue; /* ignore junk entries */ - - appendStringInfoString(buf, sep); - sep = ", "; - colno++; - - /* - * Put the new field text into targetbuf so we can decide after we've - * got it whether or not it needs to go on a new line. - */ - resetStringInfo(&targetbuf); - context->buf = &targetbuf; - - /* - * We special-case Var nodes rather than using get_rule_expr. This is - * needed because get_rule_expr will display a whole-row Var as - * "foo.*", which is the preferred notation in most contexts, but at - * the top level of a SELECT list it's not right (the parser will - * expand that notation into multiple columns, yielding behavior - * different from a whole-row Var). We need to call get_variable - * directly so that we can tell it to do the right thing, and so that - * we can get the attribute name which is the default AS label. - */ - if (tle->expr && (IsA(tle->expr, Var))) - { - attname = get_variable((Var *) tle->expr, 0, true, context); - } - else - { - get_rule_expr((Node *) tle->expr, context, true); - /* We'll show the AS name unless it's this: */ - attname = "?column?"; - } - - /* - * Figure out what the result column should be called. In the context - * of a view, use the view's tuple descriptor (so as to pick up the - * effects of any column RENAME that's been done on the view). - * Otherwise, just use what we can find in the TLE. - */ - if (resultDesc && colno <= resultDesc->natts) - colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); - else - colname = tle->resname; - - /* Show AS unless the column's name is correct as-is */ - if (colname) /* resname could be NULL */ - { - if (attname == NULL || strcmp(attname, colname) != 0) - appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); - } - - /* Restore context's output buffer */ - context->buf = buf; - - /* Consider line-wrapping if enabled */ - if (PRETTY_INDENT(context) && context->wrapColumn >= 0) - { - int leading_nl_pos; - - /* Does the new field start with a new line? */ - if (targetbuf.len > 0 && targetbuf.data[0] == '\n') - leading_nl_pos = 0; - else - leading_nl_pos = -1; - - /* If so, we shouldn't add anything */ - if (leading_nl_pos >= 0) - { - /* instead, remove any trailing spaces currently in buf */ - removeStringInfoSpaces(buf); - } - else - { - char *trailing_nl; - - /* Locate the start of the current line in the output buffer */ - trailing_nl = strrchr(buf->data, '\n'); - if (trailing_nl == NULL) - trailing_nl = buf->data; - else - trailing_nl++; - - /* - * Add a newline, plus some indentation, if the new field is - * not the first and either the new field would cause an - * overflow or the last field used more than one line. - */ - if (colno > 1 && - ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || - last_was_multiline)) - appendContextKeyword(context, "", -PRETTYINDENT_STD, - PRETTYINDENT_STD, PRETTYINDENT_VAR); - } - - /* Remember this field's multiline status for next iteration */ - last_was_multiline = - (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); - } - - /* Add the new field */ - appendStringInfoString(buf, targetbuf.data); - } - - /* clean up */ - pfree(targetbuf.data); -} - -static void -get_setop_query(Node *setOp, Query *query, deparse_context *context, - TupleDesc resultDesc) -{ - StringInfo buf = context->buf; - bool need_paren; - - /* Guard against excessively long or deeply-nested queries */ - CHECK_FOR_INTERRUPTS(); - check_stack_depth(); - - if (IsA(setOp, RangeTblRef)) - { - RangeTblRef *rtr = (RangeTblRef *) setOp; - RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); - Query *subquery = rte->subquery; - - Assert(subquery != NULL); - Assert(subquery->setOperations == NULL); - /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ - need_paren = (subquery->cteList || - subquery->sortClause || - subquery->rowMarks || - subquery->limitOffset || - subquery->limitCount); - if (need_paren) - appendStringInfoChar(buf, '('); - get_query_def(subquery, buf, context->namespaces, resultDesc, - context->prettyFlags, context->wrapColumn, - context->indentLevel); - if (need_paren) - appendStringInfoChar(buf, ')'); - } - else if (IsA(setOp, SetOperationStmt)) - { - SetOperationStmt *op = (SetOperationStmt *) setOp; - int subindent; - - /* - * We force parens when nesting two SetOperationStmts, except when the - * lefthand input is another setop of the same kind. Syntactically, - * we could omit parens in rather more cases, but it seems best to use - * parens to flag cases where the setop operator changes. If we use - * parens, we also increase the indentation level for the child query. - * - * There are some cases in which parens are needed around a leaf query - * too, but those are more easily handled at the next level down (see - * code above). - */ - if (IsA(op->larg, SetOperationStmt)) - { - SetOperationStmt *lop = (SetOperationStmt *) op->larg; - - if (op->op == lop->op && op->all == lop->all) - need_paren = false; - else - need_paren = true; - } - else - need_paren = false; - - if (need_paren) - { - appendStringInfoChar(buf, '('); - subindent = PRETTYINDENT_STD; - appendContextKeyword(context, "", subindent, 0, 0); - } - else - subindent = 0; - - get_setop_query(op->larg, query, context, resultDesc); - - if (need_paren) - appendContextKeyword(context, ") ", -subindent, 0, 0); - else if (PRETTY_INDENT(context)) - appendContextKeyword(context, "", -subindent, 0, 0); - else - appendStringInfoChar(buf, ' '); - - switch (op->op) - { - case SETOP_UNION: - appendStringInfoString(buf, "UNION "); - break; - case SETOP_INTERSECT: - appendStringInfoString(buf, "INTERSECT "); - break; - case SETOP_EXCEPT: - appendStringInfoString(buf, "EXCEPT "); - break; - default: - elog(ERROR, "unrecognized set op: %d", - (int) op->op); - } - if (op->all) - appendStringInfoString(buf, "ALL "); - - /* Always parenthesize if RHS is another setop */ - need_paren = IsA(op->rarg, SetOperationStmt); - - /* - * The indentation code here is deliberately a bit different from that - * for the lefthand input, because we want the line breaks in - * different places. - */ - if (need_paren) - { - appendStringInfoChar(buf, '('); - subindent = PRETTYINDENT_STD; - } - else - subindent = 0; - appendContextKeyword(context, "", subindent, 0, 0); - - get_setop_query(op->rarg, query, context, resultDesc); - - if (PRETTY_INDENT(context)) - context->indentLevel -= subindent; - if (need_paren) - appendContextKeyword(context, ")", 0, 0, 0); - } - else - { - elog(ERROR, "unrecognized node type: %d", - (int) nodeTag(setOp)); - } -} - -/* - * Display a sort/group clause. - * - * Also returns the expression tree, so caller need not find it again. - */ -static Node * -get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, - deparse_context *context) -{ - StringInfo buf = context->buf; - TargetEntry *tle; - Node *expr; - - tle = get_sortgroupref_tle(ref, tlist); - expr = (Node *) tle->expr; - - /* - * Use column-number form if requested by caller. Otherwise, if - * expression is a constant, force it to be dumped with an explicit cast - * as decoration --- this is because a simple integer constant is - * ambiguous (and will be misinterpreted by findTargetlistEntry()) if we - * dump it without any decoration. If it's anything more complex than a - * simple Var, then force extra parens around it, to ensure it can't be - * misinterpreted as a cube() or rollup() construct. - */ - if (force_colno) - { - Assert(!tle->resjunk); - appendStringInfo(buf, "%d", tle->resno); - } - else if (expr && IsA(expr, Const)) - get_const_expr((Const *) expr, context, 1); - else if (!expr || IsA(expr, Var)) - get_rule_expr(expr, context, true); - else - { - /* - * We must force parens for function-like expressions even if - * PRETTY_PAREN is off, since those are the ones in danger of - * misparsing. For other expressions we need to force them only if - * PRETTY_PAREN is on, since otherwise the expression will output them - * itself. (We can't skip the parens.) - */ - bool need_paren = (PRETTY_PAREN(context) - || IsA(expr, FuncExpr) - || IsA(expr, Aggref) - || IsA(expr, WindowFunc)); - - if (need_paren) - appendStringInfoChar(context->buf, '('); - get_rule_expr(expr, context, true); - if (need_paren) - appendStringInfoChar(context->buf, ')'); - } - - return expr; -} - -/* - * Display a GroupingSet - */ -static void -get_rule_groupingset(GroupingSet *gset, List *targetlist, - bool omit_parens, deparse_context *context) -{ - ListCell *l; - StringInfo buf = context->buf; - bool omit_child_parens = true; - char *sep = ""; - - switch (gset->kind) - { - case GROUPING_SET_EMPTY: - appendStringInfoString(buf, "()"); - return; - - case GROUPING_SET_SIMPLE: - { - if (!omit_parens || list_length(gset->content) != 1) - appendStringInfoChar(buf, '('); - - foreach(l, gset->content) - { - Index ref = lfirst_int(l); - - appendStringInfoString(buf, sep); - get_rule_sortgroupclause(ref, targetlist, - false, context); - sep = ", "; - } - - if (!omit_parens || list_length(gset->content) != 1) - appendStringInfoChar(buf, ')'); - } - return; - - case GROUPING_SET_ROLLUP: - appendStringInfoString(buf, "ROLLUP("); - break; - case GROUPING_SET_CUBE: - appendStringInfoString(buf, "CUBE("); - break; - case GROUPING_SET_SETS: - appendStringInfoString(buf, "GROUPING SETS ("); - omit_child_parens = false; - break; - } - - foreach(l, gset->content) - { - appendStringInfoString(buf, sep); - get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); - sep = ", "; - } - - appendStringInfoChar(buf, ')'); -} - -/* - * Display an ORDER BY list. - */ -static void -get_rule_orderby(List *orderList, List *targetList, - bool force_colno, deparse_context *context) -{ - StringInfo buf = context->buf; - const char *sep; - ListCell *l; - - sep = ""; - foreach(l, orderList) - { - SortGroupClause *srt = (SortGroupClause *) lfirst(l); - Node *sortexpr; - Oid sortcoltype; - TypeCacheEntry *typentry; - - appendStringInfoString(buf, sep); - sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, - force_colno, context); - sortcoltype = exprType(sortexpr); - /* See whether operator is default < or > for datatype */ - typentry = lookup_type_cache(sortcoltype, - TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); - if (srt->sortop == typentry->lt_opr) - { - /* ASC is default, so emit nothing for it */ - if (srt->nulls_first) - appendStringInfoString(buf, " NULLS FIRST"); - } - else if (srt->sortop == typentry->gt_opr) - { - appendStringInfoString(buf, " DESC"); - /* DESC defaults to NULLS FIRST */ - if (!srt->nulls_first) - appendStringInfoString(buf, " NULLS LAST"); - } - else - { - appendStringInfo(buf, " USING %s", - generate_operator_name(srt->sortop, - sortcoltype, - sortcoltype)); - /* be specific to eliminate ambiguity */ - if (srt->nulls_first) - appendStringInfoString(buf, " NULLS FIRST"); - else - appendStringInfoString(buf, " NULLS LAST"); - } - sep = ", "; - } -} - -/* - * Display a WINDOW clause. - * - * Note that the windowClause list might contain only anonymous window - * specifications, in which case we should print nothing here. - */ -static void -get_rule_windowclause(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - const char *sep; - ListCell *l; - - sep = NULL; - foreach(l, query->windowClause) - { - WindowClause *wc = (WindowClause *) lfirst(l); - - if (wc->name == NULL) - continue; /* ignore anonymous windows */ - - if (sep == NULL) - appendContextKeyword(context, " WINDOW ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - else - appendStringInfoString(buf, sep); - - appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); - - get_rule_windowspec(wc, query->targetList, context); - - sep = ", "; - } -} - -/* - * Display a window definition - */ -static void -get_rule_windowspec(WindowClause *wc, List *targetList, - deparse_context *context) -{ - StringInfo buf = context->buf; - bool needspace = false; - const char *sep; - ListCell *l; - - appendStringInfoChar(buf, '('); - if (wc->refname) - { - appendStringInfoString(buf, quote_identifier(wc->refname)); - needspace = true; - } - /* partition clauses are always inherited, so only print if no refname */ - if (wc->partitionClause && !wc->refname) - { - if (needspace) - appendStringInfoChar(buf, ' '); - appendStringInfoString(buf, "PARTITION BY "); - sep = ""; - foreach(l, wc->partitionClause) - { - SortGroupClause *grp = (SortGroupClause *) lfirst(l); - - appendStringInfoString(buf, sep); - get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, - false, context); - sep = ", "; - } - needspace = true; - } - /* print ordering clause only if not inherited */ - if (wc->orderClause && !wc->copiedOrder) - { - if (needspace) - appendStringInfoChar(buf, ' '); - appendStringInfoString(buf, "ORDER BY "); - get_rule_orderby(wc->orderClause, targetList, false, context); - needspace = true; - } - /* framing clause is never inherited, so print unless it's default */ - if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) - { - if (needspace) - appendStringInfoChar(buf, ' '); - if (wc->frameOptions & FRAMEOPTION_RANGE) - appendStringInfoString(buf, "RANGE "); - else if (wc->frameOptions & FRAMEOPTION_ROWS) - appendStringInfoString(buf, "ROWS "); - else if (wc->frameOptions & FRAMEOPTION_GROUPS) - appendStringInfoString(buf, "GROUPS "); - else - Assert(false); - if (wc->frameOptions & FRAMEOPTION_BETWEEN) - appendStringInfoString(buf, "BETWEEN "); - if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) - appendStringInfoString(buf, "UNBOUNDED PRECEDING "); - else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) - appendStringInfoString(buf, "CURRENT ROW "); - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) - { - get_rule_expr(wc->startOffset, context, false); - if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) - appendStringInfoString(buf, " PRECEDING "); - else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) - appendStringInfoString(buf, " FOLLOWING "); - else - Assert(false); - } - else - Assert(false); - if (wc->frameOptions & FRAMEOPTION_BETWEEN) - { - appendStringInfoString(buf, "AND "); - if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) - appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); - else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) - appendStringInfoString(buf, "CURRENT ROW "); - else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) - { - get_rule_expr(wc->endOffset, context, false); - if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) - appendStringInfoString(buf, " PRECEDING "); - else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) - appendStringInfoString(buf, " FOLLOWING "); - else - Assert(false); - } - else - Assert(false); - } - if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) - appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); - else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) - appendStringInfoString(buf, "EXCLUDE GROUP "); - else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) - appendStringInfoString(buf, "EXCLUDE TIES "); - /* we will now have a trailing space; remove it */ - buf->len--; - } - appendStringInfoChar(buf, ')'); -} - -/* ---------- - * get_insert_query_def - Parse back an INSERT parsetree - * ---------- - */ -static void -get_insert_query_def(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - RangeTblEntry *select_rte = NULL; - RangeTblEntry *values_rte = NULL; - RangeTblEntry *rte; - char *sep; - ListCell *l; - List *strippedexprs; - - /* Insert the WITH clause if given */ - get_with_clause(query, context); - - /* - * If it's an INSERT ... SELECT or multi-row VALUES, there will be a - * single RTE for the SELECT or VALUES. Plain VALUES has neither. - */ - foreach(l, query->rtable) - { - rte = (RangeTblEntry *) lfirst(l); - - if (rte->rtekind == RTE_SUBQUERY) - { - if (select_rte) - elog(ERROR, "too many subquery RTEs in INSERT"); - select_rte = rte; - } - - if (rte->rtekind == RTE_VALUES) - { - if (values_rte) - elog(ERROR, "too many values RTEs in INSERT"); - values_rte = rte; - } - } - if (select_rte && values_rte) - elog(ERROR, "both subquery and values RTEs in INSERT"); - - /* - * Start the query with INSERT INTO relname - */ - rte = rt_fetch(query->resultRelation, query->rtable); - Assert(rte->rtekind == RTE_RELATION); - - if (PRETTY_INDENT(context)) - { - context->indentLevel += PRETTYINDENT_STD; - appendStringInfoChar(buf, ' '); - } - appendStringInfo(buf, "INSERT INTO %s ", - generate_relation_or_shard_name(rte->relid, - context->distrelid, - context->shardid, NIL)); - /* INSERT requires AS keyword for target alias */ - if (rte->alias != NULL) - appendStringInfo(buf, "AS %s ", - quote_identifier(get_rtable_name(query->resultRelation, context))); - - /* - * Add the insert-column-names list. Any indirection decoration needed on - * the column names can be inferred from the top targetlist. - */ - strippedexprs = NIL; - sep = ""; - if (query->targetList) - appendStringInfoChar(buf, '('); - foreach(l, query->targetList) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - - if (tle->resjunk) - continue; /* ignore junk entries */ - - appendStringInfoString(buf, sep); - sep = ", "; - - /* - * Put out name of target column; look in the catalogs, not at - * tle->resname, since resname will fail to track RENAME. - */ - appendStringInfoString(buf, - quote_identifier(get_attname(rte->relid, - tle->resno, - false))); - - /* - * Print any indirection needed (subfields or subscripts), and strip - * off the top-level nodes representing the indirection assignments. - * Add the stripped expressions to strippedexprs. (If it's a - * single-VALUES statement, the stripped expressions are the VALUES to - * print below. Otherwise they're just Vars and not really - * interesting.) - */ - strippedexprs = lappend(strippedexprs, - processIndirection((Node *) tle->expr, - context)); - } - if (query->targetList) - appendStringInfoString(buf, ") "); - - if (query->override) - { - if (query->override == OVERRIDING_SYSTEM_VALUE) - appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); - else if (query->override == OVERRIDING_USER_VALUE) - appendStringInfoString(buf, "OVERRIDING USER VALUE "); - } - - if (select_rte) - { - /* Add the SELECT */ - get_query_def(select_rte->subquery, buf, NIL, NULL, - context->prettyFlags, context->wrapColumn, - context->indentLevel); - } - else if (values_rte) - { - /* Add the multi-VALUES expression lists */ - get_values_def(values_rte->values_lists, context); - } - else if (strippedexprs) - { - /* Add the single-VALUES expression list */ - appendContextKeyword(context, "VALUES (", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); - get_rule_expr((Node *) strippedexprs, context, false); - appendStringInfoChar(buf, ')'); - } - else - { - /* No expressions, so it must be DEFAULT VALUES */ - appendStringInfoString(buf, "DEFAULT VALUES"); - } - - /* Add ON CONFLICT if present */ - if (query->onConflict) - { - OnConflictExpr *confl = query->onConflict; - - appendStringInfoString(buf, " ON CONFLICT"); - - if (confl->arbiterElems) - { - /* Add the single-VALUES expression list */ - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) confl->arbiterElems, context, false); - appendStringInfoChar(buf, ')'); - - /* Add a WHERE clause (for partial indexes) if given */ - if (confl->arbiterWhere != NULL) - { - bool save_varprefix; - - /* - * Force non-prefixing of Vars, since parser assumes that they - * belong to target relation. WHERE clause does not use - * InferenceElem, so this is separately required. - */ - save_varprefix = context->varprefix; - context->varprefix = false; - - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(confl->arbiterWhere, context, false); - - context->varprefix = save_varprefix; - } - } - else if (OidIsValid(confl->constraint)) - { - char *constraint = get_constraint_name(confl->constraint); - int64 shardId = context->shardid; - - if (shardId > 0) - { - AppendShardIdToName(&constraint, shardId); - } - - if (!constraint) - elog(ERROR, "cache lookup failed for constraint %u", - confl->constraint); - appendStringInfo(buf, " ON CONSTRAINT %s", - quote_identifier(constraint)); - } - - if (confl->action == ONCONFLICT_NOTHING) - { - appendStringInfoString(buf, " DO NOTHING"); - } - else - { - appendStringInfoString(buf, " DO UPDATE SET "); - /* Deparse targetlist */ - get_update_query_targetlist_def(query, confl->onConflictSet, - context, rte); - - /* Add a WHERE clause if given */ - if (confl->onConflictWhere != NULL) - { - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(confl->onConflictWhere, context, false); - } - } - } - - /* Add RETURNING if present */ - if (query->returningList) - { - appendContextKeyword(context, " RETURNING", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_target_list(query->returningList, context, NULL); - } -} - - -/* ---------- - * get_update_query_def - Parse back an UPDATE parsetree - * ---------- - */ -static void -get_update_query_def(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - RangeTblEntry *rte; - - /* Insert the WITH clause if given */ - get_with_clause(query, context); - - /* - * Start the query with UPDATE relname SET - */ - rte = rt_fetch(query->resultRelation, query->rtable); - - if (PRETTY_INDENT(context)) - { - appendStringInfoChar(buf, ' '); - context->indentLevel += PRETTYINDENT_STD; - } - - /* if it's a shard, do differently */ - if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - char *fragmentSchemaName = NULL; - char *fragmentTableName = NULL; - - ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); - - /* use schema and table name from the remote alias */ - appendStringInfo(buf, "UPDATE %s%s", - only_marker(rte), - generate_fragment_name(fragmentSchemaName, fragmentTableName)); - - if(rte->eref != NULL) - appendStringInfo(buf, " %s", - quote_identifier(get_rtable_name(query->resultRelation, context))); - } - else - { - appendStringInfo(buf, "UPDATE %s%s", - only_marker(rte), - generate_relation_or_shard_name(rte->relid, - context->distrelid, - context->shardid, NIL)); - - if (rte->alias != NULL) - appendStringInfo(buf, " %s", - quote_identifier(get_rtable_name(query->resultRelation, context))); - } - - appendStringInfoString(buf, " SET "); - - /* Deparse targetlist */ - get_update_query_targetlist_def(query, query->targetList, context, rte); - - /* Add the FROM clause if needed */ - get_from_clause(query, " FROM ", context); - - /* Add a WHERE clause if given */ - if (query->jointree->quals != NULL) - { - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(query->jointree->quals, context, false); - } - - /* Add RETURNING if present */ - if (query->returningList) - { - appendContextKeyword(context, " RETURNING", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_target_list(query->returningList, context, NULL); - } -} - - -/* ---------- - * get_update_query_targetlist_def - Parse back an UPDATE targetlist - * ---------- - */ -static void -get_update_query_targetlist_def(Query *query, List *targetList, - deparse_context *context, RangeTblEntry *rte) -{ - StringInfo buf = context->buf; - ListCell *l; - ListCell *next_ma_cell; - int remaining_ma_columns; - const char *sep; - SubLink *cur_ma_sublink; - List *ma_sublinks; - - /* - * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks - * into a list. We expect them to appear, in ID order, in resjunk tlist - * entries. - */ - ma_sublinks = NIL; - if (query->hasSubLinks) /* else there can't be any */ - { - foreach(l, targetList) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - - if (tle->resjunk && IsA(tle->expr, SubLink)) - { - SubLink *sl = (SubLink *) tle->expr; - - if (sl->subLinkType == MULTIEXPR_SUBLINK) - { - ma_sublinks = lappend(ma_sublinks, sl); - Assert(sl->subLinkId == list_length(ma_sublinks)); - } - } - } - } - next_ma_cell = list_head(ma_sublinks); - cur_ma_sublink = NULL; - remaining_ma_columns = 0; - - /* Add the comma separated list of 'attname = value' */ - sep = ""; - foreach(l, targetList) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - Node *expr; - - if (tle->resjunk) - continue; /* ignore junk entries */ - - /* Emit separator (OK whether we're in multiassignment or not) */ - appendStringInfoString(buf, sep); - sep = ", "; - - /* - * Check to see if we're starting a multiassignment group: if so, - * output a left paren. - */ - if (next_ma_cell != NULL && cur_ma_sublink == NULL) - { - /* - * We must dig down into the expr to see if it's a PARAM_MULTIEXPR - * Param. That could be buried under FieldStores and - * SubscriptingRefs and CoerceToDomains (cf processIndirection()), - * and underneath those there could be an implicit type coercion. - * Because we would ignore implicit type coercions anyway, we - * don't need to be as careful as processIndirection() is about - * descending past implicit CoerceToDomains. - */ - expr = (Node *) tle->expr; - while (expr) - { - if (IsA(expr, FieldStore)) - { - FieldStore *fstore = (FieldStore *) expr; - - expr = (Node *) linitial(fstore->newvals); - } - else if (IsA(expr, SubscriptingRef)) - { - SubscriptingRef *sbsref = (SubscriptingRef *) expr; - - if (sbsref->refassgnexpr == NULL) - break; - expr = (Node *) sbsref->refassgnexpr; - } - else if (IsA(expr, CoerceToDomain)) - { - CoerceToDomain *cdomain = (CoerceToDomain *) expr; - - if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) - break; - expr = (Node *) cdomain->arg; - } - else - break; - } - expr = strip_implicit_coercions(expr); - - if (expr && IsA(expr, Param) && - ((Param *) expr)->paramkind == PARAM_MULTIEXPR) - { - cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); - next_ma_cell = lnext(ma_sublinks, next_ma_cell); - remaining_ma_columns = count_nonjunk_tlist_entries( - ((Query *) cur_ma_sublink->subselect)->targetList); - Assert(((Param *) expr)->paramid == - ((cur_ma_sublink->subLinkId << 16) | 1)); - appendStringInfoChar(buf, '('); - } - } - - /* - * Put out name of target column; look in the catalogs, not at - * tle->resname, since resname will fail to track RENAME. - */ - appendStringInfoString(buf, - quote_identifier(get_attname(rte->relid, - tle->resno, - false))); - - /* - * Print any indirection needed (subfields or subscripts), and strip - * off the top-level nodes representing the indirection assignments. - */ - expr = processIndirection((Node *) tle->expr, context); - - /* - * If we're in a multiassignment, skip printing anything more, unless - * this is the last column; in which case, what we print should be the - * sublink, not the Param. - */ - if (cur_ma_sublink != NULL) - { - if (--remaining_ma_columns > 0) - continue; /* not the last column of multiassignment */ - appendStringInfoChar(buf, ')'); - expr = (Node *) cur_ma_sublink; - cur_ma_sublink = NULL; - } - - appendStringInfoString(buf, " = "); - - get_rule_expr(expr, context, false); - } -} - - -/* ---------- - * get_delete_query_def - Parse back a DELETE parsetree - * ---------- - */ -static void -get_delete_query_def(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - RangeTblEntry *rte; - - /* Insert the WITH clause if given */ - get_with_clause(query, context); - - /* - * Start the query with DELETE FROM relname - */ - rte = rt_fetch(query->resultRelation, query->rtable); - - if (PRETTY_INDENT(context)) - { - appendStringInfoChar(buf, ' '); - context->indentLevel += PRETTYINDENT_STD; - } - - /* if it's a shard, do differently */ - if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - char *fragmentSchemaName = NULL; - char *fragmentTableName = NULL; - - ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); - - /* use schema and table name from the remote alias */ - appendStringInfo(buf, "DELETE FROM %s%s", - only_marker(rte), - generate_fragment_name(fragmentSchemaName, fragmentTableName)); - - if(rte->eref != NULL) - appendStringInfo(buf, " %s", - quote_identifier(get_rtable_name(query->resultRelation, context))); - } - else - { - appendStringInfo(buf, "DELETE FROM %s%s", - only_marker(rte), - generate_relation_or_shard_name(rte->relid, - context->distrelid, - context->shardid, NIL)); - - if (rte->alias != NULL) - appendStringInfo(buf, " %s", - quote_identifier(get_rtable_name(query->resultRelation, context))); - } - - /* Add the USING clause if given */ - get_from_clause(query, " USING ", context); - - /* Add a WHERE clause if given */ - if (query->jointree->quals != NULL) - { - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(query->jointree->quals, context, false); - } - - /* Add RETURNING if present */ - if (query->returningList) - { - appendContextKeyword(context, " RETURNING", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_target_list(query->returningList, context, NULL); - } -} - - -/* ---------- - * get_utility_query_def - Parse back a UTILITY parsetree - * ---------- - */ -static void -get_utility_query_def(Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - - if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) - { - NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; - - appendContextKeyword(context, "", - 0, PRETTYINDENT_STD, 1); - appendStringInfo(buf, "NOTIFY %s", - quote_identifier(stmt->conditionname)); - if (stmt->payload) - { - appendStringInfoString(buf, ", "); - simple_quote_literal(buf, stmt->payload); - } - } - else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) - { - TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; - List *relationList = stmt->relations; - ListCell *relationCell = NULL; - - appendContextKeyword(context, "", - 0, PRETTYINDENT_STD, 1); - - appendStringInfo(buf, "TRUNCATE TABLE"); - - foreach(relationCell, relationList) - { - RangeVar *relationVar = (RangeVar *) lfirst(relationCell); - Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); - char *relationName = generate_relation_or_shard_name(relationId, - context->distrelid, - context->shardid, NIL); - appendStringInfo(buf, " %s", relationName); - - if (lnext(relationList, relationCell) != NULL) - { - appendStringInfo(buf, ","); - } - } - - if (stmt->restart_seqs) - { - appendStringInfo(buf, " RESTART IDENTITY"); - } - - if (stmt->behavior == DROP_CASCADE) - { - appendStringInfo(buf, " CASCADE"); - } - } - else - { - /* Currently only NOTIFY utility commands can appear in rules */ - elog(ERROR, "unexpected utility statement type"); - } -} - -/* - * Display a Var appropriately. - * - * In some cases (currently only when recursing into an unnamed join) - * the Var's varlevelsup has to be interpreted with respect to a context - * above the current one; levelsup indicates the offset. - * - * If istoplevel is true, the Var is at the top level of a SELECT's - * targetlist, which means we need special treatment of whole-row Vars. - * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a - * dirty hack to prevent "tab.*" from being expanded into multiple columns. - * (The parser will strip the useless coercion, so no inefficiency is added in - * dump and reload.) We used to print just "tab" in such cases, but that is - * ambiguous and will yield the wrong result if "tab" is also a plain column - * name in the query. - * - * Returns the attname of the Var, or NULL if the Var has no attname (because - * it is a whole-row Var or a subplan output reference). - */ -static char * -get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) -{ - StringInfo buf = context->buf; - RangeTblEntry *rte; - AttrNumber attnum; - Index varno; - AttrNumber varattno; - int netlevelsup; - deparse_namespace *dpns; - deparse_columns *colinfo; - char *refname; - char *attname; - - /* Find appropriate nesting depth */ - netlevelsup = var->varlevelsup + levelsup; - if (netlevelsup >= list_length(context->namespaces)) - elog(ERROR, "bogus varlevelsup: %d offset %d", - var->varlevelsup, levelsup); - dpns = (deparse_namespace *) list_nth(context->namespaces, - netlevelsup); - - varno = var->varno; - varattno = var->varattno; - - - if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { - rte = rt_fetch(var->varnosyn, dpns->rtable); - - /* - * if the rte var->varnosyn points to is not a regular table and it is a join - * then the correct relname will be found with var->varnosyn and var->varattnosyn - */ - if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { - varno = var->varnosyn; - varattno = var->varattnosyn; - } - } - - /* - * Try to find the relevant RTE in this rtable. In a plan tree, it's - * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig - * down into the subplans, or INDEX_VAR, which is resolved similarly. Also - * find the aliases previously assigned for this RTE. - */ - if (varno >= 1 && varno <= list_length(dpns->rtable)) - { - - /* - * We might have been asked to map child Vars to some parent relation. - */ - if (context->appendparents && dpns->appendrels) - { - - Index pvarno = varno; - AttrNumber pvarattno = varattno; - AppendRelInfo *appinfo = dpns->appendrels[pvarno]; - bool found = false; - - /* Only map up to inheritance parents, not UNION ALL appendrels */ - while (appinfo && - rt_fetch(appinfo->parent_relid, - dpns->rtable)->rtekind == RTE_RELATION) - { - found = false; - if (pvarattno > 0) /* system columns stay as-is */ - { - if (pvarattno > appinfo->num_child_cols) - break; /* safety check */ - pvarattno = appinfo->parent_colnos[pvarattno - 1]; - if (pvarattno == 0) - break; /* Var is local to child */ - } - - pvarno = appinfo->parent_relid; - found = true; - - /* If the parent is itself a child, continue up. */ - Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); - appinfo = dpns->appendrels[pvarno]; - } - - /* - * If we found an ancestral rel, and that rel is included in - * appendparents, print that column not the original one. - */ - if (found && bms_is_member(pvarno, context->appendparents)) - { - varno = pvarno; - varattno = pvarattno; - } - } - - rte = rt_fetch(varno, dpns->rtable); - refname = (char *) list_nth(dpns->rtable_names, varno - 1); - colinfo = deparse_columns_fetch(varno, dpns); - attnum = varattno; - } - else - { - resolve_special_varno((Node *) var, context, get_special_variable, - NULL); - return NULL; - } - - /* - * The planner will sometimes emit Vars referencing resjunk elements of a - * subquery's target list (this is currently only possible if it chooses - * to generate a "physical tlist" for a SubqueryScan or CteScan node). - * Although we prefer to print subquery-referencing Vars using the - * subquery's alias, that's not possible for resjunk items since they have - * no alias. So in that case, drill down to the subplan and print the - * contents of the referenced tlist item. This works because in a plan - * tree, such Vars can only occur in a SubqueryScan or CteScan node, and - * we'll have set dpns->inner_plan to reference the child plan node. - */ - if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && - attnum > list_length(rte->eref->colnames) && - dpns->inner_plan) - { - TargetEntry *tle; - deparse_namespace save_dpns; - - tle = get_tle_by_resno(dpns->inner_tlist, attnum); - if (!tle) - elog(ERROR, "invalid attnum %d for relation \"%s\"", - attnum, rte->eref->aliasname); - - Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_plan, &save_dpns); - - /* - * Force parentheses because our caller probably assumed a Var is a - * simple expression. - */ - if (!IsA(tle->expr, Var)) - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) tle->expr, context, true); - if (!IsA(tle->expr, Var)) - appendStringInfoChar(buf, ')'); - - pop_child_plan(dpns, &save_dpns); - return NULL; - } - - /* - * If it's an unnamed join, look at the expansion of the alias variable. - * If it's a simple reference to one of the input vars, then recursively - * print the name of that var instead. When it's not a simple reference, - * we have to just print the unqualified join column name. (This can only - * happen with "dangerous" merged columns in a JOIN USING; we took pains - * previously to make the unqualified column name unique in such cases.) - * - * This wouldn't work in decompiling plan trees, because we don't store - * joinaliasvars lists after planning; but a plan tree should never - * contain a join alias variable. - */ - if (rte->rtekind == RTE_JOIN && rte->alias == NULL) - { - if (rte->joinaliasvars == NIL) - elog(ERROR, "cannot decompile join alias var in plan tree"); - if (attnum > 0) - { - Var *aliasvar; - - aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); - /* we intentionally don't strip implicit coercions here */ - if (aliasvar && IsA(aliasvar, Var)) - { - return get_variable(aliasvar, var->varlevelsup + levelsup, - istoplevel, context); - } - } - - /* - * Unnamed join has no refname. (Note: since it's unnamed, there is - * no way the user could have referenced it to create a whole-row Var - * for it. So we don't have to cover that case below.) - */ - Assert(refname == NULL); - } - - if (attnum == InvalidAttrNumber) - attname = NULL; - else if (attnum > 0) - { - /* Get column name to use from the colinfo struct */ - if (attnum > colinfo->num_cols) - elog(ERROR, "invalid attnum %d for relation \"%s\"", - attnum, rte->eref->aliasname); - attname = colinfo->colnames[attnum - 1]; - if (attname == NULL) /* dropped column? */ - elog(ERROR, "invalid attnum %d for relation \"%s\"", - attnum, rte->eref->aliasname); - } - else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - /* System column on a Citus shard */ - attname = get_attname(rte->relid, attnum, false); - } - else - { - /* System column - name is fixed, get it from the catalog */ - attname = get_rte_attribute_name(rte, attnum); - } - - if (refname && (context->varprefix || attname == NULL)) - { - appendStringInfoString(buf, quote_identifier(refname)); - appendStringInfoChar(buf, '.'); - } - if (attname) - appendStringInfoString(buf, quote_identifier(attname)); - else - { - appendStringInfoChar(buf, '*'); - - if (istoplevel) - { - if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - /* use rel.*::shard_name instead of rel.*::table_name */ - appendStringInfo(buf, "::%s", - generate_rte_shard_name(rte)); - } - else - { - appendStringInfo(buf, "::%s", - format_type_with_typemod(var->vartype, - var->vartypmod)); - } - } - } - - return attname; -} - -/* - * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This - * routine is actually a callback for get_special_varno, which handles finding - * the correct TargetEntry. We get the expression contained in that - * TargetEntry and just need to deparse it, a job we can throw back on - * get_rule_expr. - */ -static void -get_special_variable(Node *node, deparse_context *context, void *callback_arg) -{ - StringInfo buf = context->buf; - - /* - * For a non-Var referent, force parentheses because our caller probably - * assumed a Var is a simple expression. - */ - if (!IsA(node, Var)) - appendStringInfoChar(buf, '('); - get_rule_expr(node, context, true); - if (!IsA(node, Var)) - appendStringInfoChar(buf, ')'); -} - -/* - * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, - * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, - * invoke the callback provided. - */ -static void -resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) -{ - Var *var; - deparse_namespace *dpns; - - /* This function is recursive, so let's be paranoid. */ - check_stack_depth(); - - /* If it's not a Var, invoke the callback. */ - if (!IsA(node, Var)) - { - (*callback) (node, context, callback_arg); - return; - } - - /* Find appropriate nesting depth */ - var = (Var *) node; - dpns = (deparse_namespace *) list_nth(context->namespaces, - var->varlevelsup); - - /* - * It's a special RTE, so recurse. - */ - if (var->varno == OUTER_VAR && dpns->outer_tlist) - { - TargetEntry *tle; - deparse_namespace save_dpns; - Bitmapset *save_appendparents; - - tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); - if (!tle) - elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); - - /* If we're descending to the first child of an Append or MergeAppend, - * update appendparents. This will affect deparsing of all Vars - * appearing within the eventually-resolved subexpression. - */ - save_appendparents = context->appendparents; - - if (IsA(dpns->plan, Append)) - context->appendparents = bms_union(context->appendparents, - ((Append *) dpns->plan)->apprelids); - else if (IsA(dpns->plan, MergeAppend)) - context->appendparents = bms_union(context->appendparents, - ((MergeAppend *) dpns->plan)->apprelids); - - push_child_plan(dpns, dpns->outer_plan, &save_dpns); - resolve_special_varno((Node *) tle->expr, context, - callback, callback_arg); - pop_child_plan(dpns, &save_dpns); - context->appendparents = save_appendparents; - return; - } - else if (var->varno == INNER_VAR && dpns->inner_tlist) - { - TargetEntry *tle; - deparse_namespace save_dpns; - - tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); - if (!tle) - elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); - - push_child_plan(dpns, dpns->inner_plan, &save_dpns); - resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); - pop_child_plan(dpns, &save_dpns); - return; - } - else if (var->varno == INDEX_VAR && dpns->index_tlist) - { - TargetEntry *tle; - - tle = get_tle_by_resno(dpns->index_tlist, var->varattno); - if (!tle) - elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); - - resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); - return; - } - else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) - elog(ERROR, "bogus varno: %d", var->varno); - - /* Not special. Just invoke the callback. */ - (*callback) (node, context, callback_arg); -} - -/* - * Get the name of a field of an expression of composite type. The - * expression is usually a Var, but we handle other cases too. - * - * levelsup is an extra offset to interpret the Var's varlevelsup correctly. - * - * This is fairly straightforward when the expression has a named composite - * type; we need only look up the type in the catalogs. However, the type - * could also be RECORD. Since no actual table or view column is allowed to - * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE - * or to a subquery output. We drill down to find the ultimate defining - * expression and attempt to infer the field name from it. We ereport if we - * can't determine the name. - * - * Similarly, a PARAM of type RECORD has to refer to some expression of - * a determinable composite type. - */ -static const char * -get_name_for_var_field(Var *var, int fieldno, - int levelsup, deparse_context *context) -{ - RangeTblEntry *rte; - AttrNumber attnum; - int netlevelsup; - deparse_namespace *dpns; - Index varno; - AttrNumber varattno; - TupleDesc tupleDesc; - Node *expr; - - /* - * If it's a RowExpr that was expanded from a whole-row Var, use the - * column names attached to it. - */ - if (IsA(var, RowExpr)) - { - RowExpr *r = (RowExpr *) var; - - if (fieldno > 0 && fieldno <= list_length(r->colnames)) - return strVal(list_nth(r->colnames, fieldno - 1)); - } - - /* - * If it's a Param of type RECORD, try to find what the Param refers to. - */ - if (IsA(var, Param)) - { - Param *param = (Param *) var; - ListCell *ancestor_cell; - - expr = find_param_referent(param, context, &dpns, &ancestor_cell); - if (expr) - { - /* Found a match, so recurse to decipher the field name */ - deparse_namespace save_dpns; - const char *result; - - push_ancestor_plan(dpns, ancestor_cell, &save_dpns); - result = get_name_for_var_field((Var *) expr, fieldno, - 0, context); - pop_ancestor_plan(dpns, &save_dpns); - return result; - } - } - - /* - * If it's a Var of type RECORD, we have to find what the Var refers to; - * if not, we can use get_expr_result_tupdesc(). - */ - if (!IsA(var, Var) || - var->vartype != RECORDOID) - { - tupleDesc = get_expr_result_tupdesc((Node *) var, false); - /* Got the tupdesc, so we can extract the field name */ - Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); - return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); - } - - /* Find appropriate nesting depth */ - netlevelsup = var->varlevelsup + levelsup; - if (netlevelsup >= list_length(context->namespaces)) - elog(ERROR, "bogus varlevelsup: %d offset %d", - var->varlevelsup, levelsup); - dpns = (deparse_namespace *) list_nth(context->namespaces, - netlevelsup); - - varno = var->varno; - varattno = var->varattno; - - if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { - rte = rt_fetch(var->varnosyn, dpns->rtable); - - /* - * if the rte var->varnosyn points to is not a regular table and it is a join - * then the correct relname will be found with var->varnosyn and var->varattnosyn - */ - if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { - varno = var->varnosyn; - varattno = var->varattnosyn; - } - } - - /* - * Try to find the relevant RTE in this rtable. In a plan tree, it's - * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig - * down into the subplans, or INDEX_VAR, which is resolved similarly. - */ - if (varno >= 1 && varno <= list_length(dpns->rtable)) - { - rte = rt_fetch(varno, dpns->rtable); - attnum = varattno; - } - else if (varno == OUTER_VAR && dpns->outer_tlist) - { - TargetEntry *tle; - deparse_namespace save_dpns; - const char *result; - - tle = get_tle_by_resno(dpns->outer_tlist, varattno); - if (!tle) - elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); - - Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->outer_plan, &save_dpns); - - result = get_name_for_var_field((Var *) tle->expr, fieldno, - levelsup, context); - - pop_child_plan(dpns, &save_dpns); - return result; - } - else if (varno == INNER_VAR && dpns->inner_tlist) - { - TargetEntry *tle; - deparse_namespace save_dpns; - const char *result; - - tle = get_tle_by_resno(dpns->inner_tlist, varattno); - if (!tle) - elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); - - Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_plan, &save_dpns); - - result = get_name_for_var_field((Var *) tle->expr, fieldno, - levelsup, context); - - pop_child_plan(dpns, &save_dpns); - return result; - } - else if (varno == INDEX_VAR && dpns->index_tlist) - { - TargetEntry *tle; - const char *result; - - tle = get_tle_by_resno(dpns->index_tlist, varattno); - if (!tle) - elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); - - Assert(netlevelsup == 0); - - result = get_name_for_var_field((Var *) tle->expr, fieldno, - levelsup, context); - - return result; - } - else - { - elog(ERROR, "bogus varno: %d", varno); - return NULL; /* keep compiler quiet */ - } - - if (attnum == InvalidAttrNumber) - { - /* Var is whole-row reference to RTE, so select the right field */ - return get_rte_attribute_name(rte, fieldno); - } - - /* - * This part has essentially the same logic as the parser's - * expandRecordVariable() function, but we are dealing with a different - * representation of the input context, and we only need one field name - * not a TupleDesc. Also, we need special cases for finding subquery and - * CTE subplans when deparsing Plan trees. - */ - expr = (Node *) var; /* default if we can't drill down */ - - switch (rte->rtekind) - { - case RTE_RELATION: - case RTE_VALUES: - case RTE_NAMEDTUPLESTORE: - case RTE_RESULT: - - /* - * This case should not occur: a column of a table or values list - * shouldn't have type RECORD. Fall through and fail (most - * likely) at the bottom. - */ - break; - case RTE_SUBQUERY: - /* Subselect-in-FROM: examine sub-select's output expr */ - { - if (rte->subquery) - { - TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, - attnum); - - if (ste == NULL || ste->resjunk) - elog(ERROR, "subquery %s does not have attribute %d", - rte->eref->aliasname, attnum); - expr = (Node *) ste->expr; - if (IsA(expr, Var)) - { - /* - * Recurse into the sub-select to see what its Var - * refers to. We have to build an additional level of - * namespace to keep in step with varlevelsup in the - * subselect. - */ - deparse_namespace mydpns; - const char *result; - - set_deparse_for_query(&mydpns, rte->subquery, - context->namespaces); - - context->namespaces = lcons(&mydpns, - context->namespaces); - - result = get_name_for_var_field((Var *) expr, fieldno, - 0, context); - - context->namespaces = - list_delete_first(context->namespaces); - - return result; - } - /* else fall through to inspect the expression */ - } - else - { - /* - * We're deparsing a Plan tree so we don't have complete - * RTE entries (in particular, rte->subquery is NULL). But - * the only place we'd see a Var directly referencing a - * SUBQUERY RTE is in a SubqueryScan plan node, and we can - * look into the child plan's tlist instead. - */ - TargetEntry *tle; - deparse_namespace save_dpns; - const char *result; - - if (!dpns->inner_plan) - elog(ERROR, "failed to find plan for subquery %s", - rte->eref->aliasname); - tle = get_tle_by_resno(dpns->inner_tlist, attnum); - if (!tle) - elog(ERROR, "bogus varattno for subquery var: %d", - attnum); - Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_plan, &save_dpns); - - result = get_name_for_var_field((Var *) tle->expr, fieldno, - levelsup, context); - - pop_child_plan(dpns, &save_dpns); - return result; - } - } - break; - case RTE_JOIN: - /* Join RTE --- recursively inspect the alias variable */ - if (rte->joinaliasvars == NIL) - elog(ERROR, "cannot decompile join alias var in plan tree"); - Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); - expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); - Assert(expr != NULL); - /* we intentionally don't strip implicit coercions here */ - if (IsA(expr, Var)) - return get_name_for_var_field((Var *) expr, fieldno, - var->varlevelsup + levelsup, - context); - /* else fall through to inspect the expression */ - break; - case RTE_FUNCTION: - case RTE_TABLEFUNC: - - /* - * We couldn't get here unless a function is declared with one of - * its result columns as RECORD, which is not allowed. - */ - break; - case RTE_CTE: - /* CTE reference: examine subquery's output expr */ - { - CommonTableExpr *cte = NULL; - Index ctelevelsup; - ListCell *lc; - - /* - * Try to find the referenced CTE using the namespace stack. - */ - ctelevelsup = rte->ctelevelsup + netlevelsup; - if (ctelevelsup >= list_length(context->namespaces)) - lc = NULL; - else - { - deparse_namespace *ctedpns; - - ctedpns = (deparse_namespace *) - list_nth(context->namespaces, ctelevelsup); - foreach(lc, ctedpns->ctes) - { - cte = (CommonTableExpr *) lfirst(lc); - if (strcmp(cte->ctename, rte->ctename) == 0) - break; - } - } - if (lc != NULL) - { - Query *ctequery = (Query *) cte->ctequery; - TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), - attnum); - - if (ste == NULL || ste->resjunk) - elog(ERROR, "subquery %s does not have attribute %d", - rte->eref->aliasname, attnum); - expr = (Node *) ste->expr; - if (IsA(expr, Var)) - { - /* - * Recurse into the CTE to see what its Var refers to. - * We have to build an additional level of namespace - * to keep in step with varlevelsup in the CTE. - * Furthermore it could be an outer CTE, so we may - * have to delete some levels of namespace. - */ - List *save_nslist = context->namespaces; - List *new_nslist; - deparse_namespace mydpns; - const char *result; - - set_deparse_for_query(&mydpns, ctequery, - context->namespaces); - - new_nslist = list_copy_tail(context->namespaces, - ctelevelsup); - context->namespaces = lcons(&mydpns, new_nslist); - - result = get_name_for_var_field((Var *) expr, fieldno, - 0, context); - - context->namespaces = save_nslist; - - return result; - } - /* else fall through to inspect the expression */ - } - else - { - /* - * We're deparsing a Plan tree so we don't have a CTE - * list. But the only place we'd see a Var directly - * referencing a CTE RTE is in a CteScan plan node, and we - * can look into the subplan's tlist instead. - */ - TargetEntry *tle; - deparse_namespace save_dpns; - const char *result; - - if (!dpns->inner_plan) - elog(ERROR, "failed to find plan for CTE %s", - rte->eref->aliasname); - tle = get_tle_by_resno(dpns->inner_tlist, attnum); - if (!tle) - elog(ERROR, "bogus varattno for subquery var: %d", - attnum); - Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_plan, &save_dpns); - - result = get_name_for_var_field((Var *) tle->expr, fieldno, - levelsup, context); - - pop_child_plan(dpns, &save_dpns); - return result; - } - } - break; - } - - /* - * We now have an expression we can't expand any more, so see if - * get_expr_result_tupdesc() can do anything with it. - */ - tupleDesc = get_expr_result_tupdesc(expr, false); - /* Got the tupdesc, so we can extract the field name */ - Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); - return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); -} - -/* - * Try to find the referenced expression for a PARAM_EXEC Param that might - * reference a parameter supplied by an upper NestLoop or SubPlan plan node. - * - * If successful, return the expression and set *dpns_p and *ancestor_cell_p - * appropriately for calling push_ancestor_plan(). If no referent can be - * found, return NULL. - */ -static Node * -find_param_referent(Param *param, deparse_context *context, - deparse_namespace **dpns_p, ListCell **ancestor_cell_p) -{ - /* Initialize output parameters to prevent compiler warnings */ - *dpns_p = NULL; - *ancestor_cell_p = NULL; - - /* - * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or - * SubPlan argument. This will necessarily be in some ancestor of the - * current expression's Plan. - */ - if (param->paramkind == PARAM_EXEC) - { - deparse_namespace *dpns; - Plan *child_plan; - bool in_same_plan_level; - ListCell *lc; - - dpns = (deparse_namespace *) linitial(context->namespaces); - child_plan = dpns->plan; - in_same_plan_level = true; - - foreach(lc, dpns->ancestors) - { - Node *ancestor = (Node *) lfirst(lc); - ListCell *lc2; - - /* - * NestLoops transmit params to their inner child only; also, once - * we've crawled up out of a subplan, this couldn't possibly be - * the right match. - */ - if (IsA(ancestor, NestLoop) && - child_plan == innerPlan(ancestor) && - in_same_plan_level) - { - NestLoop *nl = (NestLoop *) ancestor; - - foreach(lc2, nl->nestParams) - { - NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); - - if (nlp->paramno == param->paramid) - { - /* Found a match, so return it */ - *dpns_p = dpns; - *ancestor_cell_p = lc; - return (Node *) nlp->paramval; - } - } - } - - /* - * Check to see if we're crawling up from a subplan. - */ - if(IsA(ancestor, SubPlan)) - { - SubPlan *subplan = (SubPlan *) ancestor; - ListCell *lc3; - ListCell *lc4; - - /* Matched subplan, so check its arguments */ - forboth(lc3, subplan->parParam, lc4, subplan->args) - { - int paramid = lfirst_int(lc3); - Node *arg = (Node *) lfirst(lc4); - - if (paramid == param->paramid) - { - /* - * Found a match, so return it. But, since Vars in - * the arg are to be evaluated in the surrounding - * context, we have to point to the next ancestor item - * that is *not* a SubPlan. - */ - ListCell *rest; - - for_each_cell(rest, dpns->ancestors, - lnext(dpns->ancestors, lc)) - { - Node *ancestor2 = (Node *) lfirst(rest); - - if (!IsA(ancestor2, SubPlan)) - { - *dpns_p = dpns; - *ancestor_cell_p = rest; - return arg; - } - } - elog(ERROR, "SubPlan cannot be outermost ancestor"); - } - } - - /* We have emerged from a subplan. */ - in_same_plan_level = false; - - /* SubPlan isn't a kind of Plan, so skip the rest */ - continue; - } - - /* - * Check to see if we're emerging from an initplan of the current - * ancestor plan. Initplans never have any parParams, so no need - * to search that list, but we need to know if we should reset - * in_same_plan_level. - */ - foreach(lc2, ((Plan *) ancestor)->initPlan) - { - SubPlan *subplan = lfirst_node(SubPlan, lc2); - - if (child_plan != (Plan *) list_nth(dpns->subplans, - subplan->plan_id - 1)) - continue; - - /* No parameters to be had here. */ - Assert(subplan->parParam == NIL); - - /* We have emerged from an initplan. */ - in_same_plan_level = false; - break; - } - - /* No luck, crawl up to next ancestor */ - child_plan = (Plan *) ancestor; - } - } - - /* No referent found */ - return NULL; -} - -/* - * Display a Param appropriately. - */ -static void -get_parameter(Param *param, deparse_context *context) -{ - Node *expr; - deparse_namespace *dpns; - ListCell *ancestor_cell; - - /* - * If it's a PARAM_EXEC parameter, try to locate the expression from which - * the parameter was computed. Note that failing to find a referent isn't - * an error, since the Param might well be a subplan output rather than an - * input. - */ - expr = find_param_referent(param, context, &dpns, &ancestor_cell); - if (expr) - { - /* Found a match, so print it */ - deparse_namespace save_dpns; - bool save_varprefix; - bool need_paren; - - /* Switch attention to the ancestor plan node */ - push_ancestor_plan(dpns, ancestor_cell, &save_dpns); - - /* - * Force prefixing of Vars, since they won't belong to the relation - * being scanned in the original plan node. - */ - save_varprefix = context->varprefix; - context->varprefix = true; - - /* - * A Param's expansion is typically a Var, Aggref, or upper-level - * Param, which wouldn't need extra parentheses. Otherwise, insert - * parens to ensure the expression looks atomic. - */ - need_paren = !(IsA(expr, Var) || - IsA(expr, Aggref) || - IsA(expr, Param)); - if (need_paren) - appendStringInfoChar(context->buf, '('); - - get_rule_expr(expr, context, false); - - if (need_paren) - appendStringInfoChar(context->buf, ')'); - - context->varprefix = save_varprefix; - - pop_ancestor_plan(dpns, &save_dpns); - - return; - } - - /* - * If it's an external parameter, see if the outermost namespace provides - * function argument names. - */ - if (param->paramkind == PARAM_EXTERN) - { - dpns = lfirst(list_tail(context->namespaces)); - if (dpns->argnames) - { - char *argname = dpns->argnames[param->paramid - 1]; - - if (argname) - { - bool should_qualify = false; - ListCell *lc; - - /* - * Qualify the parameter name if there are any other deparse - * namespaces with range tables. This avoids qualifying in - * trivial cases like "RETURN a + b", but makes it safe in all - * other cases. - */ - foreach(lc, context->namespaces) - { - deparse_namespace *dp_ns = lfirst(lc); - - if (list_length(dp_ns->rtable_names) > 0) - { - should_qualify = true; - break; - } - } - if (should_qualify) - { - appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); - appendStringInfoChar(context->buf, '.'); - } - - appendStringInfoString(context->buf, quote_identifier(argname)); - return; - } - } - } - - /* - * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. - * For composite types, add cast to the parameter to ease remote node detect - * the type. - */ - if (param->paramtype >= FirstNormalObjectId) - { - char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); - - appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); - } - else - { - appendStringInfo(context->buf, "$%d", param->paramid); - } -} - -/* - * get_simple_binary_op_name - * - * helper function for isSimpleNode - * will return single char binary operator name, or NULL if it's not - */ -static const char * -get_simple_binary_op_name(OpExpr *expr) -{ - List *args = expr->args; - - if (list_length(args) == 2) - { - /* binary operator */ - Node *arg1 = (Node *) linitial(args); - Node *arg2 = (Node *) lsecond(args); - const char *op; - - op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); - if (strlen(op) == 1) - return op; - } - return NULL; -} - - -/* - * isSimpleNode - check if given node is simple (doesn't need parenthesizing) - * - * true : simple in the context of parent node's type - * false : not simple - */ -static bool -isSimpleNode(Node *node, Node *parentNode, int prettyFlags) -{ - if (!node) - return false; - - switch (nodeTag(node)) - { - case T_Var: - case T_Const: - case T_Param: - case T_CoerceToDomainValue: - case T_SetToDefault: - case T_CurrentOfExpr: - /* single words: always simple */ - return true; - - case T_SubscriptingRef: - case T_ArrayExpr: - case T_RowExpr: - case T_CoalesceExpr: - case T_MinMaxExpr: - case T_SQLValueFunction: - case T_XmlExpr: - case T_NextValueExpr: - case T_NullIfExpr: - case T_Aggref: - case T_WindowFunc: - case T_FuncExpr: - /* function-like: name(..) or name[..] */ - return true; - - /* CASE keywords act as parentheses */ - case T_CaseExpr: - return true; - - case T_FieldSelect: - - /* - * appears simple since . has top precedence, unless parent is - * T_FieldSelect itself! - */ - return (IsA(parentNode, FieldSelect) ? false : true); - - case T_FieldStore: - - /* - * treat like FieldSelect (probably doesn't matter) - */ - return (IsA(parentNode, FieldStore) ? false : true); - - case T_CoerceToDomain: - /* maybe simple, check args */ - return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, - node, prettyFlags); - case T_RelabelType: - return isSimpleNode((Node *) ((RelabelType *) node)->arg, - node, prettyFlags); - case T_CoerceViaIO: - return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, - node, prettyFlags); - case T_ArrayCoerceExpr: - return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, - node, prettyFlags); - case T_ConvertRowtypeExpr: - return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, - node, prettyFlags); - - case T_OpExpr: - { - /* depends on parent node type; needs further checking */ - if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) - { - const char *op; - const char *parentOp; - bool is_lopriop; - bool is_hipriop; - bool is_lopriparent; - bool is_hipriparent; - - op = get_simple_binary_op_name((OpExpr *) node); - if (!op) - return false; - - /* We know only the basic operators + - and * / % */ - is_lopriop = (strchr("+-", *op) != NULL); - is_hipriop = (strchr("*/%", *op) != NULL); - if (!(is_lopriop || is_hipriop)) - return false; - - parentOp = get_simple_binary_op_name((OpExpr *) parentNode); - if (!parentOp) - return false; - - is_lopriparent = (strchr("+-", *parentOp) != NULL); - is_hipriparent = (strchr("*/%", *parentOp) != NULL); - if (!(is_lopriparent || is_hipriparent)) - return false; - - if (is_hipriop && is_lopriparent) - return true; /* op binds tighter than parent */ - - if (is_lopriop && is_hipriparent) - return false; - - /* - * Operators are same priority --- can skip parens only if - * we have (a - b) - c, not a - (b - c). - */ - if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) - return true; - - return false; - } - /* else do the same stuff as for T_SubLink et al. */ - } - /* FALLTHROUGH */ - - case T_SubLink: - case T_NullTest: - case T_BooleanTest: - case T_DistinctExpr: - switch (nodeTag(parentNode)) - { - case T_FuncExpr: - { - /* special handling for casts */ - CoercionForm type = ((FuncExpr *) parentNode)->funcformat; - - if (type == COERCE_EXPLICIT_CAST || - type == COERCE_IMPLICIT_CAST) - return false; - return true; /* own parentheses */ - } - case T_BoolExpr: /* lower precedence */ - case T_SubscriptingRef: /* other separators */ - case T_ArrayExpr: /* other separators */ - case T_RowExpr: /* other separators */ - case T_CoalesceExpr: /* own parentheses */ - case T_MinMaxExpr: /* own parentheses */ - case T_XmlExpr: /* own parentheses */ - case T_NullIfExpr: /* other separators */ - case T_Aggref: /* own parentheses */ - case T_WindowFunc: /* own parentheses */ - case T_CaseExpr: /* other separators */ - return true; - default: - return false; - } - - case T_BoolExpr: - switch (nodeTag(parentNode)) - { - case T_BoolExpr: - if (prettyFlags & PRETTYFLAG_PAREN) - { - BoolExprType type; - BoolExprType parentType; - - type = ((BoolExpr *) node)->boolop; - parentType = ((BoolExpr *) parentNode)->boolop; - switch (type) - { - case NOT_EXPR: - case AND_EXPR: - if (parentType == AND_EXPR || parentType == OR_EXPR) - return true; - break; - case OR_EXPR: - if (parentType == OR_EXPR) - return true; - break; - } - } - return false; - case T_FuncExpr: - { - /* special handling for casts */ - CoercionForm type = ((FuncExpr *) parentNode)->funcformat; - - if (type == COERCE_EXPLICIT_CAST || - type == COERCE_IMPLICIT_CAST) - return false; - return true; /* own parentheses */ - } - case T_SubscriptingRef: /* other separators */ - case T_ArrayExpr: /* other separators */ - case T_RowExpr: /* other separators */ - case T_CoalesceExpr: /* own parentheses */ - case T_MinMaxExpr: /* own parentheses */ - case T_XmlExpr: /* own parentheses */ - case T_NullIfExpr: /* other separators */ - case T_Aggref: /* own parentheses */ - case T_WindowFunc: /* own parentheses */ - case T_CaseExpr: /* other separators */ - return true; - default: - return false; - } - - default: - break; - } - /* those we don't know: in dubio complexo */ - return false; -} - - -/* - * appendContextKeyword - append a keyword to buffer - * - * If prettyPrint is enabled, perform a line break, and adjust indentation. - * Otherwise, just append the keyword. - */ -static void -appendContextKeyword(deparse_context *context, const char *str, - int indentBefore, int indentAfter, int indentPlus) -{ - StringInfo buf = context->buf; - - if (PRETTY_INDENT(context)) - { - int indentAmount; - - context->indentLevel += indentBefore; - - /* remove any trailing spaces currently in the buffer ... */ - removeStringInfoSpaces(buf); - /* ... then add a newline and some spaces */ - appendStringInfoChar(buf, '\n'); - - if (context->indentLevel < PRETTYINDENT_LIMIT) - indentAmount = Max(context->indentLevel, 0) + indentPlus; - else - { - /* - * If we're indented more than PRETTYINDENT_LIMIT characters, try - * to conserve horizontal space by reducing the per-level - * indentation. For best results the scale factor here should - * divide all the indent amounts that get added to indentLevel - * (PRETTYINDENT_STD, etc). It's important that the indentation - * not grow unboundedly, else deeply-nested trees use O(N^2) - * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. - */ - indentAmount = PRETTYINDENT_LIMIT + - (context->indentLevel - PRETTYINDENT_LIMIT) / - (PRETTYINDENT_STD / 2); - indentAmount %= PRETTYINDENT_LIMIT; - /* scale/wrap logic affects indentLevel, but not indentPlus */ - indentAmount += indentPlus; - } - appendStringInfoSpaces(buf, indentAmount); - - appendStringInfoString(buf, str); - - context->indentLevel += indentAfter; - if (context->indentLevel < 0) - context->indentLevel = 0; - } - else - appendStringInfoString(buf, str); -} - -/* - * removeStringInfoSpaces - delete trailing spaces from a buffer. - * - * Possibly this should move to stringinfo.c at some point. - */ -static void -removeStringInfoSpaces(StringInfo str) -{ - while (str->len > 0 && str->data[str->len - 1] == ' ') - str->data[--(str->len)] = '\0'; -} - - -/* - * get_rule_expr_paren - deparse expr using get_rule_expr, - * embracing the string with parentheses if necessary for prettyPrint. - * - * Never embrace if prettyFlags=0, because it's done in the calling node. - * - * Any node that does *not* embrace its argument node by sql syntax (with - * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should - * use get_rule_expr_paren instead of get_rule_expr so parentheses can be - * added. - */ -static void -get_rule_expr_paren(Node *node, deparse_context *context, - bool showimplicit, Node *parentNode) -{ - bool need_paren; - - need_paren = PRETTY_PAREN(context) && - !isSimpleNode(node, parentNode, context->prettyFlags); - - if (need_paren) - appendStringInfoChar(context->buf, '('); - - get_rule_expr(node, context, showimplicit); - - if (need_paren) - appendStringInfoChar(context->buf, ')'); -} - - -/* ---------- - * get_rule_expr - Parse back an expression - * - * Note: showimplicit determines whether we display any implicit cast that - * is present at the top of the expression tree. It is a passed argument, - * not a field of the context struct, because we change the value as we - * recurse down into the expression. In general we suppress implicit casts - * when the result type is known with certainty (eg, the arguments of an - * OR must be boolean). We display implicit casts for arguments of functions - * and operators, since this is needed to be certain that the same function - * or operator will be chosen when the expression is re-parsed. - * ---------- - */ -static void -get_rule_expr(Node *node, deparse_context *context, - bool showimplicit) -{ - StringInfo buf = context->buf; - - if (node == NULL) - return; - - /* Guard against excessively long or deeply-nested queries */ - CHECK_FOR_INTERRUPTS(); - check_stack_depth(); - - /* - * Each level of get_rule_expr must emit an indivisible term - * (parenthesized if necessary) to ensure result is reparsed into the same - * expression tree. The only exception is that when the input is a List, - * we emit the component items comma-separated with no surrounding - * decoration; this is convenient for most callers. - */ - switch (nodeTag(node)) - { - case T_Var: - (void) get_variable((Var *) node, 0, false, context); - break; - - case T_Const: - get_const_expr((Const *) node, context, 0); - break; - - case T_Param: - get_parameter((Param *) node, context); - break; - - case T_Aggref: - get_agg_expr((Aggref *) node, context, (Aggref *) node); - break; - - case T_GroupingFunc: - { - GroupingFunc *gexpr = (GroupingFunc *) node; - - appendStringInfoString(buf, "GROUPING("); - get_rule_expr((Node *) gexpr->args, context, true); - appendStringInfoChar(buf, ')'); - } - break; - - case T_WindowFunc: - get_windowfunc_expr((WindowFunc *) node, context); - break; - - case T_SubscriptingRef: - { - SubscriptingRef *sbsref = (SubscriptingRef *) node; - bool need_parens; - - /* - * If the argument is a CaseTestExpr, we must be inside a - * FieldStore, ie, we are assigning to an element of an array - * within a composite column. Since we already punted on - * displaying the FieldStore's target information, just punt - * here too, and display only the assignment source - * expression. - */ - if (IsA(sbsref->refexpr, CaseTestExpr)) - { - Assert(sbsref->refassgnexpr); - get_rule_expr((Node *) sbsref->refassgnexpr, - context, showimplicit); - break; - } - - /* - * Parenthesize the argument unless it's a simple Var or a - * FieldSelect. (In particular, if it's another - * SubscriptingRef, we *must* parenthesize to avoid - * confusion.) - */ - need_parens = !IsA(sbsref->refexpr, Var) && - !IsA(sbsref->refexpr, FieldSelect); - if (need_parens) - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); - if (need_parens) - appendStringInfoChar(buf, ')'); - - /* - * If there's a refassgnexpr, we want to print the node in the - * format "container[subscripts] := refassgnexpr". This is - * not legal SQL, so decompilation of INSERT or UPDATE - * statements should always use processIndirection as part of - * the statement-level syntax. We should only see this when - * EXPLAIN tries to print the targetlist of a plan resulting - * from such a statement. - */ - if (sbsref->refassgnexpr) - { - Node *refassgnexpr; - - /* - * Use processIndirection to print this node's subscripts - * as well as any additional field selections or - * subscripting in immediate descendants. It returns the - * RHS expr that is actually being "assigned". - */ - refassgnexpr = processIndirection(node, context); - appendStringInfoString(buf, " := "); - get_rule_expr(refassgnexpr, context, showimplicit); - } - else - { - /* Just an ordinary container fetch, so print subscripts */ - printSubscripts(sbsref, context); - } - } - break; - - case T_FuncExpr: - get_func_expr((FuncExpr *) node, context, showimplicit); - break; - - case T_NamedArgExpr: - { - NamedArgExpr *na = (NamedArgExpr *) node; - - appendStringInfo(buf, "%s => ", quote_identifier(na->name)); - get_rule_expr((Node *) na->arg, context, showimplicit); - } - break; - - case T_OpExpr: - get_oper_expr((OpExpr *) node, context); - break; - - case T_DistinctExpr: - { - DistinctExpr *expr = (DistinctExpr *) node; - List *args = expr->args; - Node *arg1 = (Node *) linitial(args); - Node *arg2 = (Node *) lsecond(args); - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(arg1, context, true, node); - appendStringInfoString(buf, " IS DISTINCT FROM "); - get_rule_expr_paren(arg2, context, true, node); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - break; - - case T_NullIfExpr: - { - NullIfExpr *nullifexpr = (NullIfExpr *) node; - - appendStringInfoString(buf, "NULLIF("); - get_rule_expr((Node *) nullifexpr->args, context, true); - appendStringInfoChar(buf, ')'); - } - break; - - case T_ScalarArrayOpExpr: - { - ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; - List *args = expr->args; - Node *arg1 = (Node *) linitial(args); - Node *arg2 = (Node *) lsecond(args); - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(arg1, context, true, node); - appendStringInfo(buf, " %s %s (", - generate_operator_name(expr->opno, - exprType(arg1), - get_base_element_type(exprType(arg2))), - expr->useOr ? "ANY" : "ALL"); - get_rule_expr_paren(arg2, context, true, node); - - /* - * There's inherent ambiguity in "x op ANY/ALL (y)" when y is - * a bare sub-SELECT. Since we're here, the sub-SELECT must - * be meant as a scalar sub-SELECT yielding an array value to - * be used in ScalarArrayOpExpr; but the grammar will - * preferentially interpret such a construct as an ANY/ALL - * SubLink. To prevent misparsing the output that way, insert - * a dummy coercion (which will be stripped by parse analysis, - * so no inefficiency is added in dump and reload). This is - * indeed most likely what the user wrote to get the construct - * accepted in the first place. - */ - if (IsA(arg2, SubLink) && - ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) - appendStringInfo(buf, "::%s", - format_type_with_typemod(exprType(arg2), - exprTypmod(arg2))); - appendStringInfoChar(buf, ')'); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - break; - - case T_BoolExpr: - { - BoolExpr *expr = (BoolExpr *) node; - Node *first_arg = linitial(expr->args); - ListCell *arg; - - switch (expr->boolop) - { - case AND_EXPR: - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(first_arg, context, - false, node); - for_each_from(arg, expr->args, 1) - { - appendStringInfoString(buf, " AND "); - get_rule_expr_paren((Node *) lfirst(arg), context, - false, node); - } - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - break; - - case OR_EXPR: - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(first_arg, context, - false, node); - for_each_from(arg, expr->args, 1) - { - appendStringInfoString(buf, " OR "); - get_rule_expr_paren((Node *) lfirst(arg), context, - false, node); - } - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - break; - - case NOT_EXPR: - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - appendStringInfoString(buf, "NOT "); - get_rule_expr_paren(first_arg, context, - false, node); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - break; - - default: - elog(ERROR, "unrecognized boolop: %d", - (int) expr->boolop); - } - } - break; - - case T_SubLink: - get_sublink_expr((SubLink *) node, context); - break; - - case T_SubPlan: - { - SubPlan *subplan = (SubPlan *) node; - - /* - * We cannot see an already-planned subplan in rule deparsing, - * only while EXPLAINing a query plan. We don't try to - * reconstruct the original SQL, just reference the subplan - * that appears elsewhere in EXPLAIN's result. - */ - if (subplan->useHashTable) - appendStringInfo(buf, "(hashed %s)", subplan->plan_name); - else - appendStringInfo(buf, "(%s)", subplan->plan_name); - } - break; - - case T_AlternativeSubPlan: - { - AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; - ListCell *lc; - - /* - * This case cannot be reached in normal usage, since no - * AlternativeSubPlan can appear either in parsetrees or - * finished plan trees. We keep it just in case somebody - * wants to use this code to print planner data structures. - */ - appendStringInfoString(buf, "(alternatives: "); - foreach(lc, asplan->subplans) - { - SubPlan *splan = lfirst_node(SubPlan, lc); - - if (splan->useHashTable) - appendStringInfo(buf, "hashed %s", splan->plan_name); - else - appendStringInfoString(buf, splan->plan_name); - if (lnext(asplan->subplans, lc)) - appendStringInfoString(buf, " or "); - } - appendStringInfoChar(buf, ')'); - } - break; - - case T_FieldSelect: - { - FieldSelect *fselect = (FieldSelect *) node; - Node *arg = (Node *) fselect->arg; - int fno = fselect->fieldnum; - const char *fieldname; - bool need_parens; - - /* - * Parenthesize the argument unless it's an SubscriptingRef or - * another FieldSelect. Note in particular that it would be - * WRONG to not parenthesize a Var argument; simplicity is not - * the issue here, having the right number of names is. - */ - need_parens = !IsA(arg, SubscriptingRef) && - !IsA(arg, FieldSelect); - if (need_parens) - appendStringInfoChar(buf, '('); - get_rule_expr(arg, context, true); - if (need_parens) - appendStringInfoChar(buf, ')'); - - /* - * Get and print the field name. - */ - fieldname = get_name_for_var_field((Var *) arg, fno, - 0, context); - appendStringInfo(buf, ".%s", quote_identifier(fieldname)); - } - break; - - case T_FieldStore: - { - FieldStore *fstore = (FieldStore *) node; - bool need_parens; - - /* - * There is no good way to represent a FieldStore as real SQL, - * so decompilation of INSERT or UPDATE statements should - * always use processIndirection as part of the - * statement-level syntax. We should only get here when - * EXPLAIN tries to print the targetlist of a plan resulting - * from such a statement. The plan case is even harder than - * ordinary rules would be, because the planner tries to - * collapse multiple assignments to the same field or subfield - * into one FieldStore; so we can see a list of target fields - * not just one, and the arguments could be FieldStores - * themselves. We don't bother to try to print the target - * field names; we just print the source arguments, with a - * ROW() around them if there's more than one. This isn't - * terribly complete, but it's probably good enough for - * EXPLAIN's purposes; especially since anything more would be - * either hopelessly confusing or an even poorer - * representation of what the plan is actually doing. - */ - need_parens = (list_length(fstore->newvals) != 1); - if (need_parens) - appendStringInfoString(buf, "ROW("); - get_rule_expr((Node *) fstore->newvals, context, showimplicit); - if (need_parens) - appendStringInfoChar(buf, ')'); - } - break; - - case T_RelabelType: - { - RelabelType *relabel = (RelabelType *) node; - Node *arg = (Node *) relabel->arg; - - if (relabel->relabelformat == COERCE_IMPLICIT_CAST && - !showimplicit) - { - /* don't show the implicit cast */ - get_rule_expr_paren(arg, context, false, node); - } - else - { - get_coercion_expr(arg, context, - relabel->resulttype, - relabel->resulttypmod, - node); - } - } - break; - - case T_CoerceViaIO: - { - CoerceViaIO *iocoerce = (CoerceViaIO *) node; - Node *arg = (Node *) iocoerce->arg; - - if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && - !showimplicit) - { - /* don't show the implicit cast */ - get_rule_expr_paren(arg, context, false, node); - } - else - { - get_coercion_expr(arg, context, - iocoerce->resulttype, - -1, - node); - } - } - break; - - case T_ArrayCoerceExpr: - { - ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; - Node *arg = (Node *) acoerce->arg; - - if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && - !showimplicit) - { - /* don't show the implicit cast */ - get_rule_expr_paren(arg, context, false, node); - } - else - { - get_coercion_expr(arg, context, - acoerce->resulttype, - acoerce->resulttypmod, - node); - } - } - break; - - case T_ConvertRowtypeExpr: - { - ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; - Node *arg = (Node *) convert->arg; - - if (convert->convertformat == COERCE_IMPLICIT_CAST && - !showimplicit) - { - /* don't show the implicit cast */ - get_rule_expr_paren(arg, context, false, node); - } - else - { - get_coercion_expr(arg, context, - convert->resulttype, -1, - node); - } - } - break; - - case T_CollateExpr: - { - CollateExpr *collate = (CollateExpr *) node; - Node *arg = (Node *) collate->arg; - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(arg, context, showimplicit, node); - appendStringInfo(buf, " COLLATE %s", - generate_collation_name(collate->collOid)); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - break; - - case T_CaseExpr: - { - CaseExpr *caseexpr = (CaseExpr *) node; - ListCell *temp; - - appendContextKeyword(context, "CASE", - 0, PRETTYINDENT_VAR, 0); - if (caseexpr->arg) - { - appendStringInfoChar(buf, ' '); - get_rule_expr((Node *) caseexpr->arg, context, true); - } - foreach(temp, caseexpr->args) - { - CaseWhen *when = (CaseWhen *) lfirst(temp); - Node *w = (Node *) when->expr; - - if (caseexpr->arg) - { - /* - * The parser should have produced WHEN clauses of the - * form "CaseTestExpr = RHS", possibly with an - * implicit coercion inserted above the CaseTestExpr. - * For accurate decompilation of rules it's essential - * that we show just the RHS. However in an - * expression that's been through the optimizer, the - * WHEN clause could be almost anything (since the - * equality operator could have been expanded into an - * inline function). If we don't recognize the form - * of the WHEN clause, just punt and display it as-is. - */ - if (IsA(w, OpExpr)) - { - List *args = ((OpExpr *) w)->args; - - if (list_length(args) == 2 && - IsA(strip_implicit_coercions(linitial(args)), - CaseTestExpr)) - w = (Node *) lsecond(args); - } - } - - if (!PRETTY_INDENT(context)) - appendStringInfoChar(buf, ' '); - appendContextKeyword(context, "WHEN ", - 0, 0, 0); - get_rule_expr(w, context, false); - appendStringInfoString(buf, " THEN "); - get_rule_expr((Node *) when->result, context, true); - } - if (!PRETTY_INDENT(context)) - appendStringInfoChar(buf, ' '); - appendContextKeyword(context, "ELSE ", - 0, 0, 0); - get_rule_expr((Node *) caseexpr->defresult, context, true); - if (!PRETTY_INDENT(context)) - appendStringInfoChar(buf, ' '); - appendContextKeyword(context, "END", - -PRETTYINDENT_VAR, 0, 0); - } - break; - - case T_CaseTestExpr: - { - /* - * Normally we should never get here, since for expressions - * that can contain this node type we attempt to avoid - * recursing to it. But in an optimized expression we might - * be unable to avoid that (see comments for CaseExpr). If we - * do see one, print it as CASE_TEST_EXPR. - */ - appendStringInfoString(buf, "CASE_TEST_EXPR"); - } - break; - - case T_ArrayExpr: - { - ArrayExpr *arrayexpr = (ArrayExpr *) node; - - appendStringInfoString(buf, "ARRAY["); - get_rule_expr((Node *) arrayexpr->elements, context, true); - appendStringInfoChar(buf, ']'); - - /* - * If the array isn't empty, we assume its elements are - * coerced to the desired type. If it's empty, though, we - * need an explicit coercion to the array type. - */ - if (arrayexpr->elements == NIL) - appendStringInfo(buf, "::%s", - format_type_with_typemod(arrayexpr->array_typeid, -1)); - } - break; - - case T_RowExpr: - { - RowExpr *rowexpr = (RowExpr *) node; - TupleDesc tupdesc = NULL; - ListCell *arg; - int i; - char *sep; - - /* - * If it's a named type and not RECORD, we may have to skip - * dropped columns and/or claim there are NULLs for added - * columns. - */ - if (rowexpr->row_typeid != RECORDOID) - { - tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); - Assert(list_length(rowexpr->args) <= tupdesc->natts); - } - - /* - * SQL99 allows "ROW" to be omitted when there is more than - * one column, but for simplicity we always print it. - */ - appendStringInfoString(buf, "ROW("); - sep = ""; - i = 0; - foreach(arg, rowexpr->args) - { - Node *e = (Node *) lfirst(arg); - - if (tupdesc == NULL || - !TupleDescAttr(tupdesc, i)->attisdropped) - { - appendStringInfoString(buf, sep); - /* Whole-row Vars need special treatment here */ - get_rule_expr_toplevel(e, context, true); - sep = ", "; - } - i++; - } - if (tupdesc != NULL) - { - while (i < tupdesc->natts) - { - if (!TupleDescAttr(tupdesc, i)->attisdropped) - { - appendStringInfoString(buf, sep); - appendStringInfoString(buf, "NULL"); - sep = ", "; - } - i++; - } - - ReleaseTupleDesc(tupdesc); - } - appendStringInfoChar(buf, ')'); - if (rowexpr->row_format == COERCE_EXPLICIT_CAST) - appendStringInfo(buf, "::%s", - format_type_with_typemod(rowexpr->row_typeid, -1)); - } - break; - - case T_RowCompareExpr: - { - RowCompareExpr *rcexpr = (RowCompareExpr *) node; - ListCell *arg; - char *sep; - - /* - * SQL99 allows "ROW" to be omitted when there is more than - * one column, but for simplicity we always print it. - */ - appendStringInfoString(buf, "(ROW("); - sep = ""; - foreach(arg, rcexpr->largs) - { - Node *e = (Node *) lfirst(arg); - - appendStringInfoString(buf, sep); - get_rule_expr(e, context, true); - sep = ", "; - } - - /* - * We assume that the name of the first-column operator will - * do for all the rest too. This is definitely open to - * failure, eg if some but not all operators were renamed - * since the construct was parsed, but there seems no way to - * be perfect. - */ - appendStringInfo(buf, ") %s ROW(", - generate_operator_name(linitial_oid(rcexpr->opnos), - exprType(linitial(rcexpr->largs)), - exprType(linitial(rcexpr->rargs)))); - sep = ""; - foreach(arg, rcexpr->rargs) - { - Node *e = (Node *) lfirst(arg); - - appendStringInfoString(buf, sep); - get_rule_expr(e, context, true); - sep = ", "; - } - appendStringInfoString(buf, "))"); - } - break; - - case T_CoalesceExpr: - { - CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; - - appendStringInfoString(buf, "COALESCE("); - get_rule_expr((Node *) coalesceexpr->args, context, true); - appendStringInfoChar(buf, ')'); - } - break; - - case T_MinMaxExpr: - { - MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; - - switch (minmaxexpr->op) - { - case IS_GREATEST: - appendStringInfoString(buf, "GREATEST("); - break; - case IS_LEAST: - appendStringInfoString(buf, "LEAST("); - break; - } - get_rule_expr((Node *) minmaxexpr->args, context, true); - appendStringInfoChar(buf, ')'); - } - break; - - case T_SQLValueFunction: - { - SQLValueFunction *svf = (SQLValueFunction *) node; - - /* - * Note: this code knows that typmod for time, timestamp, and - * timestamptz just prints as integer. - */ - switch (svf->op) - { - case SVFOP_CURRENT_DATE: - appendStringInfoString(buf, "CURRENT_DATE"); - break; - case SVFOP_CURRENT_TIME: - appendStringInfoString(buf, "CURRENT_TIME"); - break; - case SVFOP_CURRENT_TIME_N: - appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); - break; - case SVFOP_CURRENT_TIMESTAMP: - appendStringInfoString(buf, "CURRENT_TIMESTAMP"); - break; - case SVFOP_CURRENT_TIMESTAMP_N: - appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", - svf->typmod); - break; - case SVFOP_LOCALTIME: - appendStringInfoString(buf, "LOCALTIME"); - break; - case SVFOP_LOCALTIME_N: - appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); - break; - case SVFOP_LOCALTIMESTAMP: - appendStringInfoString(buf, "LOCALTIMESTAMP"); - break; - case SVFOP_LOCALTIMESTAMP_N: - appendStringInfo(buf, "LOCALTIMESTAMP(%d)", - svf->typmod); - break; - case SVFOP_CURRENT_ROLE: - appendStringInfoString(buf, "CURRENT_ROLE"); - break; - case SVFOP_CURRENT_USER: - appendStringInfoString(buf, "CURRENT_USER"); - break; - case SVFOP_USER: - appendStringInfoString(buf, "USER"); - break; - case SVFOP_SESSION_USER: - appendStringInfoString(buf, "SESSION_USER"); - break; - case SVFOP_CURRENT_CATALOG: - appendStringInfoString(buf, "CURRENT_CATALOG"); - break; - case SVFOP_CURRENT_SCHEMA: - appendStringInfoString(buf, "CURRENT_SCHEMA"); - break; - } - } - break; - - case T_XmlExpr: - { - XmlExpr *xexpr = (XmlExpr *) node; - bool needcomma = false; - ListCell *arg; - ListCell *narg; - Const *con; - - switch (xexpr->op) - { - case IS_XMLCONCAT: - appendStringInfoString(buf, "XMLCONCAT("); - break; - case IS_XMLELEMENT: - appendStringInfoString(buf, "XMLELEMENT("); - break; - case IS_XMLFOREST: - appendStringInfoString(buf, "XMLFOREST("); - break; - case IS_XMLPARSE: - appendStringInfoString(buf, "XMLPARSE("); - break; - case IS_XMLPI: - appendStringInfoString(buf, "XMLPI("); - break; - case IS_XMLROOT: - appendStringInfoString(buf, "XMLROOT("); - break; - case IS_XMLSERIALIZE: - appendStringInfoString(buf, "XMLSERIALIZE("); - break; - case IS_DOCUMENT: - break; - } - if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) - { - if (xexpr->xmloption == XMLOPTION_DOCUMENT) - appendStringInfoString(buf, "DOCUMENT "); - else - appendStringInfoString(buf, "CONTENT "); - } - if (xexpr->name) - { - appendStringInfo(buf, "NAME %s", - quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); - needcomma = true; - } - if (xexpr->named_args) - { - if (xexpr->op != IS_XMLFOREST) - { - if (needcomma) - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, "XMLATTRIBUTES("); - needcomma = false; - } - forboth(arg, xexpr->named_args, narg, xexpr->arg_names) - { - Node *e = (Node *) lfirst(arg); - char *argname = strVal(lfirst(narg)); - - if (needcomma) - appendStringInfoString(buf, ", "); - get_rule_expr((Node *) e, context, true); - appendStringInfo(buf, " AS %s", - quote_identifier(map_xml_name_to_sql_identifier(argname))); - needcomma = true; - } - if (xexpr->op != IS_XMLFOREST) - appendStringInfoChar(buf, ')'); - } - if (xexpr->args) - { - if (needcomma) - appendStringInfoString(buf, ", "); - switch (xexpr->op) - { - case IS_XMLCONCAT: - case IS_XMLELEMENT: - case IS_XMLFOREST: - case IS_XMLPI: - case IS_XMLSERIALIZE: - /* no extra decoration needed */ - get_rule_expr((Node *) xexpr->args, context, true); - break; - case IS_XMLPARSE: - Assert(list_length(xexpr->args) == 2); - - get_rule_expr((Node *) linitial(xexpr->args), - context, true); - - con = lsecond_node(Const, xexpr->args); - Assert(!con->constisnull); - if (DatumGetBool(con->constvalue)) - appendStringInfoString(buf, - " PRESERVE WHITESPACE"); - else - appendStringInfoString(buf, - " STRIP WHITESPACE"); - break; - case IS_XMLROOT: - Assert(list_length(xexpr->args) == 3); - - get_rule_expr((Node *) linitial(xexpr->args), - context, true); - - appendStringInfoString(buf, ", VERSION "); - con = (Const *) lsecond(xexpr->args); - if (IsA(con, Const) && - con->constisnull) - appendStringInfoString(buf, "NO VALUE"); - else - get_rule_expr((Node *) con, context, false); - - con = lthird_node(Const, xexpr->args); - if (con->constisnull) - /* suppress STANDALONE NO VALUE */ ; - else - { - switch (DatumGetInt32(con->constvalue)) - { - case XML_STANDALONE_YES: - appendStringInfoString(buf, - ", STANDALONE YES"); - break; - case XML_STANDALONE_NO: - appendStringInfoString(buf, - ", STANDALONE NO"); - break; - case XML_STANDALONE_NO_VALUE: - appendStringInfoString(buf, - ", STANDALONE NO VALUE"); - break; - default: - break; - } - } - break; - case IS_DOCUMENT: - get_rule_expr_paren((Node *) xexpr->args, context, false, node); - break; - } - - } - if (xexpr->op == IS_XMLSERIALIZE) - appendStringInfo(buf, " AS %s", - format_type_with_typemod(xexpr->type, - xexpr->typmod)); - if (xexpr->op == IS_DOCUMENT) - appendStringInfoString(buf, " IS DOCUMENT"); - else - appendStringInfoChar(buf, ')'); - } - break; - - case T_NullTest: - { - NullTest *ntest = (NullTest *) node; - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren((Node *) ntest->arg, context, true, node); - - /* - * For scalar inputs, we prefer to print as IS [NOT] NULL, - * which is shorter and traditional. If it's a rowtype input - * but we're applying a scalar test, must print IS [NOT] - * DISTINCT FROM NULL to be semantically correct. - */ - if (ntest->argisrow || - !type_is_rowtype(exprType((Node *) ntest->arg))) - { - switch (ntest->nulltesttype) - { - case IS_NULL: - appendStringInfoString(buf, " IS NULL"); - break; - case IS_NOT_NULL: - appendStringInfoString(buf, " IS NOT NULL"); - break; - default: - elog(ERROR, "unrecognized nulltesttype: %d", - (int) ntest->nulltesttype); - } - } - else - { - switch (ntest->nulltesttype) - { - case IS_NULL: - appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); - break; - case IS_NOT_NULL: - appendStringInfoString(buf, " IS DISTINCT FROM NULL"); - break; - default: - elog(ERROR, "unrecognized nulltesttype: %d", - (int) ntest->nulltesttype); - } - } - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - break; - - case T_BooleanTest: - { - BooleanTest *btest = (BooleanTest *) node; - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren((Node *) btest->arg, context, false, node); - switch (btest->booltesttype) - { - case IS_TRUE: - appendStringInfoString(buf, " IS TRUE"); - break; - case IS_NOT_TRUE: - appendStringInfoString(buf, " IS NOT TRUE"); - break; - case IS_FALSE: - appendStringInfoString(buf, " IS FALSE"); - break; - case IS_NOT_FALSE: - appendStringInfoString(buf, " IS NOT FALSE"); - break; - case IS_UNKNOWN: - appendStringInfoString(buf, " IS UNKNOWN"); - break; - case IS_NOT_UNKNOWN: - appendStringInfoString(buf, " IS NOT UNKNOWN"); - break; - default: - elog(ERROR, "unrecognized booltesttype: %d", - (int) btest->booltesttype); - } - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - break; - - case T_CoerceToDomain: - { - CoerceToDomain *ctest = (CoerceToDomain *) node; - Node *arg = (Node *) ctest->arg; - - if (ctest->coercionformat == COERCE_IMPLICIT_CAST && - !showimplicit) - { - /* don't show the implicit cast */ - get_rule_expr(arg, context, false); - } - else - { - get_coercion_expr(arg, context, - ctest->resulttype, - ctest->resulttypmod, - node); - } - } - break; - - case T_CoerceToDomainValue: - appendStringInfoString(buf, "VALUE"); - break; - - case T_SetToDefault: - appendStringInfoString(buf, "DEFAULT"); - break; - - case T_CurrentOfExpr: - { - CurrentOfExpr *cexpr = (CurrentOfExpr *) node; - - if (cexpr->cursor_name) - appendStringInfo(buf, "CURRENT OF %s", - quote_identifier(cexpr->cursor_name)); - else - appendStringInfo(buf, "CURRENT OF $%d", - cexpr->cursor_param); - } - break; - - case T_NextValueExpr: - { - NextValueExpr *nvexpr = (NextValueExpr *) node; - - /* - * This isn't exactly nextval(), but that seems close enough - * for EXPLAIN's purposes. - */ - appendStringInfoString(buf, "nextval("); - simple_quote_literal(buf, - generate_relation_name(nvexpr->seqid, - NIL)); - appendStringInfoChar(buf, ')'); - } - break; - - case T_InferenceElem: - { - InferenceElem *iexpr = (InferenceElem *) node; - bool save_varprefix; - bool need_parens; - - /* - * InferenceElem can only refer to target relation, so a - * prefix is not useful, and indeed would cause parse errors. - */ - save_varprefix = context->varprefix; - context->varprefix = false; - - /* - * Parenthesize the element unless it's a simple Var or a bare - * function call. Follows pg_get_indexdef_worker(). - */ - need_parens = !IsA(iexpr->expr, Var); - if (IsA(iexpr->expr, FuncExpr) && - ((FuncExpr *) iexpr->expr)->funcformat == - COERCE_EXPLICIT_CALL) - need_parens = false; - - if (need_parens) - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) iexpr->expr, - context, false); - if (need_parens) - appendStringInfoChar(buf, ')'); - - context->varprefix = save_varprefix; - - if (iexpr->infercollid) - appendStringInfo(buf, " COLLATE %s", - generate_collation_name(iexpr->infercollid)); - - /* Add the operator class name, if not default */ - if (iexpr->inferopclass) - { - Oid inferopclass = iexpr->inferopclass; - Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); - - get_opclass_name(inferopclass, inferopcinputtype, buf); - } - } - break; - - case T_PartitionBoundSpec: - { - PartitionBoundSpec *spec = (PartitionBoundSpec *) node; - ListCell *cell; - char *sep; - - if (spec->is_default) - { - appendStringInfoString(buf, "DEFAULT"); - break; - } - - switch (spec->strategy) - { - case PARTITION_STRATEGY_HASH: - Assert(spec->modulus > 0 && spec->remainder >= 0); - Assert(spec->modulus > spec->remainder); - - appendStringInfoString(buf, "FOR VALUES"); - appendStringInfo(buf, " WITH (modulus %d, remainder %d)", - spec->modulus, spec->remainder); - break; - - case PARTITION_STRATEGY_LIST: - Assert(spec->listdatums != NIL); - - appendStringInfoString(buf, "FOR VALUES IN ("); - sep = ""; - foreach(cell, spec->listdatums) - { - Const *val = lfirst_node(Const, cell); - - appendStringInfoString(buf, sep); - get_const_expr(val, context, -1); - sep = ", "; - } - - appendStringInfoChar(buf, ')'); - break; - - case PARTITION_STRATEGY_RANGE: - Assert(spec->lowerdatums != NIL && - spec->upperdatums != NIL && - list_length(spec->lowerdatums) == - list_length(spec->upperdatums)); - - appendStringInfo(buf, "FOR VALUES FROM %s TO %s", - get_range_partbound_string(spec->lowerdatums), - get_range_partbound_string(spec->upperdatums)); - break; - - default: - elog(ERROR, "unrecognized partition strategy: %d", - (int) spec->strategy); - break; - } - } - break; - - case T_List: - { - char *sep; - ListCell *l; - - sep = ""; - foreach(l, (List *) node) - { - appendStringInfoString(buf, sep); - get_rule_expr((Node *) lfirst(l), context, showimplicit); - sep = ", "; - } - } - break; - - case T_TableFunc: - get_tablefunc((TableFunc *) node, context, showimplicit); - break; - - case T_CallStmt: - get_proc_expr((CallStmt *) node, context, showimplicit); - break; - - default: - elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); - break; - } -} - -/* - * get_rule_expr_toplevel - Parse back a toplevel expression - * - * Same as get_rule_expr(), except that if the expr is just a Var, we pass - * istoplevel = true not false to get_variable(). This causes whole-row Vars - * to get printed with decoration that will prevent expansion of "*". - * We need to use this in contexts such as ROW() and VALUES(), where the - * parser would expand "foo.*" appearing at top level. (In principle we'd - * use this in get_target_list() too, but that has additional worries about - * whether to print AS, so it needs to invoke get_variable() directly anyway.) - */ -static void -get_rule_expr_toplevel(Node *node, deparse_context *context, - bool showimplicit) -{ - if (node && IsA(node, Var)) - (void) get_variable((Var *) node, 0, true, context); - else - get_rule_expr(node, context, showimplicit); -} - -/* - * get_rule_expr_funccall - Parse back a function-call expression - * - * Same as get_rule_expr(), except that we guarantee that the output will - * look like a function call, or like one of the things the grammar treats as - * equivalent to a function call (see the func_expr_windowless production). - * This is needed in places where the grammar uses func_expr_windowless and - * you can't substitute a parenthesized a_expr. If what we have isn't going - * to look like a function call, wrap it in a dummy CAST() expression, which - * will satisfy the grammar --- and, indeed, is likely what the user wrote to - * produce such a thing. - */ -static void -get_rule_expr_funccall(Node *node, deparse_context *context, - bool showimplicit) -{ - if (looks_like_function(node)) - get_rule_expr(node, context, showimplicit); - else - { - StringInfo buf = context->buf; - - appendStringInfoString(buf, "CAST("); - /* no point in showing any top-level implicit cast */ - get_rule_expr(node, context, false); - appendStringInfo(buf, " AS %s)", - format_type_with_typemod(exprType(node), - exprTypmod(node))); - } -} - -/* - * Helper function to identify node types that satisfy func_expr_windowless. - * If in doubt, "false" is always a safe answer. - */ -static bool -looks_like_function(Node *node) -{ - if (node == NULL) - return false; /* probably shouldn't happen */ - switch (nodeTag(node)) - { - case T_FuncExpr: - /* OK, unless it's going to deparse as a cast */ - return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || - ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); - case T_NullIfExpr: - case T_CoalesceExpr: - case T_MinMaxExpr: - case T_SQLValueFunction: - case T_XmlExpr: - /* these are all accepted by func_expr_common_subexpr */ - return true; - default: - break; - } - return false; -} - - -/* - * get_oper_expr - Parse back an OpExpr node - */ -static void -get_oper_expr(OpExpr *expr, deparse_context *context) -{ - StringInfo buf = context->buf; - Oid opno = expr->opno; - List *args = expr->args; - - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - if (list_length(args) == 2) - { - /* binary operator */ - Node *arg1 = (Node *) linitial(args); - Node *arg2 = (Node *) lsecond(args); - - get_rule_expr_paren(arg1, context, true, (Node *) expr); - appendStringInfo(buf, " %s ", - generate_operator_name(opno, - exprType(arg1), - exprType(arg2))); - get_rule_expr_paren(arg2, context, true, (Node *) expr); - } - else - { - /* prefix operator */ - Node *arg = (Node *) linitial(args); - - appendStringInfo(buf, "%s ", - generate_operator_name(opno, - InvalidOid, - exprType(arg))); - get_rule_expr_paren(arg, context, true, (Node *) expr); - } - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); -} - -/* - * get_func_expr - Parse back a FuncExpr node - */ -static void -get_func_expr(FuncExpr *expr, deparse_context *context, - bool showimplicit) -{ - StringInfo buf = context->buf; - Oid funcoid = expr->funcid; - Oid argtypes[FUNC_MAX_ARGS]; - int nargs; - List *argnames; - bool use_variadic; - ListCell *l; - - /* - * If the function call came from an implicit coercion, then just show the - * first argument --- unless caller wants to see implicit coercions. - */ - if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) - { - get_rule_expr_paren((Node *) linitial(expr->args), context, - false, (Node *) expr); - return; - } - - /* - * If the function call came from a cast, then show the first argument - * plus an explicit cast operation. - */ - if (expr->funcformat == COERCE_EXPLICIT_CAST || - expr->funcformat == COERCE_IMPLICIT_CAST) - { - Node *arg = linitial(expr->args); - Oid rettype = expr->funcresulttype; - int32 coercedTypmod; - - /* Get the typmod if this is a length-coercion function */ - (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); - - get_coercion_expr(arg, context, - rettype, coercedTypmod, - (Node *) expr); - - return; - } - - /* - * If the function was called using one of the SQL spec's random special - * syntaxes, try to reproduce that. If we don't recognize the function, - * fall through. - */ - if (expr->funcformat == COERCE_SQL_SYNTAX) - { - if (get_func_sql_syntax(expr, context)) - return; - } - - - /* - * Normal function: display as proname(args). First we need to extract - * the argument datatypes. - */ - if (list_length(expr->args) > FUNC_MAX_ARGS) - ereport(ERROR, - (errcode(ERRCODE_TOO_MANY_ARGUMENTS), - errmsg("too many arguments"))); - nargs = 0; - argnames = NIL; - foreach(l, expr->args) - { - Node *arg = (Node *) lfirst(l); - - if (IsA(arg, NamedArgExpr)) - argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); - argtypes[nargs] = exprType(arg); - nargs++; - } - - appendStringInfo(buf, "%s(", - generate_function_name(funcoid, nargs, - argnames, argtypes, - expr->funcvariadic, - &use_variadic, - context->special_exprkind)); - nargs = 0; - foreach(l, expr->args) - { - if (nargs++ > 0) - appendStringInfoString(buf, ", "); - if (use_variadic && lnext(expr->args, l) == NULL) - appendStringInfoString(buf, "VARIADIC "); - get_rule_expr((Node *) lfirst(l), context, true); - } - - appendStringInfoChar(buf, ')'); -} - - -/* - * get_proc_expr - Parse back a CallStmt node - */ -static void -get_proc_expr(CallStmt *stmt, deparse_context *context, - bool showimplicit) -{ - StringInfo buf = context->buf; - Oid functionOid = stmt->funcexpr->funcid; - bool use_variadic; - Oid *argumentTypes; - List *finalArgumentList = NIL; - ListCell *argumentCell; - List *namedArgList = NIL; - int numberOfArgs = -1; - - if (!get_merged_argument_list(stmt, &namedArgList, &argumentTypes, - &finalArgumentList, &numberOfArgs)) - { - /* Nothing merged i.e. no OUT arguments */ - get_func_expr((FuncExpr *) stmt->funcexpr, context, showimplicit); - return; - } - - appendStringInfo(buf, "%s(", - generate_function_name(functionOid, numberOfArgs, - namedArgList, argumentTypes, - stmt->funcexpr->funcvariadic, - &use_variadic, - context->special_exprkind)); - int argNumber = 0; - foreach(argumentCell, finalArgumentList) - { - if (argNumber++ > 0) - appendStringInfoString(buf, ", "); - if (use_variadic && lnext(finalArgumentList, argumentCell) == NULL) - appendStringInfoString(buf, "VARIADIC "); - get_rule_expr((Node *) lfirst(argumentCell), context, true); - argNumber++; - } - - appendStringInfoChar(buf, ')'); -} - -/* - * get_agg_expr - Parse back an Aggref node - */ -static void -get_agg_expr(Aggref *aggref, deparse_context *context, - Aggref *original_aggref) -{ - StringInfo buf = context->buf; - Oid argtypes[FUNC_MAX_ARGS]; - int nargs; - bool use_variadic; - - /* - * For a combining aggregate, we look up and deparse the corresponding - * partial aggregate instead. This is necessary because our input - * argument list has been replaced; the new argument list always has just - * one element, which will point to a partial Aggref that supplies us with - * transition states to combine. - */ - if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) - { - TargetEntry *tle; - - - Assert(list_length(aggref->args) == 1); - tle = linitial_node(TargetEntry, aggref->args); - resolve_special_varno((Node *) tle->expr, context, - get_agg_combine_expr, original_aggref); - return; - } - - /* - * Mark as PARTIAL, if appropriate. We look to the original aggref so as - * to avoid printing this when recursing from the code just above. - */ - if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) - appendStringInfoString(buf, "PARTIAL "); - - /* Extract the argument types as seen by the parser */ - nargs = get_aggregate_argtypes(aggref, argtypes); - - /* Print the aggregate name, schema-qualified if needed */ - appendStringInfo(buf, "%s(%s", - generate_function_name(aggref->aggfnoid, nargs, - NIL, argtypes, - aggref->aggvariadic, - &use_variadic, - context->special_exprkind), - (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); - - if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) - { - /* - * Ordered-set aggregates do not use "*" syntax. Also, we needn't - * worry about inserting VARIADIC. So we can just dump the direct - * args as-is. - */ - Assert(!aggref->aggvariadic); - get_rule_expr((Node *) aggref->aggdirectargs, context, true); - Assert(aggref->aggorder != NIL); - appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); - get_rule_orderby(aggref->aggorder, aggref->args, false, context); - } - else - { - /* aggstar can be set only in zero-argument aggregates */ - if (aggref->aggstar) - appendStringInfoChar(buf, '*'); - else - { - ListCell *l; - int i; - - i = 0; - foreach(l, aggref->args) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - Node *arg = (Node *) tle->expr; - - Assert(!IsA(arg, NamedArgExpr)); - if (tle->resjunk) - continue; - if (i++ > 0) - appendStringInfoString(buf, ", "); - if (use_variadic && i == nargs) - appendStringInfoString(buf, "VARIADIC "); - get_rule_expr(arg, context, true); - } - } - - if (aggref->aggorder != NIL) - { - appendStringInfoString(buf, " ORDER BY "); - get_rule_orderby(aggref->aggorder, aggref->args, false, context); - } - } - - if (aggref->aggfilter != NULL) - { - appendStringInfoString(buf, ") FILTER (WHERE "); - get_rule_expr((Node *) aggref->aggfilter, context, false); - } - - appendStringInfoChar(buf, ')'); -} - -/* - * This is a helper function for get_agg_expr(). It's used when we deparse - * a combining Aggref; resolve_special_varno locates the corresponding partial - * Aggref and then calls this. - */ -static void -get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) -{ - Aggref *aggref; - Aggref *original_aggref = callback_arg; - - if (!IsA(node, Aggref)) - elog(ERROR, "combining Aggref does not point to an Aggref"); - - aggref = (Aggref *) node; - get_agg_expr(aggref, context, original_aggref); -} - -/* - * get_windowfunc_expr - Parse back a WindowFunc node - */ -static void -get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) -{ - StringInfo buf = context->buf; - Oid argtypes[FUNC_MAX_ARGS]; - int nargs; - List *argnames; - ListCell *l; - - if (list_length(wfunc->args) > FUNC_MAX_ARGS) - ereport(ERROR, - (errcode(ERRCODE_TOO_MANY_ARGUMENTS), - errmsg("too many arguments"))); - nargs = 0; - argnames = NIL; - foreach(l, wfunc->args) - { - Node *arg = (Node *) lfirst(l); - - if (IsA(arg, NamedArgExpr)) - argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); - argtypes[nargs] = exprType(arg); - nargs++; - } - - appendStringInfo(buf, "%s(", - generate_function_name(wfunc->winfnoid, nargs, - argnames, argtypes, - false, NULL, - context->special_exprkind)); - /* winstar can be set only in zero-argument aggregates */ - if (wfunc->winstar) - appendStringInfoChar(buf, '*'); - else - get_rule_expr((Node *) wfunc->args, context, true); - - if (wfunc->aggfilter != NULL) - { - appendStringInfoString(buf, ") FILTER (WHERE "); - get_rule_expr((Node *) wfunc->aggfilter, context, false); - } - - appendStringInfoString(buf, ") OVER "); - - foreach(l, context->windowClause) - { - WindowClause *wc = (WindowClause *) lfirst(l); - - if (wc->winref == wfunc->winref) - { - if (wc->name) - appendStringInfoString(buf, quote_identifier(wc->name)); - else - get_rule_windowspec(wc, context->windowTList, context); - break; - } - } - if (l == NULL) - { - if (context->windowClause) - elog(ERROR, "could not find window clause for winref %u", - wfunc->winref); - - /* - * In EXPLAIN, we don't have window context information available, so - * we have to settle for this: - */ - appendStringInfoString(buf, "(?)"); - } -} - - -/* - * get_func_sql_syntax - Parse back a SQL-syntax function call - * - * Returns true if we successfully deparsed, false if we did not - * recognize the function. - */ -static bool -get_func_sql_syntax(FuncExpr *expr, deparse_context *context) -{ - StringInfo buf = context->buf; - Oid funcoid = expr->funcid; - - switch (funcoid) - { - case F_TIMEZONE_INTERVAL_TIMESTAMP: - case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: - case F_TIMEZONE_INTERVAL_TIMETZ: - case F_TIMEZONE_TEXT_TIMESTAMP: - case F_TIMEZONE_TEXT_TIMESTAMPTZ: - case F_TIMEZONE_TEXT_TIMETZ: - /* AT TIME ZONE ... note reversed argument order */ - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, " AT TIME ZONE "); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: - case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: - case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: - case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: - case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: - case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: - case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: - case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: - case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: - case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: - case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: - case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: - case F_OVERLAPS_TIME_TIME_TIME_TIME: - /* (x1, x2) OVERLAPS (y1, y2) */ - appendStringInfoString(buf, "(("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, ", "); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, ") OVERLAPS ("); - get_rule_expr((Node *) lthird(expr->args), context, false); - appendStringInfoString(buf, ", "); - get_rule_expr((Node *) lfourth(expr->args), context, false); - appendStringInfoString(buf, "))"); - return true; - - case F_EXTRACT_TEXT_DATE: - case F_EXTRACT_TEXT_TIME: - case F_EXTRACT_TEXT_TIMETZ: - case F_EXTRACT_TEXT_TIMESTAMP: - case F_EXTRACT_TEXT_TIMESTAMPTZ: - case F_EXTRACT_TEXT_INTERVAL: - /* EXTRACT (x FROM y) */ - appendStringInfoString(buf, "EXTRACT("); - { - Const *con = (Const *) linitial(expr->args); - - Assert(IsA(con, Const) && - con->consttype == TEXTOID && - !con->constisnull); - appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); - } - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_IS_NORMALIZED: - /* IS xxx NORMALIZED */ - appendStringInfoString(buf, "(("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, ") IS"); - if (list_length(expr->args) == 2) - { - Const *con = (Const *) lsecond(expr->args); - - Assert(IsA(con, Const) && - con->consttype == TEXTOID && - !con->constisnull); - appendStringInfo(buf, " %s", - TextDatumGetCString(con->constvalue)); - } - appendStringInfoString(buf, " NORMALIZED)"); - return true; - - case F_PG_COLLATION_FOR: - /* COLLATION FOR */ - appendStringInfoString(buf, "COLLATION FOR ("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - /* - * XXX EXTRACT, a/k/a date_part(), is intentionally not covered - * yet. Add it after we change the return type to numeric. - */ - - case F_NORMALIZE: - /* NORMALIZE() */ - appendStringInfoString(buf, "NORMALIZE("); - get_rule_expr((Node *) linitial(expr->args), context, false); - if (list_length(expr->args) == 2) - { - Const *con = (Const *) lsecond(expr->args); - - Assert(IsA(con, Const) && - con->consttype == TEXTOID && - !con->constisnull); - appendStringInfo(buf, ", %s", - TextDatumGetCString(con->constvalue)); - } - appendStringInfoChar(buf, ')'); - return true; - - case F_OVERLAY_BIT_BIT_INT4: - case F_OVERLAY_BIT_BIT_INT4_INT4: - case F_OVERLAY_BYTEA_BYTEA_INT4: - case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: - case F_OVERLAY_TEXT_TEXT_INT4: - case F_OVERLAY_TEXT_TEXT_INT4_INT4: - /* OVERLAY() */ - appendStringInfoString(buf, "OVERLAY("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, " PLACING "); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) lthird(expr->args), context, false); - if (list_length(expr->args) == 4) - { - appendStringInfoString(buf, " FOR "); - get_rule_expr((Node *) lfourth(expr->args), context, false); - } - appendStringInfoChar(buf, ')'); - return true; - - case F_POSITION_BIT_BIT: - case F_POSITION_BYTEA_BYTEA: - case F_POSITION_TEXT_TEXT: - /* POSITION() ... extra parens since args are b_expr not a_expr */ - appendStringInfoString(buf, "POSITION(("); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, ") IN ("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, "))"); - return true; - - case F_SUBSTRING_BIT_INT4: - case F_SUBSTRING_BIT_INT4_INT4: - case F_SUBSTRING_BYTEA_INT4: - case F_SUBSTRING_BYTEA_INT4_INT4: - case F_SUBSTRING_TEXT_INT4: - case F_SUBSTRING_TEXT_INT4_INT4: - /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ - appendStringInfoString(buf, "SUBSTRING("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) lsecond(expr->args), context, false); - if (list_length(expr->args) == 3) - { - appendStringInfoString(buf, " FOR "); - get_rule_expr((Node *) lthird(expr->args), context, false); - } - appendStringInfoChar(buf, ')'); - return true; - - case F_SUBSTRING_TEXT_TEXT_TEXT: - /* SUBSTRING SIMILAR/ESCAPE */ - appendStringInfoString(buf, "SUBSTRING("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, " SIMILAR "); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, " ESCAPE "); - get_rule_expr((Node *) lthird(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_BTRIM_BYTEA_BYTEA: - case F_BTRIM_TEXT: - case F_BTRIM_TEXT_TEXT: - /* TRIM() */ - appendStringInfoString(buf, "TRIM(BOTH"); - if (list_length(expr->args) == 2) - { - appendStringInfoChar(buf, ' '); - get_rule_expr((Node *) lsecond(expr->args), context, false); - } - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_LTRIM_BYTEA_BYTEA: - case F_LTRIM_TEXT: - case F_LTRIM_TEXT_TEXT: - /* TRIM() */ - appendStringInfoString(buf, "TRIM(LEADING"); - if (list_length(expr->args) == 2) - { - appendStringInfoChar(buf, ' '); - get_rule_expr((Node *) lsecond(expr->args), context, false); - } - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_RTRIM_BYTEA_BYTEA: - case F_RTRIM_TEXT: - case F_RTRIM_TEXT_TEXT: - /* TRIM() */ - appendStringInfoString(buf, "TRIM(TRAILING"); - if (list_length(expr->args) == 2) - { - appendStringInfoChar(buf, ' '); - get_rule_expr((Node *) lsecond(expr->args), context, false); - } - appendStringInfoString(buf, " FROM "); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoChar(buf, ')'); - return true; - - case F_XMLEXISTS: - /* XMLEXISTS ... extra parens because args are c_expr */ - appendStringInfoString(buf, "XMLEXISTS(("); - get_rule_expr((Node *) linitial(expr->args), context, false); - appendStringInfoString(buf, ") PASSING ("); - get_rule_expr((Node *) lsecond(expr->args), context, false); - appendStringInfoString(buf, "))"); - return true; - } - return false; -} - - -/* ---------- - * get_coercion_expr - * - * Make a string representation of a value coerced to a specific type - * ---------- - */ -static void -get_coercion_expr(Node *arg, deparse_context *context, - Oid resulttype, int32 resulttypmod, - Node *parentNode) -{ - StringInfo buf = context->buf; - - /* - * Since parse_coerce.c doesn't immediately collapse application of - * length-coercion functions to constants, what we'll typically see in - * such cases is a Const with typmod -1 and a length-coercion function - * right above it. Avoid generating redundant output. However, beware of - * suppressing casts when the user actually wrote something like - * 'foo'::text::char(3). - * - * Note: it might seem that we are missing the possibility of needing to - * print a COLLATE clause for such a Const. However, a Const could only - * have nondefault collation in a post-constant-folding tree, in which the - * length coercion would have been folded too. See also the special - * handling of CollateExpr in coerce_to_target_type(): any collation - * marking will be above the coercion node, not below it. - */ - if (arg && IsA(arg, Const) && - ((Const *) arg)->consttype == resulttype && - ((Const *) arg)->consttypmod == -1) - { - /* Show the constant without normal ::typename decoration */ - get_const_expr((Const *) arg, context, -1); - } - else - { - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr_paren(arg, context, false, parentNode); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - appendStringInfo(buf, "::%s", - format_type_with_typemod(resulttype, resulttypmod)); -} - -/* ---------- - * get_const_expr - * - * Make a string representation of a Const - * - * showtype can be -1 to never show "::typename" decoration, or +1 to always - * show it, or 0 to show it only if the constant wouldn't be assumed to be - * the right type by default. - * - * If the Const's collation isn't default for its type, show that too. - * We mustn't do this when showtype is -1 (since that means the caller will - * print "::typename", and we can't put a COLLATE clause in between). It's - * caller's responsibility that collation isn't missed in such cases. - * ---------- - */ -static void -get_const_expr(Const *constval, deparse_context *context, int showtype) -{ - StringInfo buf = context->buf; - Oid typoutput; - bool typIsVarlena; - char *extval; - bool needlabel = false; - - if (constval->constisnull) - { - /* - * Always label the type of a NULL constant to prevent misdecisions - * about type when reparsing. - */ - appendStringInfoString(buf, "NULL"); - if (showtype >= 0) - { - appendStringInfo(buf, "::%s", - format_type_with_typemod(constval->consttype, - constval->consttypmod)); - get_const_collation(constval, context); - } - return; - } - - getTypeOutputInfo(constval->consttype, - &typoutput, &typIsVarlena); - - extval = OidOutputFunctionCall(typoutput, constval->constvalue); - - switch (constval->consttype) - { - case INT4OID: - - /* - * INT4 can be printed without any decoration, unless it is - * negative; in that case print it as '-nnn'::integer to ensure - * that the output will re-parse as a constant, not as a constant - * plus operator. In most cases we could get away with printing - * (-nnn) instead, because of the way that gram.y handles negative - * literals; but that doesn't work for INT_MIN, and it doesn't - * seem that much prettier anyway. - */ - if (extval[0] != '-') - appendStringInfoString(buf, extval); - else - { - appendStringInfo(buf, "'%s'", extval); - needlabel = true; /* we must attach a cast */ - } - break; - - case NUMERICOID: - - /* - * NUMERIC can be printed without quotes if it looks like a float - * constant (not an integer, and not Infinity or NaN) and doesn't - * have a leading sign (for the same reason as for INT4). - */ - if (isdigit((unsigned char) extval[0]) && - strcspn(extval, "eE.") != strlen(extval)) - { - appendStringInfoString(buf, extval); - } - else - { - appendStringInfo(buf, "'%s'", extval); - needlabel = true; /* we must attach a cast */ - } - break; - - case BITOID: - case VARBITOID: - appendStringInfo(buf, "B'%s'", extval); - break; - - case BOOLOID: - if (strcmp(extval, "t") == 0) - appendStringInfoString(buf, "true"); - else - appendStringInfoString(buf, "false"); - break; - - default: - simple_quote_literal(buf, extval); - break; - } - - pfree(extval); - - if (showtype < 0) - return; - - /* - * For showtype == 0, append ::typename unless the constant will be - * implicitly typed as the right type when it is read in. - * - * XXX this code has to be kept in sync with the behavior of the parser, - * especially make_const. - */ - switch (constval->consttype) - { - case BOOLOID: - case UNKNOWNOID: - /* These types can be left unlabeled */ - needlabel = false; - break; - case INT4OID: - /* We determined above whether a label is needed */ - break; - case NUMERICOID: - - /* - * Float-looking constants will be typed as numeric, which we - * checked above; but if there's a nondefault typmod we need to - * show it. - */ - needlabel |= (constval->consttypmod >= 0); - break; - default: - needlabel = true; - break; - } - if (needlabel || showtype > 0) - appendStringInfo(buf, "::%s", - format_type_with_typemod(constval->consttype, - constval->consttypmod)); - - get_const_collation(constval, context); -} - -/* - * helper for get_const_expr: append COLLATE if needed - */ -static void -get_const_collation(Const *constval, deparse_context *context) -{ - StringInfo buf = context->buf; - - if (OidIsValid(constval->constcollid)) - { - Oid typcollation = get_typcollation(constval->consttype); - - if (constval->constcollid != typcollation) - { - appendStringInfo(buf, " COLLATE %s", - generate_collation_name(constval->constcollid)); - } - } -} - -/* - * simple_quote_literal - Format a string as a SQL literal, append to buf - */ -static void -simple_quote_literal(StringInfo buf, const char *val) -{ - const char *valptr; - - /* - * We form the string literal according to the prevailing setting of - * standard_conforming_strings; we never use E''. User is responsible for - * making sure result is used correctly. - */ - appendStringInfoChar(buf, '\''); - for (valptr = val; *valptr; valptr++) - { - char ch = *valptr; - - if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) - appendStringInfoChar(buf, ch); - appendStringInfoChar(buf, ch); - } - appendStringInfoChar(buf, '\''); -} - - -/* ---------- - * get_sublink_expr - Parse back a sublink - * ---------- - */ -static void -get_sublink_expr(SubLink *sublink, deparse_context *context) -{ - StringInfo buf = context->buf; - Query *query = (Query *) (sublink->subselect); - char *opname = NULL; - bool need_paren; - - if (sublink->subLinkType == ARRAY_SUBLINK) - appendStringInfoString(buf, "ARRAY("); - else - appendStringInfoChar(buf, '('); - - /* - * Note that we print the name of only the first operator, when there are - * multiple combining operators. This is an approximation that could go - * wrong in various scenarios (operators in different schemas, renamed - * operators, etc) but there is not a whole lot we can do about it, since - * the syntax allows only one operator to be shown. - */ - if (sublink->testexpr) - { - if (IsA(sublink->testexpr, OpExpr)) - { - /* single combining operator */ - OpExpr *opexpr = (OpExpr *) sublink->testexpr; - - get_rule_expr(linitial(opexpr->args), context, true); - opname = generate_operator_name(opexpr->opno, - exprType(linitial(opexpr->args)), - exprType(lsecond(opexpr->args))); - } - else if (IsA(sublink->testexpr, BoolExpr)) - { - /* multiple combining operators, = or <> cases */ - char *sep; - ListCell *l; - - appendStringInfoChar(buf, '('); - sep = ""; - foreach(l, ((BoolExpr *) sublink->testexpr)->args) - { - OpExpr *opexpr = lfirst_node(OpExpr, l); - - appendStringInfoString(buf, sep); - get_rule_expr(linitial(opexpr->args), context, true); - if (!opname) - opname = generate_operator_name(opexpr->opno, - exprType(linitial(opexpr->args)), - exprType(lsecond(opexpr->args))); - sep = ", "; - } - appendStringInfoChar(buf, ')'); - } - else if (IsA(sublink->testexpr, RowCompareExpr)) - { - /* multiple combining operators, < <= > >= cases */ - RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; - - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) rcexpr->largs, context, true); - opname = generate_operator_name(linitial_oid(rcexpr->opnos), - exprType(linitial(rcexpr->largs)), - exprType(linitial(rcexpr->rargs))); - appendStringInfoChar(buf, ')'); - } - else - elog(ERROR, "unrecognized testexpr type: %d", - (int) nodeTag(sublink->testexpr)); - } - - need_paren = true; - - switch (sublink->subLinkType) - { - case EXISTS_SUBLINK: - appendStringInfoString(buf, "EXISTS "); - break; - - case ANY_SUBLINK: - if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ - appendStringInfoString(buf, " IN "); - else - appendStringInfo(buf, " %s ANY ", opname); - break; - - case ALL_SUBLINK: - appendStringInfo(buf, " %s ALL ", opname); - break; - - case ROWCOMPARE_SUBLINK: - appendStringInfo(buf, " %s ", opname); - break; - - case EXPR_SUBLINK: - case MULTIEXPR_SUBLINK: - case ARRAY_SUBLINK: - need_paren = false; - break; - - case CTE_SUBLINK: /* shouldn't occur in a SubLink */ - default: - elog(ERROR, "unrecognized sublink type: %d", - (int) sublink->subLinkType); - break; - } - - if (need_paren) - appendStringInfoChar(buf, '('); - - get_query_def(query, buf, context->namespaces, NULL, - context->prettyFlags, context->wrapColumn, - context->indentLevel); - - if (need_paren) - appendStringInfoString(buf, "))"); - else - appendStringInfoChar(buf, ')'); -} - - -/* ---------- - * get_tablefunc - Parse back a table function - * ---------- - */ -static void -get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) -{ - StringInfo buf = context->buf; - - /* XMLTABLE is the only existing implementation. */ - - appendStringInfoString(buf, "XMLTABLE("); - - if (tf->ns_uris != NIL) - { - ListCell *lc1, - *lc2; - bool first = true; - - appendStringInfoString(buf, "XMLNAMESPACES ("); - forboth(lc1, tf->ns_uris, lc2, tf->ns_names) - { - Node *expr = (Node *) lfirst(lc1); - char *name = strVal(lfirst(lc2)); - - if (!first) - appendStringInfoString(buf, ", "); - else - first = false; - - if (name != NULL) - { - get_rule_expr(expr, context, showimplicit); - appendStringInfo(buf, " AS %s", name); - } - else - { - appendStringInfoString(buf, "DEFAULT "); - get_rule_expr(expr, context, showimplicit); - } - } - appendStringInfoString(buf, "), "); - } - - appendStringInfoChar(buf, '('); - get_rule_expr((Node *) tf->rowexpr, context, showimplicit); - appendStringInfoString(buf, ") PASSING ("); - get_rule_expr((Node *) tf->docexpr, context, showimplicit); - appendStringInfoChar(buf, ')'); - - if (tf->colexprs != NIL) - { - ListCell *l1; - ListCell *l2; - ListCell *l3; - ListCell *l4; - ListCell *l5; - int colnum = 0; - - appendStringInfoString(buf, " COLUMNS "); - forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, - l4, tf->colexprs, l5, tf->coldefexprs) - { - char *colname = strVal(lfirst(l1)); - Oid typid = lfirst_oid(l2); - int32 typmod = lfirst_int(l3); - Node *colexpr = (Node *) lfirst(l4); - Node *coldefexpr = (Node *) lfirst(l5); - bool ordinality = (tf->ordinalitycol == colnum); - bool notnull = bms_is_member(colnum, tf->notnulls); - - if (colnum > 0) - appendStringInfoString(buf, ", "); - colnum++; - - appendStringInfo(buf, "%s %s", quote_identifier(colname), - ordinality ? "FOR ORDINALITY" : - format_type_with_typemod(typid, typmod)); - if (ordinality) - continue; - - if (coldefexpr != NULL) - { - appendStringInfoString(buf, " DEFAULT ("); - get_rule_expr((Node *) coldefexpr, context, showimplicit); - appendStringInfoChar(buf, ')'); - } - if (colexpr != NULL) - { - appendStringInfoString(buf, " PATH ("); - get_rule_expr((Node *) colexpr, context, showimplicit); - appendStringInfoChar(buf, ')'); - } - if (notnull) - appendStringInfoString(buf, " NOT NULL"); - } - } - - appendStringInfoChar(buf, ')'); -} - -/* ---------- - * get_from_clause - Parse back a FROM clause - * - * "prefix" is the keyword that denotes the start of the list of FROM - * elements. It is FROM when used to parse back SELECT and UPDATE, but - * is USING when parsing back DELETE. - * ---------- - */ -static void -get_from_clause(Query *query, const char *prefix, deparse_context *context) -{ - StringInfo buf = context->buf; - bool first = true; - ListCell *l; - - /* - * We use the query's jointree as a guide to what to print. However, we - * must ignore auto-added RTEs that are marked not inFromCl. (These can - * only appear at the top level of the jointree, so it's sufficient to - * check here.) This check also ensures we ignore the rule pseudo-RTEs - * for NEW and OLD. - */ - foreach(l, query->jointree->fromlist) - { - Node *jtnode = (Node *) lfirst(l); - - if (IsA(jtnode, RangeTblRef)) - { - int varno = ((RangeTblRef *) jtnode)->rtindex; - RangeTblEntry *rte = rt_fetch(varno, query->rtable); - - if (!rte->inFromCl) - continue; - } - - if (first) - { - appendContextKeyword(context, prefix, - -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); - first = false; - - get_from_clause_item(jtnode, query, context); - } - else - { - StringInfoData itembuf; - - appendStringInfoString(buf, ", "); - - /* - * Put the new FROM item's text into itembuf so we can decide - * after we've got it whether or not it needs to go on a new line. - */ - initStringInfo(&itembuf); - context->buf = &itembuf; - - get_from_clause_item(jtnode, query, context); - - /* Restore context's output buffer */ - context->buf = buf; - - /* Consider line-wrapping if enabled */ - if (PRETTY_INDENT(context) && context->wrapColumn >= 0) - { - /* Does the new item start with a new line? */ - if (itembuf.len > 0 && itembuf.data[0] == '\n') - { - /* If so, we shouldn't add anything */ - /* instead, remove any trailing spaces currently in buf */ - removeStringInfoSpaces(buf); - } - else - { - char *trailing_nl; - - /* Locate the start of the current line in the buffer */ - trailing_nl = strrchr(buf->data, '\n'); - if (trailing_nl == NULL) - trailing_nl = buf->data; - else - trailing_nl++; - - /* - * Add a newline, plus some indentation, if the new item - * would cause an overflow. - */ - if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) - appendContextKeyword(context, "", -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_VAR); - } - } - - /* Add the new item */ - appendStringInfoString(buf, itembuf.data); - - /* clean up */ - pfree(itembuf.data); - } - } -} - -static void -get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) -{ - StringInfo buf = context->buf; - deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); - - if (IsA(jtnode, RangeTblRef)) - { - int varno = ((RangeTblRef *) jtnode)->rtindex; - RangeTblEntry *rte = rt_fetch(varno, query->rtable); - char *refname = get_rtable_name(varno, context); - deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); - RangeTblFunction *rtfunc1 = NULL; - bool printalias; - CitusRTEKind rteKind = GetRangeTblKind(rte); - - if (rte->lateral) - appendStringInfoString(buf, "LATERAL "); - - /* Print the FROM item proper */ - switch (rte->rtekind) - { - case RTE_RELATION: - /* Normal relation RTE */ - appendStringInfo(buf, "%s%s", - only_marker(rte), - generate_relation_or_shard_name(rte->relid, - context->distrelid, - context->shardid, - context->namespaces)); - break; - case RTE_SUBQUERY: - /* Subquery RTE */ - appendStringInfoChar(buf, '('); - get_query_def(rte->subquery, buf, context->namespaces, NULL, - context->prettyFlags, context->wrapColumn, - context->indentLevel); - appendStringInfoChar(buf, ')'); - break; - case RTE_FUNCTION: - /* if it's a shard, do differently */ - if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) - { - char *fragmentSchemaName = NULL; - char *fragmentTableName = NULL; - - ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); - - /* use schema and table name from the remote alias */ - appendStringInfo(buf, "%s%s", - only_marker(rte), - generate_fragment_name(fragmentSchemaName, - fragmentTableName)); - break; - } - - /* Function RTE */ - rtfunc1 = (RangeTblFunction *) linitial(rte->functions); - - /* - * Omit ROWS FROM() syntax for just one function, unless it - * has both a coldeflist and WITH ORDINALITY. If it has both, - * we must use ROWS FROM() syntax to avoid ambiguity about - * whether the coldeflist includes the ordinality column. - */ - if (list_length(rte->functions) == 1 && - (rtfunc1->funccolnames == NIL || !rte->funcordinality)) - { - get_rule_expr_funccall(rtfunc1->funcexpr, context, true); - /* we'll print the coldeflist below, if it has one */ - } - else - { - bool all_unnest; - ListCell *lc; - - /* - * If all the function calls in the list are to unnest, - * and none need a coldeflist, then collapse the list back - * down to UNNEST(args). (If we had more than one - * built-in unnest function, this would get more - * difficult.) - * - * XXX This is pretty ugly, since it makes not-terribly- - * future-proof assumptions about what the parser would do - * with the output; but the alternative is to emit our - * nonstandard ROWS FROM() notation for what might have - * been a perfectly spec-compliant multi-argument - * UNNEST(). - */ - all_unnest = true; - foreach(lc, rte->functions) - { - RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); - - if (!IsA(rtfunc->funcexpr, FuncExpr) || - ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || - rtfunc->funccolnames != NIL) - { - all_unnest = false; - break; - } - } - - if (all_unnest) - { - List *allargs = NIL; - - foreach(lc, rte->functions) - { - RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); - List *args = ((FuncExpr *) rtfunc->funcexpr)->args; - - allargs = list_concat(allargs, args); - } - - appendStringInfoString(buf, "UNNEST("); - get_rule_expr((Node *) allargs, context, true); - appendStringInfoChar(buf, ')'); - } - else - { - int funcno = 0; - - appendStringInfoString(buf, "ROWS FROM("); - foreach(lc, rte->functions) - { - RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); - - if (funcno > 0) - appendStringInfoString(buf, ", "); - get_rule_expr_funccall(rtfunc->funcexpr, context, true); - if (rtfunc->funccolnames != NIL) - { - /* Reconstruct the column definition list */ - appendStringInfoString(buf, " AS "); - get_from_clause_coldeflist(rtfunc, - NULL, - context); - } - funcno++; - } - appendStringInfoChar(buf, ')'); - } - /* prevent printing duplicate coldeflist below */ - rtfunc1 = NULL; - } - if (rte->funcordinality) - appendStringInfoString(buf, " WITH ORDINALITY"); - break; - case RTE_TABLEFUNC: - get_tablefunc(rte->tablefunc, context, true); - break; - case RTE_VALUES: - /* Values list RTE */ - appendStringInfoChar(buf, '('); - get_values_def(rte->values_lists, context); - appendStringInfoChar(buf, ')'); - break; - case RTE_CTE: - appendStringInfoString(buf, quote_identifier(rte->ctename)); - break; - default: - elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); - break; - } - - /* Print the relation alias, if needed */ - printalias = false; - if (rte->alias != NULL) - { - /* Always print alias if user provided one */ - printalias = true; - } - else if (colinfo->printaliases) - { - /* Always print alias if we need to print column aliases */ - printalias = true; - } - else if (rte->rtekind == RTE_RELATION) - { - /* - * No need to print alias if it's same as relation name (this - * would normally be the case, but not if set_rtable_names had to - * resolve a conflict). - */ - if (strcmp(refname, get_relation_name(rte->relid)) != 0) - printalias = true; - } - else if (rte->rtekind == RTE_FUNCTION) - { - /* - * For a function RTE, always print alias. This covers possible - * renaming of the function and/or instability of the - * FigureColname rules for things that aren't simple functions. - * Note we'd need to force it anyway for the columndef list case. - */ - printalias = true; - } - else if (rte->rtekind == RTE_VALUES) - { - /* Alias is syntactically required for VALUES */ - printalias = true; - } - else if (rte->rtekind == RTE_CTE) - { - /* - * No need to print alias if it's same as CTE name (this would - * normally be the case, but not if set_rtable_names had to - * resolve a conflict). - */ - if (strcmp(refname, rte->ctename) != 0) - printalias = true; - } - else if (rte->rtekind == RTE_SUBQUERY) - { - /* subquery requires alias too */ - printalias = true; - } - if (printalias) - appendStringInfo(buf, " %s", quote_identifier(refname)); - - /* Print the column definitions or aliases, if needed */ - if (rtfunc1 && rtfunc1->funccolnames != NIL) - { - /* Reconstruct the columndef list, which is also the aliases */ - get_from_clause_coldeflist(rtfunc1, colinfo, context); - } - else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD || - (rte->alias != NULL && rte->alias->colnames != NIL)) - { - /* Else print column aliases as needed */ - get_column_alias_list(colinfo, context); - } - /* check if column's are given aliases in distributed tables */ - else if (colinfo->parentUsing != NIL) - { - Assert(colinfo->printaliases); - get_column_alias_list(colinfo, context); - } - - /* Tablesample clause must go after any alias */ - if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && - rte->tablesample) - { - get_tablesample_def(rte->tablesample, context); - } - } - else if (IsA(jtnode, JoinExpr)) - { - JoinExpr *j = (JoinExpr *) jtnode; - deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); - bool need_paren_on_right; - - need_paren_on_right = PRETTY_PAREN(context) && - !IsA(j->rarg, RangeTblRef) && - !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); - - if (!PRETTY_PAREN(context) || j->alias != NULL) - appendStringInfoChar(buf, '('); - - get_from_clause_item(j->larg, query, context); - - switch (j->jointype) - { - case JOIN_INNER: - if (j->quals) - appendContextKeyword(context, " JOIN ", - -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_JOIN); - else - appendContextKeyword(context, " CROSS JOIN ", - -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_JOIN); - break; - case JOIN_LEFT: - appendContextKeyword(context, " LEFT JOIN ", - -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_JOIN); - break; - case JOIN_FULL: - appendContextKeyword(context, " FULL JOIN ", - -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_JOIN); - break; - case JOIN_RIGHT: - appendContextKeyword(context, " RIGHT JOIN ", - -PRETTYINDENT_STD, - PRETTYINDENT_STD, - PRETTYINDENT_JOIN); - break; - default: - elog(ERROR, "unrecognized join type: %d", - (int) j->jointype); - } - - if (need_paren_on_right) - appendStringInfoChar(buf, '('); - get_from_clause_item(j->rarg, query, context); - if (need_paren_on_right) - appendStringInfoChar(buf, ')'); - - if (j->usingClause) - { - ListCell *lc; - bool first = true; - - appendStringInfoString(buf, " USING ("); - /* Use the assigned names, not what's in usingClause */ - foreach(lc, colinfo->usingNames) - { - char *colname = (char *) lfirst(lc); - - if (first) - first = false; - else - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, quote_identifier(colname)); - } - appendStringInfoChar(buf, ')'); - - if (j->join_using_alias) - appendStringInfo(buf, " AS %s", - quote_identifier(j->join_using_alias->aliasname)); - } - else if (j->quals) - { - appendStringInfoString(buf, " ON "); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, '('); - get_rule_expr(j->quals, context, false); - if (!PRETTY_PAREN(context)) - appendStringInfoChar(buf, ')'); - } - else if (j->jointype != JOIN_INNER) - { - /* If we didn't say CROSS JOIN above, we must provide an ON */ - appendStringInfoString(buf, " ON TRUE"); - } - - if (!PRETTY_PAREN(context) || j->alias != NULL) - appendStringInfoChar(buf, ')'); - - /* Yes, it's correct to put alias after the right paren ... */ - if (j->alias != NULL) - { - /* - * Note that it's correct to emit an alias clause if and only if - * there was one originally. Otherwise we'd be converting a named - * join to unnamed or vice versa, which creates semantic - * subtleties we don't want. However, we might print a different - * alias name than was there originally. - */ - appendStringInfo(buf, " %s", - quote_identifier(get_rtable_name(j->rtindex, - context))); - get_column_alias_list(colinfo, context); - } - } - else - elog(ERROR, "unrecognized node type: %d", - (int) nodeTag(jtnode)); -} - -/* - * get_column_alias_list - print column alias list for an RTE - * - * Caller must already have printed the relation's alias name. - */ -static void -get_column_alias_list(deparse_columns *colinfo, deparse_context *context) -{ - StringInfo buf = context->buf; - int i; - bool first = true; - - /* Don't print aliases if not needed */ - if (!colinfo->printaliases) - return; - - for (i = 0; i < colinfo->num_new_cols; i++) - { - char *colname = colinfo->new_colnames[i]; - - if (first) - { - appendStringInfoChar(buf, '('); - first = false; - } - else - appendStringInfoString(buf, ", "); - appendStringInfoString(buf, quote_identifier(colname)); - } - if (!first) - appendStringInfoChar(buf, ')'); -} - -/* - * get_from_clause_coldeflist - reproduce FROM clause coldeflist - * - * When printing a top-level coldeflist (which is syntactically also the - * relation's column alias list), use column names from colinfo. But when - * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the - * original coldeflist's names, which are available in rtfunc->funccolnames. - * Pass NULL for colinfo to select the latter behavior. - * - * The coldeflist is appended immediately (no space) to buf. Caller is - * responsible for ensuring that an alias or AS is present before it. - */ -static void -get_from_clause_coldeflist(RangeTblFunction *rtfunc, - deparse_columns *colinfo, - deparse_context *context) -{ - StringInfo buf = context->buf; - ListCell *l1; - ListCell *l2; - ListCell *l3; - ListCell *l4; - int i; - - appendStringInfoChar(buf, '('); - - i = 0; - forfour(l1, rtfunc->funccoltypes, - l2, rtfunc->funccoltypmods, - l3, rtfunc->funccolcollations, - l4, rtfunc->funccolnames) - { - Oid atttypid = lfirst_oid(l1); - int32 atttypmod = lfirst_int(l2); - Oid attcollation = lfirst_oid(l3); - char *attname; - - if (colinfo) - attname = colinfo->colnames[i]; - else - attname = strVal(lfirst(l4)); - - Assert(attname); /* shouldn't be any dropped columns here */ - - if (i > 0) - appendStringInfoString(buf, ", "); - appendStringInfo(buf, "%s %s", - quote_identifier(attname), - format_type_with_typemod(atttypid, atttypmod)); - if (OidIsValid(attcollation) && - attcollation != get_typcollation(atttypid)) - appendStringInfo(buf, " COLLATE %s", - generate_collation_name(attcollation)); - - i++; - } - - appendStringInfoChar(buf, ')'); -} - -/* - * get_tablesample_def - print a TableSampleClause - */ -static void -get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) -{ - StringInfo buf = context->buf; - Oid argtypes[1]; - int nargs; - ListCell *l; - - /* - * We should qualify the handler's function name if it wouldn't be - * resolved by lookup in the current search path. - */ - argtypes[0] = INTERNALOID; - appendStringInfo(buf, " TABLESAMPLE %s (", - generate_function_name(tablesample->tsmhandler, 1, - NIL, argtypes, - false, NULL, EXPR_KIND_NONE)); - - nargs = 0; - foreach(l, tablesample->args) - { - if (nargs++ > 0) - appendStringInfoString(buf, ", "); - get_rule_expr((Node *) lfirst(l), context, false); - } - appendStringInfoChar(buf, ')'); - - if (tablesample->repeatable != NULL) - { - appendStringInfoString(buf, " REPEATABLE ("); - get_rule_expr((Node *) tablesample->repeatable, context, false); - appendStringInfoChar(buf, ')'); - } -} - - -/* - * get_opclass_name - fetch name of an index operator class - * - * The opclass name is appended (after a space) to buf. - * - * Output is suppressed if the opclass is the default for the given - * actual_datatype. (If you don't want this behavior, just pass - * InvalidOid for actual_datatype.) - */ -static void -get_opclass_name(Oid opclass, Oid actual_datatype, - StringInfo buf) -{ - HeapTuple ht_opc; - Form_pg_opclass opcrec; - char *opcname; - char *nspname; - - ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); - if (!HeapTupleIsValid(ht_opc)) - elog(ERROR, "cache lookup failed for opclass %u", opclass); - opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); - - if (!OidIsValid(actual_datatype) || - GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) - { - /* Okay, we need the opclass name. Do we need to qualify it? */ - opcname = NameStr(opcrec->opcname); - if (OpclassIsVisible(opclass)) - appendStringInfo(buf, " %s", quote_identifier(opcname)); - else - { - nspname = get_namespace_name(opcrec->opcnamespace); - appendStringInfo(buf, " %s.%s", - quote_identifier(nspname), - quote_identifier(opcname)); - } - } - ReleaseSysCache(ht_opc); -} - -/* - * processIndirection - take care of array and subfield assignment - * - * We strip any top-level FieldStore or assignment SubscriptingRef nodes that - * appear in the input, printing them as decoration for the base column - * name (which we assume the caller just printed). We might also need to - * strip CoerceToDomain nodes, but only ones that appear above assignment - * nodes. - * - * Returns the subexpression that's to be assigned. - */ -static Node * -processIndirection(Node *node, deparse_context *context) -{ - StringInfo buf = context->buf; - CoerceToDomain *cdomain = NULL; - - for (;;) - { - if (node == NULL) - break; - if (IsA(node, FieldStore)) - { - FieldStore *fstore = (FieldStore *) node; - Oid typrelid; - char *fieldname; - - /* lookup tuple type */ - typrelid = get_typ_typrelid(fstore->resulttype); - if (!OidIsValid(typrelid)) - elog(ERROR, "argument type %s of FieldStore is not a tuple type", - format_type_be(fstore->resulttype)); - - /* - * Print the field name. There should only be one target field in - * stored rules. There could be more than that in executable - * target lists, but this function cannot be used for that case. - */ - Assert(list_length(fstore->fieldnums) == 1); - fieldname = get_attname(typrelid, - linitial_int(fstore->fieldnums), false); - appendStringInfo(buf, ".%s", quote_identifier(fieldname)); - - /* - * We ignore arg since it should be an uninteresting reference to - * the target column or subcolumn. - */ - node = (Node *) linitial(fstore->newvals); - } - else if (IsA(node, SubscriptingRef)) - { - SubscriptingRef *sbsref = (SubscriptingRef *) node; - - if (sbsref->refassgnexpr == NULL) - break; - printSubscripts(sbsref, context); - - /* - * We ignore refexpr since it should be an uninteresting reference - * to the target column or subcolumn. - */ - node = (Node *) sbsref->refassgnexpr; - } - else if (IsA(node, CoerceToDomain)) - { - cdomain = (CoerceToDomain *) node; - /* If it's an explicit domain coercion, we're done */ - if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) - break; - /* Tentatively descend past the CoerceToDomain */ - node = (Node *) cdomain->arg; - } - else - break; - } - - /* - * If we descended past a CoerceToDomain whose argument turned out not to - * be a FieldStore or array assignment, back up to the CoerceToDomain. - * (This is not enough to be fully correct if there are nested implicit - * CoerceToDomains, but such cases shouldn't ever occur.) - */ - if (cdomain && node == (Node *) cdomain->arg) - node = (Node *) cdomain; - - return node; -} - -static void -printSubscripts(SubscriptingRef *sbsref, deparse_context *context) -{ - StringInfo buf = context->buf; - ListCell *lowlist_item; - ListCell *uplist_item; - - lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ - foreach(uplist_item, sbsref->refupperindexpr) - { - appendStringInfoChar(buf, '['); - if (lowlist_item) - { - /* If subexpression is NULL, get_rule_expr prints nothing */ - get_rule_expr((Node *) lfirst(lowlist_item), context, false); - appendStringInfoChar(buf, ':'); - lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); - } - /* If subexpression is NULL, get_rule_expr prints nothing */ - get_rule_expr((Node *) lfirst(uplist_item), context, false); - appendStringInfoChar(buf, ']'); - } -} - -/* - * get_relation_name - * Get the unqualified name of a relation specified by OID - * - * This differs from the underlying get_rel_name() function in that it will - * throw error instead of silently returning NULL if the OID is bad. - */ -static char * -get_relation_name(Oid relid) -{ - char *relname = get_rel_name(relid); - - if (!relname) - elog(ERROR, "cache lookup failed for relation %u", relid); - return relname; -} - -/* - * generate_relation_or_shard_name - * Compute the name to display for a relation or shard - * - * If the provided relid is equal to the provided distrelid, this function - * returns a shard-extended relation name; otherwise, it falls through to a - * simple generate_relation_name call. - */ -static char * -generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, - List *namespaces) -{ - char *relname = NULL; - - if (relid == distrelid) - { - relname = get_relation_name(relid); - - if (shardid > 0) - { - Oid schemaOid = get_rel_namespace(relid); - char *schemaName = get_namespace_name(schemaOid); - - AppendShardIdToName(&relname, shardid); - - relname = quote_qualified_identifier(schemaName, relname); - } - } - else - { - relname = generate_relation_name(relid, namespaces); - } - - return relname; -} - -/* - * generate_relation_name - * Compute the name to display for a relation specified by OID - * - * The result includes all necessary quoting and schema-prefixing. - * - * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. - * We will forcibly qualify the relation name if it equals any CTE name - * visible in the namespace list. - */ -char * -generate_relation_name(Oid relid, List *namespaces) -{ - HeapTuple tp; - Form_pg_class reltup; - bool need_qual; - ListCell *nslist; - char *relname; - char *nspname; - char *result; - - tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); - if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for relation %u", relid); - reltup = (Form_pg_class) GETSTRUCT(tp); - relname = NameStr(reltup->relname); - - /* Check for conflicting CTE name */ - need_qual = false; - foreach(nslist, namespaces) - { - deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); - ListCell *ctlist; - - foreach(ctlist, dpns->ctes) - { - CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); - - if (strcmp(cte->ctename, relname) == 0) - { - need_qual = true; - break; - } - } - if (need_qual) - break; - } - - /* Otherwise, qualify the name if not visible in search path */ - if (!need_qual) - need_qual = !RelationIsVisible(relid); - - if (need_qual) - nspname = get_namespace_name(reltup->relnamespace); - else - nspname = NULL; - - result = quote_qualified_identifier(nspname, relname); - - ReleaseSysCache(tp); - - return result; -} - - -/* - * generate_rte_shard_name returns the qualified name of the shard given a - * CITUS_RTE_SHARD range table entry. - */ -static char * -generate_rte_shard_name(RangeTblEntry *rangeTableEntry) -{ - char *shardSchemaName = NULL; - char *shardTableName = NULL; - - Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); - - ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, - NULL); - - return generate_fragment_name(shardSchemaName, shardTableName); -} - - -/* - * generate_fragment_name - * Compute the name to display for a shard or merged table - * - * The result includes all necessary quoting and schema-prefixing. The schema - * name can be NULL for regular shards. For merged tables, they are always - * declared within a job-specific schema, and therefore can't have null schema - * names. - */ -static char * -generate_fragment_name(char *schemaName, char *tableName) -{ - StringInfo fragmentNameString = makeStringInfo(); - - if (schemaName != NULL) - { - appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), - quote_identifier(tableName)); - } - else - { - appendStringInfoString(fragmentNameString, quote_identifier(tableName)); - } - - return fragmentNameString->data; -} - -/* - * generate_function_name - * Compute the name to display for a function specified by OID, - * given that it is being called with the specified actual arg names and - * types. (Those matter because of ambiguous-function resolution rules.) - * - * If we're dealing with a potentially variadic function (in practice, this - * means a FuncExpr or Aggref, not some other way of calling a function), then - * has_variadic must specify whether variadic arguments have been merged, - * and *use_variadic_p will be set to indicate whether to print VARIADIC in - * the output. For non-FuncExpr cases, has_variadic should be false and - * use_variadic_p can be NULL. - * - * The result includes all necessary quoting and schema-prefixing. - */ -static char * -generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, - bool has_variadic, bool *use_variadic_p, - ParseExprKind special_exprkind) -{ - char *result; - HeapTuple proctup; - Form_pg_proc procform; - char *proname; - bool use_variadic; - char *nspname; - FuncDetailCode p_result; - Oid p_funcid; - Oid p_rettype; - bool p_retset; - int p_nvargs; - Oid p_vatype; - Oid *p_true_typeids; - bool force_qualify = false; - - proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); - if (!HeapTupleIsValid(proctup)) - elog(ERROR, "cache lookup failed for function %u", funcid); - procform = (Form_pg_proc) GETSTRUCT(proctup); - proname = NameStr(procform->proname); - - /* - * Due to parser hacks to avoid needing to reserve CUBE, we need to force - * qualification in some special cases. - */ - if (special_exprkind == EXPR_KIND_GROUP_BY) - { - if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) - force_qualify = true; - } - - /* - * Determine whether VARIADIC should be printed. We must do this first - * since it affects the lookup rules in func_get_detail(). - * - * Currently, we always print VARIADIC if the function has a merged - * variadic-array argument. Note that this is always the case for - * functions taking a VARIADIC argument type other than VARIADIC ANY. - * - * In principle, if VARIADIC wasn't originally specified and the array - * actual argument is deconstructable, we could print the array elements - * separately and not print VARIADIC, thus more nearly reproducing the - * original input. For the moment that seems like too much complication - * for the benefit, and anyway we do not know whether VARIADIC was - * originally specified if it's a non-ANY type. - */ - if (use_variadic_p) - { - /* Parser should not have set funcvariadic unless fn is variadic */ - Assert(!has_variadic || OidIsValid(procform->provariadic)); - use_variadic = has_variadic; - *use_variadic_p = use_variadic; - } - else - { - Assert(!has_variadic); - use_variadic = false; - } - - /* - * The idea here is to schema-qualify only if the parser would fail to - * resolve the correct function given the unqualified func name with the - * specified argtypes and VARIADIC flag. But if we already decided to - * force qualification, then we can skip the lookup and pretend we didn't - * find it. - */ - if (!force_qualify) - p_result = func_get_detail(list_make1(makeString(proname)), - NIL, argnames, nargs, argtypes, - !use_variadic, true, false, - &p_funcid, &p_rettype, - &p_retset, &p_nvargs, &p_vatype, - &p_true_typeids, NULL); - else - { - p_result = FUNCDETAIL_NOTFOUND; - p_funcid = InvalidOid; - } - - if ((p_result == FUNCDETAIL_NORMAL || - p_result == FUNCDETAIL_AGGREGATE || - p_result == FUNCDETAIL_WINDOWFUNC) && - p_funcid == funcid) - nspname = NULL; - else - nspname = get_namespace_name(procform->pronamespace); - - result = quote_qualified_identifier(nspname, proname); - - ReleaseSysCache(proctup); - - return result; -} - -/* - * generate_operator_name - * Compute the name to display for an operator specified by OID, - * given that it is being called with the specified actual arg types. - * (Arg types matter because of ambiguous-operator resolution rules. - * Pass InvalidOid for unused arg of a unary operator.) - * - * The result includes all necessary quoting and schema-prefixing, - * plus the OPERATOR() decoration needed to use a qualified operator name - * in an expression. - */ -char * -generate_operator_name(Oid operid, Oid arg1, Oid arg2) -{ - StringInfoData buf; - HeapTuple opertup; - Form_pg_operator operform; - char *oprname; - char *nspname; - - initStringInfo(&buf); - - opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); - if (!HeapTupleIsValid(opertup)) - elog(ERROR, "cache lookup failed for operator %u", operid); - operform = (Form_pg_operator) GETSTRUCT(opertup); - oprname = NameStr(operform->oprname); - - /* - * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, - * we don't check if the operator is in current namespace or not. This is - * because this check is costly when the operator is not in current namespace. - */ - nspname = get_namespace_name(operform->oprnamespace); - Assert(nspname != NULL); - appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); - appendStringInfoString(&buf, oprname); - appendStringInfoChar(&buf, ')'); - - ReleaseSysCache(opertup); - - return buf.data; -} - -/* - * get_one_range_partition_bound_string - * A C string representation of one range partition bound - */ -char * -get_range_partbound_string(List *bound_datums) -{ - deparse_context context; - StringInfo buf = makeStringInfo(); - ListCell *cell; - char *sep; - - memset(&context, 0, sizeof(deparse_context)); - context.buf = buf; - - appendStringInfoChar(buf, '('); - sep = ""; - foreach(cell, bound_datums) - { - PartitionRangeDatum *datum = - lfirst_node(PartitionRangeDatum, cell); - - appendStringInfoString(buf, sep); - if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) - appendStringInfoString(buf, "MINVALUE"); - else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) - appendStringInfoString(buf, "MAXVALUE"); - else - { - Const *val = castNode(Const, datum->value); - - get_const_expr(val, &context, -1); - } - sep = ", "; - } - appendStringInfoChar(buf, ')'); - - return buf->data; -} - -/* - * Collect a list of OIDs of all sequences owned by the specified relation, - * and column if specified. If deptype is not zero, then only find sequences - * with the specified dependency type. - */ -List * -getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) -{ - List *result = NIL; - Relation depRel; - ScanKeyData key[3]; - SysScanDesc scan; - HeapTuple tup; - - depRel = table_open(DependRelationId, AccessShareLock); - - ScanKeyInit(&key[0], - Anum_pg_depend_refclassid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(RelationRelationId)); - ScanKeyInit(&key[1], - Anum_pg_depend_refobjid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(relid)); - if (attnum) - ScanKeyInit(&key[2], - Anum_pg_depend_refobjsubid, - BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(attnum)); - - scan = systable_beginscan(depRel, DependReferenceIndexId, true, - NULL, attnum ? 3 : 2, key); - - while (HeapTupleIsValid(tup = systable_getnext(scan))) - { - Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); - - /* - * We assume any auto or internal dependency of a sequence on a column - * must be what we are looking for. (We need the relkind test because - * indexes can also have auto dependencies on columns.) - */ - if (deprec->classid == RelationRelationId && - deprec->objsubid == 0 && - deprec->refobjsubid != 0 && - (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && - get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) - { - if (!deptype || deprec->deptype == deptype) - result = lappend_oid(result, deprec->objid); - } - } - - systable_endscan(scan); - - table_close(depRel, AccessShareLock); - - return result; -} - -#endif /* (PG_VERSION_NUM >= PG_VERSION_14) && (PG_VERSION_NUM < PG_VERSION_15) */ diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c index 9606cd724..5b554b7c2 100644 --- a/src/backend/distributed/executor/adaptive_executor.c +++ b/src/backend/distributed/executor/adaptive_executor.c @@ -720,10 +720,8 @@ static void RebuildWaitEventSetForSessions(DistributedExecution *execution); static void AddLatchWaitEventToExecution(DistributedExecution *execution); static void ProcessWaitEvents(DistributedExecution *execution, WaitEvent *events, int eventCount, bool *cancellationReceived); -#if PG_VERSION_NUM >= PG_VERSION_15 static void RemoteSocketClosedForAnySession(DistributedExecution *execution); static void ProcessWaitEventsForSocketClosed(WaitEvent *events, int eventCount); -#endif static long MillisecondsBetweenTimestamps(instr_time startTime, instr_time endTime); static uint64 MicrosecondsBetweenTimestamps(instr_time startTime, instr_time endTime); static int WorkerPoolCompare(const void *lhsKey, const void *rhsKey); @@ -1769,11 +1767,8 @@ FindOrCreateWorkerSession(WorkerPool *workerPool, MultiConnection *connection) session->commandsSent = 0; session->waitEventSetIndex = WAIT_EVENT_SET_INDEX_NOT_INITIALIZED; -#if PG_VERSION_NUM >= PG_VERSION_15 - /* always detect closed sockets */ UpdateConnectionWaitFlags(session, WL_SOCKET_CLOSED); -#endif dlist_init(&session->pendingTaskQueue); dlist_init(&session->readyTaskQueue); @@ -1817,7 +1812,6 @@ FindOrCreateWorkerSession(WorkerPool *workerPool, MultiConnection *connection) * the events, even ignores cancellation events. Future callers of this * function should consider its limitations. */ -#if PG_VERSION_NUM >= PG_VERSION_15 static void RemoteSocketClosedForAnySession(DistributedExecution *execution) { @@ -1835,9 +1829,6 @@ RemoteSocketClosedForAnySession(DistributedExecution *execution) } -#endif - - /* * SequentialRunDistributedExecution gets a distributed execution and * executes each individual task in the execution sequentially, one @@ -2173,8 +2164,6 @@ ProcessWaitEvents(DistributedExecution *execution, WaitEvent *events, int eventC } -#if PG_VERSION_NUM >= PG_VERSION_15 - /* * ProcessWaitEventsForSocketClosed mainly checks for WL_SOCKET_CLOSED event. * If WL_SOCKET_CLOSED is found, the function sets the underlying connection's @@ -2207,9 +2196,6 @@ ProcessWaitEventsForSocketClosed(WaitEvent *events, int eventCount) } -#endif - - /* * ManageWorkerPool ensures the worker pool has the appropriate number of connections * based on the number of pending tasks. @@ -2704,7 +2690,6 @@ OpenNewConnections(WorkerPool *workerPool, int newConnectionCount, * Instead, we prefer this slight difference, which in effect has almost no * difference, but doing things in different points in time. */ -#if PG_VERSION_NUM >= PG_VERSION_15 /* we added new connections, rebuild the waitEventSet */ RebuildWaitEventSetForSessions(execution); @@ -2724,9 +2709,6 @@ OpenNewConnections(WorkerPool *workerPool, int newConnectionCount, * of the execution. */ AddLatchWaitEventToExecution(execution); -#else - execution->rebuildWaitEventSet = true; -#endif WorkerSession *session = NULL; foreach_declared_ptr(session, newSessionsList) @@ -3663,13 +3645,8 @@ UpdateConnectionWaitFlags(WorkerSession *session, int waitFlags) return; } -#if PG_VERSION_NUM >= PG_VERSION_15 - /* always detect closed sockets */ connection->waitFlags = waitFlags | WL_SOCKET_CLOSED; -#else - connection->waitFlags = waitFlags; -#endif /* without signalling the execution, the flag changes won't be reflected */ execution->waitFlagsChanged = true; @@ -3694,13 +3671,11 @@ CheckConnectionReady(WorkerSession *session) return false; } -#if PG_VERSION_NUM >= PG_VERSION_15 if ((session->latestUnconsumedWaitEvents & WL_SOCKET_CLOSED) != 0) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } -#endif /* try to send all pending data */ int sendStatus = PQflush(connection->pgConn); diff --git a/src/backend/distributed/executor/query_stats.c b/src/backend/distributed/executor/query_stats.c index ce6179b96..319041b56 100644 --- a/src/backend/distributed/executor/query_stats.c +++ b/src/backend/distributed/executor/query_stats.c @@ -140,19 +140,6 @@ static void CitusQueryStatsRemoveExpiredEntries(HTAB *existingQueryIdHash); void InitializeCitusQueryStats(void) { -/* on PG 15, we use shmem_request_hook_type */ -#if PG_VERSION_NUM < PG_VERSION_15 - - /* allocate shared memory */ - if (!IsUnderPostmaster) - { - RequestAddinShmemSpace(CitusQueryStatsSharedMemSize()); - - elog(LOG, "requesting named LWLockTranch for %s", STATS_SHARED_MEM_NAME); - RequestNamedLWLockTranche(STATS_SHARED_MEM_NAME, 1); - } -#endif - /* Install hook */ prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = CitusQueryStatsShmemStartup; diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index 683a24587..9f4427c33 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -1717,13 +1717,11 @@ ExpandCitusSupportedTypes(ObjectAddressCollector *collector, ObjectAddress targe /* * As of PostgreSQL 15, the same applies to schemas. */ -#if PG_VERSION_NUM >= PG_VERSION_15 List *schemaIdList = GetPublicationSchemas(publicationId); List *schemaDependencyList = CreateObjectAddressDependencyDefList(NamespaceRelationId, schemaIdList); result = list_concat(result, schemaDependencyList); -#endif break; } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 9abd60e5a..d6275bffc 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1739,48 +1739,6 @@ GetSequencesFromAttrDef(Oid attrdefOid) } -#if PG_VERSION_NUM < PG_VERSION_15 - -/* - * Given a pg_attrdef OID, return the relation OID and column number of - * the owning column (represented as an ObjectAddress for convenience). - * - * Returns InvalidObjectAddress if there is no such pg_attrdef entry. - */ -ObjectAddress -GetAttrDefaultColumnAddress(Oid attrdefoid) -{ - ObjectAddress result = InvalidObjectAddress; - ScanKeyData skey[1]; - HeapTuple tup; - - Relation attrdef = table_open(AttrDefaultRelationId, AccessShareLock); - ScanKeyInit(&skey[0], - Anum_pg_attrdef_oid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(attrdefoid)); - SysScanDesc scan = systable_beginscan(attrdef, AttrDefaultOidIndexId, true, - NULL, 1, skey); - - if (HeapTupleIsValid(tup = systable_getnext(scan))) - { - Form_pg_attrdef atdform = (Form_pg_attrdef) GETSTRUCT(tup); - - result.classId = RelationRelationId; - result.objectId = atdform->adrelid; - result.objectSubId = atdform->adnum; - } - - systable_endscan(scan); - table_close(attrdef, AccessShareLock); - - return result; -} - - -#endif - - /* * GetAttrDefsFromSequence returns a list of attrdef OIDs that have * a dependency on the given sequence @@ -3011,7 +2969,6 @@ SyncNodeMetadataToNodesMain(Datum main_arg) PopActiveSnapshot(); CommitTransactionCommand(); - ProcessCompletedNotifies(); if (syncedAllNodes) { diff --git a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c index abe378cdb..bd9b84e81 100644 --- a/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c +++ b/src/backend/distributed/metadata/pg_get_object_address_13_14_15.c @@ -283,9 +283,7 @@ PgGetObjectAddress(char *ttype, ArrayType *namearr, ArrayType *argsarr) case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: case OBJECT_LANGUAGE: -#if PG_VERSION_NUM >= PG_VERSION_15 case OBJECT_PARAMETER_ACL: -#endif case OBJECT_PUBLICATION: case OBJECT_ROLE: case OBJECT_SCHEMA: @@ -323,9 +321,7 @@ PgGetObjectAddress(char *ttype, ArrayType *namearr, ArrayType *argsarr) break; } -#if PG_VERSION_NUM >= PG_VERSION_15 case OBJECT_PUBLICATION_NAMESPACE: -#endif case OBJECT_USER_MAPPING: { objnode = (Node *) list_make2(linitial(name), linitial(args)); diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index f81ade91c..c8ab2a4b3 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -136,11 +136,8 @@ CreateCitusCustomScanPath(PlannerInfo *root, RelOptInfo *relOptInfo, path->custom_path.path.pathtarget = relOptInfo->reltarget; path->custom_path.path.parent = relOptInfo; -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* necessary to avoid extra Result node in PG15 */ path->custom_path.flags = CUSTOMPATH_SUPPORT_PROJECTION; -#endif /* * The 100k rows we put on the cost of the path is kind of arbitrary and could be diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 0ec3883da..a388c767c 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -1442,13 +1442,8 @@ FinalizePlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan) customScan->custom_private = list_make1(distributedPlanData); -#if (PG_VERSION_NUM >= PG_VERSION_15) - /* necessary to avoid extra Result node in PG15 */ customScan->flags = CUSTOMPATH_SUPPORT_BACKWARD_SCAN | CUSTOMPATH_SUPPORT_PROJECTION; -#else - customScan->flags = CUSTOMPATH_SUPPORT_BACKWARD_SCAN; -#endif /* * Fast path queries cannot have any subplans by definition, so skip diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 9c0ba3cd3..e1d917ca0 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -38,8 +38,6 @@ #include "distributed/shard_pruning.h" #include "distributed/shared_library_init.h" -#if PG_VERSION_NUM >= PG_VERSION_15 - static int SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, CitusTableCacheEntry *targetRelation); @@ -100,8 +98,6 @@ static char * MergeCommandResultIdPrefix(uint64 planId); static void ErrorIfMergeHasReturningList(Query *query); static Node * GetMergeJoinCondition(Query *mergeQuery); -#endif - /* * CreateMergePlan @@ -118,13 +114,6 @@ CreateMergePlan(uint64 planId, Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext, ParamListInfo boundParams) { - /* function is void for pre-15 versions of Postgres */ - #if PG_VERSION_NUM < PG_VERSION_15 - - ereport(ERROR, (errmsg("MERGE is not supported in pre-15 Postgres versions"))); - - #else - Oid targetRelationId = ModifyQueryResultRelationId(originalQuery); /* @@ -153,8 +142,6 @@ CreateMergePlan(uint64 planId, Query *originalQuery, Query *query, } return distributedPlan; - - #endif } @@ -184,9 +171,6 @@ GetMergeJoinTree(Query *mergeQuery) } -#if PG_VERSION_NUM >= PG_VERSION_15 - - /* * GetMergeJoinCondition returns the quals of the ON condition */ @@ -1443,9 +1427,6 @@ SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, } -#endif - - /* * ExtractMergeSourceRangeTableEntry returns the range table entry of source * table or source query in USING clause. @@ -1453,13 +1434,6 @@ SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, RangeTblEntry * ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk) { - /* function is void for pre-15 versions of Postgres */ - #if PG_VERSION_NUM < PG_VERSION_15 - - ereport(ERROR, (errmsg("MERGE is not supported in pre-15 Postgres versions"))); - - #else - Assert(IsMergeQuery(query)); List *fromList = query->jointree->fromlist; @@ -1498,8 +1472,6 @@ ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk) RangeTblEntry *subqueryRte = rt_fetch(reference->rtindex, query->rtable); return subqueryRte; - - #endif } @@ -1516,13 +1488,6 @@ ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk) Var * FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) { - /* function is void for pre-15 versions of Postgres */ - #if PG_VERSION_NUM < PG_VERSION_15 - - ereport(ERROR, (errmsg("MERGE is not supported in pre-15 Postgres versions"))); - - #else - Assert(IsMergeQuery(query)); if (!IsCitusTableType(targetRelationId, DISTRIBUTED_TABLE)) @@ -1593,8 +1558,6 @@ FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) } return NULL; - - #endif } diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 527fd073c..58d9068f5 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -1289,8 +1289,8 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) } /* resolve OIDs of unknown (user-defined) types */ - Query *analyzedQuery = parse_analyze_varparams_compat(parseTree, queryString, - ¶mTypes, &numParams, NULL); + Query *analyzedQuery = parse_analyze_varparams(parseTree, queryString, + ¶mTypes, &numParams, NULL); /* pg_rewrite_query is a wrapper around QueryRewrite with some debugging logic */ List *queryList = pg_rewrite_query(analyzedQuery); diff --git a/src/backend/distributed/shardsplit/shardsplit_decoder.c b/src/backend/distributed/shardsplit/shardsplit_decoder.c index 20dd01b0c..837009530 100644 --- a/src/backend/distributed/shardsplit/shardsplit_decoder.c +++ b/src/backend/distributed/shardsplit/shardsplit_decoder.c @@ -122,11 +122,7 @@ update_replication_progress(LogicalDecodingContext *ctx, bool skipped_xact) */ if (ctx->end_xact || ++changes_count >= CHANGES_THRESHOLD) { -#if (PG_VERSION_NUM >= PG_VERSION_15) OutputPluginUpdateProgress(ctx, skipped_xact); -#else - OutputPluginUpdateProgress(ctx); -#endif changes_count = 0; } } diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index d7fa094ed..33faaa6c8 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -174,15 +174,11 @@ static bool FinishedStartupCitusBackend = false; static object_access_hook_type PrevObjectAccessHook = NULL; -#if PG_VERSION_NUM >= PG_VERSION_15 static shmem_request_hook_type prev_shmem_request_hook = NULL; -#endif void _PG_init(void); -#if PG_VERSION_NUM >= PG_VERSION_15 static void citus_shmem_request(void); -#endif static void CitusObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg); static void DoInitialCleanup(void); @@ -475,10 +471,8 @@ _PG_init(void) original_client_auth_hook = ClientAuthentication_hook; ClientAuthentication_hook = CitusAuthHook; -#if PG_VERSION_NUM >= PG_VERSION_15 prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = citus_shmem_request; -#endif InitializeMaintenanceDaemon(); @@ -602,8 +596,6 @@ AdjustDynamicLibraryPathForCdcDecoders(void) } -#if PG_VERSION_NUM >= PG_VERSION_15 - /* * Requests any additional shared memory required for citus. */ @@ -624,9 +616,6 @@ citus_shmem_request(void) } -#endif - - /* * DoInitialCleanup does cleanup at start time. * Currently it: diff --git a/src/backend/distributed/test/fake_am.c b/src/backend/distributed/test/fake_am.c index 928051942..8829d0d8b 100644 --- a/src/backend/distributed/test/fake_am.c +++ b/src/backend/distributed/test/fake_am.c @@ -310,7 +310,7 @@ fake_relation_set_new_filenode(Relation rel, */ *minmulti = GetOldestMultiXactId(); - SMgrRelation srel = RelationCreateStorage_compat(*newrnode, persistence, true); + SMgrRelation srel = RelationCreateStorage(*newrnode, persistence, true); /* * If required, set up an init fork for an unlogged table so that it can diff --git a/src/backend/distributed/test/shared_connection_counters.c b/src/backend/distributed/test/shared_connection_counters.c index c59602887..e5c685e65 100644 --- a/src/backend/distributed/test/shared_connection_counters.c +++ b/src/backend/distributed/test/shared_connection_counters.c @@ -49,13 +49,8 @@ makeIntConst(int val, int location) { A_Const *n = makeNode(A_Const); -#if PG_VERSION_NUM >= PG_VERSION_15 n->val.ival.type = T_Integer; n->val.ival.ival = val; -#else - n->val.type = T_Integer; - n->val.val.ival = val; -#endif n->location = location; return (Node *) n; diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index 866b18fd2..19b03978e 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -519,15 +519,6 @@ UserHasPermissionToViewStatsOf(Oid currentUserId, Oid backendOwnedId) void InitializeBackendManagement(void) { -/* on PG 15, we use shmem_request_hook_type */ -#if PG_VERSION_NUM < PG_VERSION_15 - - /* allocate shared memory */ - if (!IsUnderPostmaster) - { - RequestAddinShmemSpace(BackendManagementShmemSize()); - } -#endif prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = BackendManagementShmemInit; } diff --git a/src/backend/distributed/utils/background_jobs.c b/src/backend/distributed/utils/background_jobs.c index 73a635f21..c9cabc16c 100644 --- a/src/backend/distributed/utils/background_jobs.c +++ b/src/backend/distributed/utils/background_jobs.c @@ -1393,87 +1393,6 @@ CalculateBackoffDelay(int retryCount) } -#if PG_VERSION_NUM < PG_VERSION_15 -static const char * -error_severity(int elevel) -{ - const char *prefix; - - switch (elevel) - { - case DEBUG1: - case DEBUG2: - case DEBUG3: - case DEBUG4: - case DEBUG5: - { - prefix = gettext_noop("DEBUG"); - break; - } - - case LOG: - case LOG_SERVER_ONLY: - { - prefix = gettext_noop("LOG"); - break; - } - - case INFO: - { - prefix = gettext_noop("INFO"); - break; - } - - case NOTICE: - { - prefix = gettext_noop("NOTICE"); - break; - } - - case WARNING: - { - prefix = gettext_noop("WARNING"); - break; - } - - case WARNING_CLIENT_ONLY: - { - prefix = gettext_noop("WARNING"); - break; - } - - case ERROR: - { - prefix = gettext_noop("ERROR"); - break; - } - - case FATAL: - { - prefix = gettext_noop("FATAL"); - break; - } - - case PANIC: - { - prefix = gettext_noop("PANIC"); - break; - } - - default: - { - prefix = "???"; - break; - } - } - - return prefix; -} - - -#endif - - /* * bgw_generate_returned_message - * generates the message to be inserted into the job_run_details table diff --git a/src/backend/distributed/utils/citus_stat_tenants.c b/src/backend/distributed/utils/citus_stat_tenants.c index 6af5c0d58..1ca4fc6f1 100644 --- a/src/backend/distributed/utils/citus_stat_tenants.c +++ b/src/backend/distributed/utils/citus_stat_tenants.c @@ -15,6 +15,7 @@ #include "unistd.h" #include "access/hash.h" +#include "common/pg_prng.h" #include "executor/execdesc.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -38,10 +39,6 @@ #include "distributed/tuplestore.h" #include "distributed/utils/citus_stat_tenants.h" -#if (PG_VERSION_NUM >= PG_VERSION_15) - #include "common/pg_prng.h" -#endif - static void AttributeMetricsIfApplicable(void); ExecutorEnd_hook_type prev_ExecutorEnd = NULL; @@ -298,13 +295,7 @@ AttributeTask(char *tenantId, int colocationId, CmdType commandType) /* If the tenant is not found in the hash table, we will track the query with a probability of StatTenantsSampleRateForNewTenants. */ if (!found) { -#if (PG_VERSION_NUM >= PG_VERSION_15) double randomValue = pg_prng_double(&pg_global_prng_state); -#else - - /* Generate a random double between 0 and 1 */ - double randomValue = (double) random() / MAX_RANDOM_VALUE; -#endif bool shouldTrackQuery = randomValue <= StatTenantsSampleRateForNewTenants; if (!shouldTrackQuery) { diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index d9b29cdb0..d6908aced 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -14,14 +14,6 @@ #include "pg_version_constants.h" -#if PG_VERSION_NUM >= PG_VERSION_15 -#define ExecARDeleteTriggers_compat(a, b, c, d, e, f) \ - ExecARDeleteTriggers(a, b, c, d, e, f) -#else -#define ExecARDeleteTriggers_compat(a, b, c, d, e, f) \ - ExecARDeleteTriggers(a, b, c, d, e) -#endif - #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE #define ExplainPropertyLong(qlabel, value, es) \ diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index caf40ff95..72f495aa2 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -525,13 +525,11 @@ extern List * PostprocessAlterSequenceSchemaStmt(Node *node, const char *querySt extern List * PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); -#if (PG_VERSION_NUM >= PG_VERSION_15) extern List * PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -#endif extern List * PreprocessDropSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool @@ -547,10 +545,8 @@ extern List * AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, isPostprocess); extern List * AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -#if (PG_VERSION_NUM >= PG_VERSION_15) extern List * AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -#endif extern List * RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt); @@ -754,8 +750,6 @@ extern List * CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern void CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaName, uint64 shardId); -extern List * PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); extern void AlterTriggerRenameEventExtendNames(RenameStmt *renameTriggerStmt, char *schemaName, uint64 shardId); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 8c4c58c88..3431a29c5 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -267,9 +267,7 @@ extern char * DeparseDropSequenceStmt(Node *node); extern char * DeparseRenameSequenceStmt(Node *node); extern char * DeparseAlterSequenceSchemaStmt(Node *node); extern char * DeparseAlterSequenceOwnerStmt(Node *node); -#if (PG_VERSION_NUM >= PG_VERSION_15) extern char * DeparseAlterSequencePersistenceStmt(Node *node); -#endif extern char * DeparseGrantOnSequenceStmt(Node *node); /* forward declarations for qualify_sequence_stmt.c */ @@ -277,9 +275,7 @@ extern void QualifyRenameSequenceStmt(Node *node); extern void QualifyDropSequenceStmt(Node *node); extern void QualifyAlterSequenceSchemaStmt(Node *node); extern void QualifyAlterSequenceOwnerStmt(Node *node); -#if (PG_VERSION_NUM >= PG_VERSION_15) extern void QualifyAlterSequencePersistenceStmt(Node *node); -#endif extern void QualifyGrantOnSequenceStmt(Node *node); #endif /* CITUS_DEPARSER_H */ diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index d7234e4bc..ed65db07c 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -28,11 +28,6 @@ #define CURSOR_OPT_FORCE_DISTRIBUTED 0x080000 -/* Hack to compile Citus on pre-MERGE Postgres versions */ -#if PG_VERSION_NUM < PG_VERSION_15 -#define CMD_MERGE CMD_UNKNOWN -#endif - /* level of planner calls */ extern int PlannerLevel; diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index 56b34bc21..a2e86f2ba 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -128,9 +128,6 @@ extern List * IdentitySequenceDependencyCommandList(Oid targetRelationId); extern List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName); extern List * GetSequencesFromAttrDef(Oid attrdefOid); -#if PG_VERSION_NUM < PG_VERSION_15 -ObjectAddress GetAttrDefaultColumnAddress(Oid attrdefoid); -#endif extern List * GetAttrDefsFromSequence(Oid seqOid); extern void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, AttrNumber attnum, char depType); diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 0db4f9a26..43a381135 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -286,76 +286,6 @@ typedef RangeTblEntry RTEPermissionInfo; #endif -#if PG_VERSION_NUM >= PG_VERSION_15 -#define ProcessCompletedNotifies() -#define RelationCreateStorage_compat(a, b, c) RelationCreateStorage(a, b, c) -#define parse_analyze_varparams_compat(a, b, c, d, e) parse_analyze_varparams(a, b, c, d, \ - e) -#define CREATE_SEQUENCE_COMMAND \ - "CREATE %sSEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \ - " MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \ - " START WITH " INT64_FORMAT " CACHE " INT64_FORMAT " %sCYCLE" -#else - -#include "nodes/value.h" -#include "storage/smgr.h" -#include "utils/int8.h" -#include "utils/rel.h" - -typedef Value String; - -#ifdef HAVE_LONG_INT_64 -#define strtoi64(str, endptr, base) ((int64) strtol(str, endptr, base)) -#define strtou64(str, endptr, base) ((uint64) strtoul(str, endptr, base)) -#else -#define strtoi64(str, endptr, base) ((int64) strtoll(str, endptr, base)) -#define strtou64(str, endptr, base) ((uint64) strtoull(str, endptr, base)) -#endif -#define RelationCreateStorage_compat(a, b, c) RelationCreateStorage(a, b) -#define parse_analyze_varparams_compat(a, b, c, d, e) parse_analyze_varparams(a, b, c, d) -#define pgstat_init_relation(r) pgstat_initstats(r) -#define pg_analyze_and_rewrite_fixedparams(a, b, c, d, e) pg_analyze_and_rewrite(a, b, c, \ - d, e) -#define boolVal(v) intVal(v) -#define makeBoolean(val) makeInteger(val) - -static inline int64 -pg_strtoint64(char *s) -{ - int64 result; - (void) scanint8(s, false, &result); - return result; -} - - -/* - * RelationGetSmgr got backported in 13.10 and 14.7 so redefining it for any - * version higher causes compilation errors due to redefining of the function. - * We want to use it in all versions. So we backport it ourselves in earlier - * versions, and rely on the Postgres provided version in the later versions. - */ -#if PG_VERSION_NUM < 140007 -static inline SMgrRelation -RelationGetSmgr(Relation rel) -{ - if (unlikely(rel->rd_smgr == NULL)) - { - smgrsetowner(&(rel->rd_smgr), smgropen(rel->rd_node, rel->rd_backend)); - } - return rel->rd_smgr; -} - - -#endif - - -#define CREATE_SEQUENCE_COMMAND \ - "CREATE SEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \ - " MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \ - " START WITH " INT64_FORMAT " CACHE " INT64_FORMAT " %sCYCLE" - -#endif - #define SetListCellPtr(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define fcGetArgValue(fc, n) ((fc)->args[n].value) @@ -365,4 +295,9 @@ RelationGetSmgr(Relation rel) #define fcSetArg(fc, n, value) fcSetArgExt(fc, n, value, false) #define fcSetArgNull(fc, n) fcSetArgExt(fc, n, (Datum) 0, true) +#define CREATE_SEQUENCE_COMMAND \ + "CREATE %sSEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \ + " MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \ + " START WITH " INT64_FORMAT " CACHE " INT64_FORMAT " %sCYCLE" + #endif /* PG_VERSION_COMPAT_H */ diff --git a/src/include/pg_version_constants.h b/src/include/pg_version_constants.h index ba2a9a03e..c8bfd319e 100644 --- a/src/include/pg_version_constants.h +++ b/src/include/pg_version_constants.h @@ -11,7 +11,6 @@ #ifndef PG_VERSION_CONSTANTS #define PG_VERSION_CONSTANTS -#define PG_VERSION_14 140000 #define PG_VERSION_15 150000 #define PG_VERSION_16 160000 #define PG_VERSION_17 170000 diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 59bf6d97c..3d9fcb8c2 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -108,19 +108,13 @@ s/(ERROR: |WARNING: |error:) invalid socket/\1 connection not open/g /^\s*invalid socket$/d # pg15 changes -# can be removed when dropping PG13&14 support -#if (PG_VERSION_NUM >= PG_VERSION_14) && (PG_VERSION_NUM < PG_VERSION_15) -# (This is not preprocessor directive, but a reminder for the developer that will drop PG14 support ) -s/is not a PostgreSQL server process/is not a PostgreSQL backend process/g s/ AS "\?column\?"//g -s/".*\.(.*)": (found .* removable)/"\1": \2/g # We ignore multiline error messages, and substitute first line with a single line # alternative that is used in some older libpq versions. s/(ERROR: |WARNING: |error:) server closed the connection unexpectedly/\1 connection not open/g /^\s*This probably means the server terminated abnormally$/d /^\s*before or while processing the request.$/d /^\s*connection not open$/d -#endif /* (PG_VERSION_NUM >= PG_VERSION_13) && (PG_VERSION_NUM < PG_VERSION_14) */ # intermediate_results s/(ERROR.*)pgsql_job_cache\/([0-9]+_[0-9]+_[0-9]+)\/(.*).data/\1pgsql_job_cache\/xx_x_xxx\/\3.data/g diff --git a/src/test/regress/expected/citus_local_tables_queries.out b/src/test/regress/expected/citus_local_tables_queries.out index 2b2761644..148e08a4b 100644 --- a/src/test/regress/expected/citus_local_tables_queries.out +++ b/src/test/regress/expected/citus_local_tables_queries.out @@ -1,17 +1,6 @@ -- -- CITUS_LOCAL_TABLES_QUERIES -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - \set VERBOSITY terse SET citus.next_shard_id TO 1509000; SET citus.shard_replication_factor TO 1; diff --git a/src/test/regress/expected/citus_local_tables_queries_0.out b/src/test/regress/expected/citus_local_tables_queries_0.out deleted file mode 100644 index 4da695c89..000000000 --- a/src/test/regress/expected/citus_local_tables_queries_0.out +++ /dev/null @@ -1,1168 +0,0 @@ --- --- CITUS_LOCAL_TABLES_QUERIES --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -\set VERBOSITY terse -SET citus.next_shard_id TO 1509000; -SET citus.shard_replication_factor TO 1; -SET citus.enable_local_execution TO ON; -SET citus.log_local_commands TO ON; -CREATE SCHEMA citus_local_table_queries; -SET search_path TO citus_local_table_queries; --- ensure that coordinator is added to pg_dist_node -SET client_min_messages to ERROR; -SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -RESET client_min_messages; -CREATE TABLE dummy_reference_table(a int unique, b int); -SELECT create_reference_table('dummy_reference_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE citus_local_table(a int, b int); -ALTER TABLE citus_local_table ADD CONSTRAINT fkey_to_dummy_1 FOREIGN KEY (a) REFERENCES dummy_reference_table(a); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1509001, 'citus_local_table_queries', 1509000, 'citus_local_table_queries', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_to_dummy_1 FOREIGN KEY (a) REFERENCES dummy_reference_table(a);') -CREATE TABLE citus_local_table_2(a int, b int); -ALTER TABLE citus_local_table_2 ADD CONSTRAINT fkey_to_dummy_2 FOREIGN KEY (a) REFERENCES dummy_reference_table(a); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1509002, 'citus_local_table_queries', 1509000, 'citus_local_table_queries', 'ALTER TABLE citus_local_table_2 ADD CONSTRAINT fkey_to_dummy_2 FOREIGN KEY (a) REFERENCES dummy_reference_table(a);') -CREATE TABLE reference_table(a int, b int); -SELECT create_reference_table('reference_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE distributed_table(a int, b int); -SELECT create_distributed_table('distributed_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE postgres_local_table(a int, b int); --- Define a helper function to truncate & insert some data into our test tables --- We should call this function at some places in this test file to prevent --- test to take a long time. --- We shouldn't use LIMIT in INSERT SELECT queries to make the test faster as --- LIMIT would force planner to wrap SELECT query in an intermediate result and --- this might reduce the coverage of the test cases. -CREATE FUNCTION clear_and_init_test_tables() RETURNS void AS $$ - BEGIN - SET client_min_messages to ERROR; - - TRUNCATE postgres_local_table, citus_local_table, reference_table, distributed_table, dummy_reference_table, citus_local_table_2; - - INSERT INTO dummy_reference_table SELECT i, i FROM generate_series(0, 5) i; - INSERT INTO citus_local_table SELECT i, i FROM generate_series(0, 5) i; - INSERT INTO citus_local_table_2 SELECT i, i FROM generate_series(0, 5) i; - INSERT INTO postgres_local_table SELECT i, i FROM generate_series(0, 5) i; - INSERT INTO distributed_table SELECT i, i FROM generate_series(0, 5) i; - INSERT INTO reference_table SELECT i, i FROM generate_series(0, 5) i; - - RESET client_min_messages; - END; -$$ LANGUAGE plpgsql; ---------------------------------------------------------------------- ----- SELECT ---- ---------------------------------------------------------------------- -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - --- join between citus local tables and reference tables would succeed -SELECT count(*) FROM citus_local_table, reference_table WHERE citus_local_table.a = reference_table.a; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.reference_table_1509003 reference_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) - count ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT * FROM citus_local_table, reference_table WHERE citus_local_table.a = reference_table.a ORDER BY 1,2,3,4 FOR UPDATE; -NOTICE: executing the command locally: SELECT citus_local_table.a, citus_local_table.b, reference_table.a, reference_table.b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.reference_table_1509003 reference_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) ORDER BY citus_local_table.a, citus_local_table.b, reference_table.a, reference_table.b FOR UPDATE OF citus_local_table FOR UPDATE OF reference_table - a | b | a | b ---------------------------------------------------------------------- - 0 | 0 | 0 | 0 - 1 | 1 | 1 | 1 - 2 | 2 | 2 | 2 - 3 | 3 | 3 | 3 - 4 | 4 | 4 | 4 - 5 | 5 | 5 | 5 -(6 rows) - --- should work -WITH cte_1 AS - (SELECT * FROM citus_local_table, reference_table WHERE citus_local_table.a = reference_table.a ORDER BY 1,2,3,4 FOR UPDATE) -SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: WITH cte_1 AS (SELECT citus_local_table.a, citus_local_table.b, reference_table.a, reference_table.b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.reference_table_1509003 reference_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) ORDER BY citus_local_table.a, citus_local_table.b, reference_table.a, reference_table.b FOR UPDATE OF citus_local_table FOR UPDATE OF reference_table) SELECT count(*) AS count FROM cte_1 cte_1(a, b, a_1, b_1) - count ---------------------------------------------------------------------- - 6 -(1 row) - --- should work as joins are between ctes -WITH cte_citus_local_table AS - (SELECT * FROM citus_local_table), -cte_postgres_local_table AS - (SELECT * FROM postgres_local_table), -cte_distributed_table AS - (SELECT * FROM distributed_table) -SELECT count(*) FROM cte_distributed_table, cte_citus_local_table, cte_postgres_local_table -WHERE cte_citus_local_table.a = 1 AND cte_distributed_table.a = 1; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - --- should fail as we don't support direct joins between distributed/local tables -SELECT count(*) FROM distributed_table d1, distributed_table d2, citus_local_table; -ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns --- local table inside subquery should just work -SELECT count(*) FROM -( - SELECT * FROM (SELECT * FROM citus_local_table) as subquery_inner -) as subquery_top; -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT subquery_inner.a, subquery_inner.b FROM (SELECT citus_local_table.a, citus_local_table.b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table) subquery_inner) subquery_top - count ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - --- join between citus/postgres local tables should just work -SELECT count(*) FROM -( - SELECT * FROM (SELECT count(*) FROM citus_local_table, postgres_local_table) as subquery_inner -) as subquery_top; -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT subquery_inner.count FROM (SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.postgres_local_table) subquery_inner) subquery_top - count ---------------------------------------------------------------------- - 1 -(1 row) - --- should fail as we don't support direct joins between distributed/local tables -SELECT count(*) FROM -( - SELECT *, random() FROM (SELECT *, random() FROM citus_local_table, distributed_table) as subquery_inner -) as subquery_top; -NOTICE: executing the command locally: SELECT NULL::integer AS "dummy-1" FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true - count ---------------------------------------------------------------------- - 36 -(1 row) - --- should fail as we don't support direct joins between distributed/local tables -SELECT count(*) FROM -( - SELECT *, random() - FROM ( - WITH cte_1 AS (SELECT *, random() FROM citus_local_table, distributed_table) SELECT * FROM cte_1) as subquery_inner -) as subquery_top; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT subquery_inner.a, subquery_inner.b, subquery_inner.a_1 AS a, subquery_inner.b_1 AS b, subquery_inner.random, random() AS random FROM (SELECT cte_1.a, cte_1.b, cte_1.a_1 AS a, cte_1.b_1 AS b, cte_1.random FROM (SELECT intermediate_result.a, intermediate_result.b, intermediate_result.a_1 AS a, intermediate_result.b_1 AS b, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer, a_1 integer, b_1 integer, random double precision)) cte_1(a, b, a_1, b_1, random)) subquery_inner(a, b, a_1, b_1, random)) subquery_top(a, b, a_1, b_1, random, random_1) - count ---------------------------------------------------------------------- - 36 -(1 row) - --- should be fine -SELECT count(*) FROM -( - SELECT *, random() - FROM ( - WITH cte_1 AS (SELECT *, random() FROM citus_local_table), cte_2 AS (SELECT * FROM distributed_table) SELECT count(*) FROM cte_1, cte_2 - ) as subquery_inner -) as subquery_top; -NOTICE: executing the command locally: SELECT a, b, random() AS random FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT subquery_inner.count, random() AS random FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) subquery_inner) subquery_top - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - --- prepared statement -PREPARE citus_local_only AS SELECT count(*) FROM citus_local_table; --- execute 6 times, local tables without params -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -EXECUTE citus_local_only; -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - --- execute 6 times, with param -PREPARE citus_local_only_p(int) AS SELECT count(*) FROM citus_local_table WHERE a = $1; -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - --- do not evalute the function --- show the logs -EXECUTE citus_local_only_p(random()); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE citus_local_only_p(random()); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) $1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -PREPARE mixed_query(int, int, int) AS - WITH cte_citus_local_table AS - (SELECT * FROM citus_local_table WHERE a = $1), - cte_postgres_local_table AS - (SELECT * FROM postgres_local_table WHERE a = $2), - cte_distributed_table AS - (SELECT * FROM distributed_table WHERE a = $3), - cte_mixes AS (SELECT * FROM cte_distributed_table, cte_citus_local_table, cte_postgres_local_table) - SELECT count(*) FROM cte_mixes; -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE mixed_query(1,2,3); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - --- anonymous columns -WITH a AS (SELECT a, '' FROM citus_local_table GROUP BY a) SELECT a.a FROM a ORDER BY 1 LIMIT 5; -NOTICE: executing the command locally: SELECT a FROM (SELECT citus_local_table.a, ''::text FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table GROUP BY citus_local_table.a) a(a, "?column?") ORDER BY a LIMIT 5 - a ---------------------------------------------------------------------- - 0 - 1 - 2 - 3 - 4 -(5 rows) - -WITH a AS (SELECT b, '' FROM citus_local_table WHERE a = 1) SELECT * FROM a, a b ORDER BY 1 LIMIT 5; -NOTICE: executing the command locally: WITH a AS (SELECT citus_local_table.b, ''::text FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) 1)) SELECT a.b, a."?column?", b.b, b."?column?" FROM a a(b, "?column?"), a b(b, "?column?") ORDER BY a.b LIMIT 5 - b | ?column? | b | ?column? ---------------------------------------------------------------------- - 1 | | 1 | -(1 row) - --- weird expression on citus/pg table joins should be fine -SELECT * FROM citus_local_table, postgres_local_table -WHERE citus_local_table.a - postgres_local_table.a = 0 -ORDER BY 1,2,3,4 -LIMIT 10; -NOTICE: executing the command locally: SELECT citus_local_table.a, citus_local_table.b, postgres_local_table.a, postgres_local_table.b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.postgres_local_table WHERE ((citus_local_table.a OPERATOR(pg_catalog.-) postgres_local_table.a) OPERATOR(pg_catalog.=) 0) ORDER BY citus_local_table.a, citus_local_table.b, postgres_local_table.a, postgres_local_table.b LIMIT 10 - a | b | a | b ---------------------------------------------------------------------- - 0 | 0 | 0 | 0 - 1 | 1 | 1 | 1 - 2 | 2 | 2 | 2 - 3 | 3 | 3 | 3 - 4 | 4 | 4 | 4 - 5 | 5 | 5 | 5 -(6 rows) - --- set operations should just work -SELECT * FROM citus_local_table UNION SELECT * FROM postgres_local_table UNION SELECT * FROM distributed_table ORDER BY 1,2; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -NOTICE: executing the command locally: SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) UNION SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) UNION SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ORDER BY 1, 2 - a | b ---------------------------------------------------------------------- - 0 | 0 - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(6 rows) - -(SELECT * FROM citus_local_table ORDER BY 1,2 LIMIT 5) INTERSECT (SELECT i, i FROM generate_series(0, 100) i) ORDER BY 1, 2; -NOTICE: executing the command locally: (SELECT citus_local_table.a, citus_local_table.b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table ORDER BY citus_local_table.a, citus_local_table.b LIMIT 5) INTERSECT SELECT i.i, i.i FROM generate_series(0, 100) i(i) ORDER BY 1, 2 - a | b ---------------------------------------------------------------------- - 0 | 0 - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 -(5 rows) - --- should just work as recursive planner kicks in -SELECT count(*) FROM distributed_table WHERE a IN (SELECT a FROM citus_local_table); -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - count ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT count(*) FROM citus_local_table WHERE a IN (SELECT a FROM distributed_table); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer))) - count ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT count(*) FROM reference_table WHERE a IN (SELECT a FROM citus_local_table); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.reference_table_1509003 reference_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT citus_local_table.a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table)) - count ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT count(*) FROM citus_local_table WHERE a IN (SELECT a FROM reference_table); -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT reference_table.a FROM citus_local_table_queries.reference_table_1509003 reference_table)) - count ---------------------------------------------------------------------- - 6 -(1 row) - --- nested recursive queries should just work -SELECT count(*) FROM citus_local_table - WHERE a IN - (SELECT a FROM distributed_table WHERE a IN - (SELECT b FROM citus_local_table WHERE b IN (SELECT b FROM postgres_local_table))); -NOTICE: executing the command locally: SELECT b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (b OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(b integer))) -NOTICE: executing the command locally: SELECT count(*) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.a FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(a integer))) - count ---------------------------------------------------------------------- - 6 -(1 row) - --- local outer joins -SELECT count(*) FROM citus_local_table LEFT JOIN reference_table ON (true); -NOTICE: executing the command locally: SELECT count(*) AS count FROM (citus_local_table_queries.citus_local_table_1509001 citus_local_table LEFT JOIN citus_local_table_queries.reference_table_1509003 reference_table ON (true)) - count ---------------------------------------------------------------------- - 36 -(1 row) - -SELECT count(*) FROM reference_table - LEFT JOIN citus_local_table ON (true) - LEFT JOIN postgres_local_table ON (true) - LEFT JOIN reference_table r2 ON (true); -NOTICE: executing the command locally: SELECT count(*) AS count FROM (((citus_local_table_queries.reference_table_1509003 reference_table LEFT JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) LEFT JOIN citus_local_table_queries.postgres_local_table ON (true)) LEFT JOIN citus_local_table_queries.reference_table_1509003 r2 ON (true)) - count ---------------------------------------------------------------------- - 1296 -(1 row) - -SELECT count(*) FROM citus_local_table LEFT JOIN distributed_table ON (true); -NOTICE: executing the command locally: SELECT NULL::integer AS "dummy-1" FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM ((SELECT NULL::integer AS a, NULL::integer AS b FROM (SELECT intermediate_result."dummy-1" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("dummy-1" integer)) citus_local_table_1) citus_local_table LEFT JOIN (SELECT NULL::integer AS a, NULL::integer AS b FROM (SELECT intermediate_result."dummy-1" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("dummy-1" integer)) distributed_table_1) distributed_table ON (true)) - count ---------------------------------------------------------------------- - 36 -(1 row) - --- distinct in subquery on CTE -WITH one_row AS ( - SELECT a from citus_local_table WHERE b = 1 -) -SELECT - * -FROM - distributed_table -WHERE - b IN (SELECT DISTINCT a FROM one_row) -ORDER BY - 1, 2 -LIMIT - 1; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (b OPERATOR(pg_catalog.=) 1) - a | b ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH one_row_2 AS ( - SELECT a from distributed_table WHERE b = 1 -) -SELECT - * -FROM - citus_local_table -WHERE - b IN (SELECT DISTINCT a FROM one_row_2) -ORDER BY - 1 ,2 -LIMIT - 1; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (b OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer))) ORDER BY a, b LIMIT 1 - a | b ---------------------------------------------------------------------- - 1 | 1 -(1 row) - --- join between citus local tables and distributed tables would fail -SELECT count(*) FROM citus_local_table, distributed_table; -NOTICE: executing the command locally: SELECT NULL::integer AS "dummy-1" FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true - count ---------------------------------------------------------------------- - 36 -(1 row) - -SELECT * FROM citus_local_table, distributed_table ORDER BY 1,2,3,4 FOR UPDATE; -ERROR: could not run distributed query with FOR UPDATE/SHARE commands --- join between citus local tables and postgres local tables are okey -SELECT count(citus_local_table.b), count(postgres_local_table.a) -FROM citus_local_table, postgres_local_table -WHERE citus_local_table.a = postgres_local_table.b; -NOTICE: executing the command locally: SELECT count(citus_local_table.b) AS count, count(postgres_local_table.a) AS count FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table, citus_local_table_queries.postgres_local_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) postgres_local_table.b) - count | count ---------------------------------------------------------------------- - 6 | 6 -(1 row) - --- select for update is just OK -SELECT * FROM citus_local_table ORDER BY 1,2 FOR UPDATE; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table ORDER BY a, b FOR UPDATE OF citus_local_table - a | b ---------------------------------------------------------------------- - 0 | 0 - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(6 rows) - ---------------------------------------------------------------------- ------ INSERT SELECT ----- ---------------------------------------------------------------------- --- simple INSERT SELECT is OK -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO citus_local_table -SELECT * from reference_table; -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a, b) SELECT a, b FROM citus_local_table_queries.reference_table_1509003 reference_table -INSERT INTO reference_table -SELECT * from citus_local_table; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO citus_local_table -SELECT * from distributed_table; -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO distributed_table -SELECT * from citus_local_table; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -INSERT INTO citus_local_table -SELECT * from citus_local_table_2; -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a, b) SELECT a, b FROM citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2 -INSERT INTO citus_local_table -SELECT sum(a), b from citus_local_table_2 -GROUP BY b; -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a, b) SELECT sum(a) AS sum, b FROM citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2 GROUP BY b -INSERT INTO citus_local_table -SELECT * from postgres_local_table; -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO postgres_local_table -SELECT * from citus_local_table; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table --- INSERT SELECT with local joins are OK -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO citus_local_table -SELECT reference_table.* FROM reference_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a, b) SELECT reference_table.a, reference_table.b FROM (citus_local_table_queries.reference_table_1509003 reference_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) -INSERT INTO reference_table -SELECT reference_table.* FROM reference_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT reference_table.a, reference_table.b FROM (citus_local_table_queries.reference_table_1509003 reference_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO reference_table -SELECT reference_table.* FROM reference_table, postgres_local_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT reference_table.a, reference_table.b FROM citus_local_table_queries.reference_table_1509003 reference_table, (citus_local_table_queries.postgres_local_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) -NOTICE: executing the copy locally for shard xxxxx -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO distributed_table -SELECT reference_table.* FROM reference_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT reference_table.a, reference_table.b FROM (citus_local_table_queries.reference_table_1509003 reference_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) -INSERT INTO distributed_table -SELECT reference_table.* FROM reference_table, postgres_local_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT reference_table.a, reference_table.b FROM citus_local_table_queries.reference_table_1509003 reference_table, (citus_local_table_queries.postgres_local_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) -INSERT INTO postgres_local_table -SELECT reference_table.* FROM reference_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT reference_table.a, reference_table.b FROM (citus_local_table_queries.reference_table_1509003 reference_table JOIN citus_local_table_queries.citus_local_table_1509001 citus_local_table ON (true)) --- INSERT SELECT that joins reference and distributed tables is also OK -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO citus_local_table -SELECT reference_table.* FROM reference_table -JOIN distributed_table ON (true); -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO citus_local_table -SELECT reference_table.* -FROM reference_table, distributed_table; -NOTICE: executing the copy locally for shard xxxxx --- INSERT SELECT that joins citus local and distributed table directly will fail .. -INSERT INTO citus_local_table -SELECT distributed_table.* FROM distributed_table -JOIN citus_local_table ON (true); -NOTICE: executing the command locally: SELECT NULL::integer AS "dummy-1" FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the copy locally for shard xxxxx --- .. but when wrapped into a CTE, join works fine -INSERT INTO citus_local_table -SELECT distributed_table.* FROM distributed_table -JOIN (WITH cte AS (SELECT * FROM citus_local_table) SELECT * FROM cte) as foo ON (true); -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -NOTICE: executing the copy locally for shard xxxxx --- multi row insert is OK -INSERT INTO citus_local_table VALUES (1, 2), (3, 4); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a, b) VALUES (1,2), (3,4) ---------------------------------------------------------------------- ------ DELETE / UPDATE ----- ---------------------------------------------------------------------- --- modifications using citus local tables and postgres local tables --- are not supported, see below four tests -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -DELETE FROM citus_local_table -USING postgres_local_table -WHERE citus_local_table.b = postgres_local_table.b; -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table USING citus_local_table_queries.postgres_local_table WHERE (citus_local_table.b OPERATOR(pg_catalog.=) postgres_local_table.b) -UPDATE citus_local_table -SET b = 5 -FROM postgres_local_table -WHERE citus_local_table.a = 3 AND citus_local_table.b = postgres_local_table.b; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 citus_local_table SET b = 5 FROM citus_local_table_queries.postgres_local_table WHERE ((citus_local_table.a OPERATOR(pg_catalog.=) 3) AND (citus_local_table.b OPERATOR(pg_catalog.=) postgres_local_table.b)) -DELETE FROM postgres_local_table -USING citus_local_table -WHERE citus_local_table.b = postgres_local_table.b; -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.postgres_local_table USING citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (citus_local_table.b OPERATOR(pg_catalog.=) postgres_local_table.b) -UPDATE postgres_local_table -SET b = 5 -FROM citus_local_table -WHERE citus_local_table.a = 3 AND citus_local_table.b = postgres_local_table.b; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.postgres_local_table SET b = 5 FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE ((citus_local_table.a OPERATOR(pg_catalog.=) 3) AND (citus_local_table.b OPERATOR(pg_catalog.=) postgres_local_table.b)) --- no direct joins supported -UPDATE distributed_table -SET b = 6 -FROM citus_local_table -WHERE citus_local_table.a = distributed_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -UPDATE reference_table -SET b = 6 -FROM citus_local_table -WHERE citus_local_table.a = reference_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the command locally: UPDATE citus_local_table_queries.reference_table_1509003 reference_table SET b = 6 FROM (SELECT citus_local_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) citus_local_table_1) citus_local_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) --- should not work, add HINT use CTEs -UPDATE citus_local_table -SET b = 6 -FROM distributed_table -WHERE citus_local_table.a = distributed_table.a; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 citus_local_table SET b = 6 FROM (SELECT distributed_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) distributed_table_1) distributed_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) distributed_table.a) --- should work, add HINT use CTEs -UPDATE citus_local_table -SET b = 6 -FROM reference_table -WHERE citus_local_table.a = reference_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.reference_table_1509003 reference_table WHERE true -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 citus_local_table SET b = 6 FROM (SELECT reference_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) reference_table_1) reference_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) --- should not work, add HINT use CTEs -DELETE FROM distributed_table -USING citus_local_table -WHERE citus_local_table.a = distributed_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true --- should not work, add HINT use CTEs -DELETE FROM citus_local_table -USING distributed_table -WHERE citus_local_table.a = distributed_table.a; -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table USING (SELECT distributed_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) distributed_table_1) distributed_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) distributed_table.a) -DELETE FROM reference_table -USING citus_local_table -WHERE citus_local_table.a = reference_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.reference_table_1509003 reference_table USING (SELECT citus_local_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) citus_local_table_1) citus_local_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) --- should work, add HINT use CTEs -DELETE FROM citus_local_table -USING reference_table -WHERE citus_local_table.a = reference_table.a; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.reference_table_1509003 reference_table WHERE true -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table USING (SELECT reference_table_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) reference_table_1) reference_table WHERE (citus_local_table.a OPERATOR(pg_catalog.=) reference_table.a) --- just works -DELETE FROM citus_local_table -WHERE citus_local_table.a IN (SELECT a FROM distributed_table); -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer))) --- just works -DELETE FROM citus_local_table -WHERE citus_local_table.a IN (SELECT a FROM reference_table); -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.reference_table_1509003 reference_table -NOTICE: executing the command locally: DELETE FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE (a OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer))) --- just works -WITH distributed_table_cte AS (SELECT * FROM distributed_table) -UPDATE citus_local_table -SET b = 6 -FROM distributed_table_cte -WHERE citus_local_table.a = distributed_table_cte.a; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 citus_local_table SET b = 6 FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) distributed_table_cte WHERE (citus_local_table.a OPERATOR(pg_catalog.=) distributed_table_cte.a) -SET citus.log_local_commands to off; --- just works -WITH reference_table_cte AS (SELECT * FROM reference_table) -UPDATE citus_local_table -SET b = 6 -FROM reference_table_cte -WHERE citus_local_table.a = reference_table_cte.a; -set citus.log_local_commands to on; ---------------------------------------------------------------------- ------ VIEW QUERIES ----- ---------------------------------------------------------------------- -CREATE MATERIALIZED VIEW mat_view_4 AS -SELECT count(*) -FROM citus_local_table -JOIN reference_table -USING (a); -NOTICE: executing the command locally: SELECT count(*) AS count FROM (citus_local_table_queries.citus_local_table_1509001 citus_local_table(a, b) JOIN citus_local_table_queries.reference_table_1509003 reference_table(a, b) USING (a)) --- ok -SELECT count(*) FROM mat_view_4; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- should work -SELECT count(*) FROM distributed_table WHERE b in -(SELECT count FROM mat_view_4); - count ---------------------------------------------------------------------- - 1 -(1 row) - -CREATE VIEW view_2 AS -SELECT count(*) -FROM citus_local_table -JOIN citus_local_table_2 USING (a) -JOIN distributed_table USING (a); --- should fail as view contains direct local dist join -SELECT count(*) FROM view_2; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table WHERE true -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2 WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) view_2 - count ---------------------------------------------------------------------- - 1 -(1 row) - -CREATE VIEW view_3 -AS SELECT count(*) -FROM citus_local_table_2 -JOIN reference_table -USING (a); --- ok -SELECT count(*) FROM view_3; -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT count(*) AS count FROM (citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2(a, b) JOIN citus_local_table_queries.reference_table_1509003 reference_table(a, b) USING (a))) view_3 - count ---------------------------------------------------------------------- - 1 -(1 row) - --- view treated as subquery, so should work -SELECT count(*) FROM view_3, distributed_table; -NOTICE: executing the command locally: SELECT a FROM citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2 WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM ((SELECT citus_local_table_2_1.a, NULL::integer AS b FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) citus_local_table_2_1) citus_local_table_2 JOIN citus_local_table_queries.reference_table_1509003 reference_table(a, b) USING (a)) - count ---------------------------------------------------------------------- - 6 -(1 row) - ---------------------------------------------------------------------- --- Some other tests with subqueries & CTE's -- ---------------------------------------------------------------------- -SELECT clear_and_init_test_tables(); - clear_and_init_test_tables ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) AS a, count(*) AS b -FROM reference_table -JOIN (SELECT count(*) as a, count(*) as b - FROM citus_local_table_2 - JOIN (SELECT count(*) as a, count(*) as b - FROM postgres_local_table - JOIN (SELECT count(*) as a, count(*) as b - FROM reference_table as table_4677) subquery5108 - USING (a)) subquery7132 - USING (b)) subquery7294 -USING (a); -NOTICE: executing the command locally: SELECT count(*) AS a, count(*) AS b FROM (citus_local_table_queries.reference_table_1509003 reference_table(a, b) JOIN (SELECT count(*) AS a, count(*) AS b FROM (citus_local_table_queries.citus_local_table_2_1509002 citus_local_table_2(a, b) JOIN (SELECT count(*) AS a, count(*) AS b FROM (citus_local_table_queries.postgres_local_table JOIN (SELECT count(*) AS a, count(*) AS b FROM citus_local_table_queries.reference_table_1509003 table_4677) subquery5108 USING (a))) subquery7132 USING (b))) subquery7294 USING (a)) - a | b ---------------------------------------------------------------------- - 1 | 1 -(1 row) - --- direct join inside CTE not supported -WITH cte AS ( -UPDATE citus_local_table lt SET a = mt.a -FROM distributed_table mt WHERE mt.b = lt.b -RETURNING lt.b, lt.a -) SELECT * FROM cte JOIN distributed_table mt ON mt.b = cte.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 lt SET a = mt.a FROM (SELECT mt_1.a, mt_1.b FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) mt_1) mt WHERE (mt.b OPERATOR(pg_catalog.=) lt.b) RETURNING lt.b, lt.a - b | a | a | b ---------------------------------------------------------------------- - 0 | 0 | 0 | 0 - 1 | 1 | 1 | 1 - 2 | 2 | 2 | 2 - 3 | 3 | 3 | 3 - 4 | 4 | 4 | 4 - 5 | 5 | 5 | 5 -(6 rows) - --- join with CTE just works -UPDATE citus_local_table -SET a=5 -FROM (SELECT avg(distributed_table.b) as avg_b - FROM distributed_table) as foo -WHERE -foo.avg_b = citus_local_table.b; -NOTICE: executing the command locally: UPDATE citus_local_table_queries.citus_local_table_1509001 citus_local_table SET a = 5 FROM (SELECT intermediate_result.avg_b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(avg_b numeric)) foo WHERE (foo.avg_b OPERATOR(pg_catalog.=) (citus_local_table.b)::numeric) --- should work -UPDATE distributed_table -SET b = avg_a -FROM (SELECT avg(citus_local_table.a) as avg_a FROM citus_local_table) as foo -WHERE foo.avg_a = distributed_table.a -RETURNING distributed_table.*; -NOTICE: executing the command locally: SELECT avg(a) AS avg_a FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table - a | b ---------------------------------------------------------------------- -(0 rows) - --- it is unfortunate that recursive planner cannot detect this --- but expected to not work -UPDATE citus_local_table -SET a=5 -FROM (SELECT b FROM distributed_table) AS foo -WHERE foo.b = citus_local_table.b; -ERROR: local table citus_local_table cannot be joined with these distributed tables ---------------------------------------------------------------------- --- test different execution paths -- ---------------------------------------------------------------------- --- a bit different explain output than for postgres local tables -EXPLAIN (COSTS FALSE) -INSERT INTO citus_local_table -SELECT * FROM distributed_table -ORDER BY distributed_table.* -LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Limit - -> Sort - Sort Key: remote_scan.worker_column_3 - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Limit - -> Sort - Sort Key: distributed_table.* - -> Seq Scan on distributed_table_1509004 distributed_table -(14 rows) - --- show that we do not pull to coordinator -EXPLAIN (COSTS FALSE) -INSERT INTO citus_local_table -SELECT * FROM citus_local_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Insert on citus_local_table_1509001 citus_table_alias - -> Seq Scan on citus_local_table_1509001 citus_local_table -(7 rows) - -EXPLAIN (COSTS FALSE) -INSERT INTO citus_local_table -SELECT reference_table.* FROM reference_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Insert on citus_local_table_1509001 citus_table_alias - -> Seq Scan on reference_table_1509003 reference_table -(7 rows) - -EXPLAIN (COSTS FALSE) -INSERT INTO citus_local_table -SELECT reference_table.* FROM reference_table, postgres_local_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Nested Loop - -> Seq Scan on reference_table_1509003 reference_table - -> Materialize - -> Seq Scan on postgres_local_table -(11 rows) - --- show that we pull to coordinator when a distributed table is involved -EXPLAIN (COSTS FALSE) -INSERT INTO citus_local_table -SELECT reference_table.* FROM reference_table, distributed_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Nested Loop - -> Seq Scan on distributed_table_1509004 distributed_table - -> Materialize - -> Seq Scan on reference_table_1509003 reference_table -(11 rows) - --- truncate tables & add unique constraints to be able to define foreign keys -TRUNCATE reference_table, citus_local_table, distributed_table; -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.reference_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.citus_local_table_xxxxx CASCADE -ALTER TABLE reference_table ADD CONSTRAINT pkey_ref PRIMARY KEY (a); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1509003, 'citus_local_table_queries', 'ALTER TABLE reference_table ADD CONSTRAINT pkey_ref PRIMARY KEY (a);') -ALTER TABLE citus_local_table ADD CONSTRAINT pkey_c PRIMARY KEY (a); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1509001, 'citus_local_table_queries', 'ALTER TABLE citus_local_table ADD CONSTRAINT pkey_c PRIMARY KEY (a);') --- define a foreign key chain distributed table -> reference table -> citus local table --- to test sequential execution -ALTER TABLE distributed_table ADD CONSTRAINT fkey_dist_to_ref FOREIGN KEY(a) REFERENCES reference_table(a) ON DELETE RESTRICT; -ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(a) REFERENCES citus_local_table(a) ON DELETE RESTRICT; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1509003, 'citus_local_table_queries', 1509001, 'citus_local_table_queries', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(a) REFERENCES citus_local_table(a) ON DELETE RESTRICT;') -INSERT INTO citus_local_table VALUES (1); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 (a) VALUES (1) -INSERT INTO reference_table VALUES (1); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.reference_table_1509003 (a) VALUES (1) -BEGIN; - INSERT INTO citus_local_table VALUES (1) ON CONFLICT (a) DO NOTHING; -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (1) ON CONFLICT(a) DO NOTHING - INSERT INTO distributed_table VALUES (1); - -- should show sequential as first inserting into citus local table - -- would force the xact block to use sequential execution - show citus.multi_shard_modify_mode; - citus.multi_shard_modify_mode ---------------------------------------------------------------------- - sequential -(1 row) - -ROLLBACK; -BEGIN; - TRUNCATE distributed_table; - -- should error out as we truncated distributed_table via parallel execution - TRUNCATE citus_local_table CASCADE; -NOTICE: truncate cascades to table "reference_table" -NOTICE: truncate cascades to table "distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.citus_local_table_xxxxx CASCADE -ERROR: cannot execute DDL on table "citus_local_table" because there was a parallel DDL access to distributed table "distributed_table" in the same transaction -ROLLBACK; -BEGIN; - SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; - TRUNCATE distributed_table; - -- should work fine as we already switched to sequential execution - -- before parallel truncate - TRUNCATE citus_local_table CASCADE; -NOTICE: truncate cascades to table "reference_table" -NOTICE: truncate cascades to table "distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.citus_local_table_xxxxx CASCADE -NOTICE: truncate cascades to table "reference_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.reference_table_xxxxx CASCADE -ROLLBACK; -ALTER TABLE distributed_table DROP CONSTRAINT fkey_dist_to_ref; -BEGIN; - INSERT INTO citus_local_table VALUES (1) ON CONFLICT (a) DO NOTHING; -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (1) ON CONFLICT(a) DO NOTHING - show citus.multi_shard_modify_mode; - citus.multi_shard_modify_mode ---------------------------------------------------------------------- - sequential -(1 row) - -ROLLBACK; --- remove uniqueness constraint and dependent foreign key constraint for next tests -ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1509003, 'citus_local_table_queries', 1509001, 'citus_local_table_queries', 'ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;') -ALTER TABLE citus_local_table DROP CONSTRAINT pkey_c; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1509001, 'citus_local_table_queries', 'ALTER TABLE citus_local_table DROP CONSTRAINT pkey_c;') -COPY citus_local_table(a) FROM PROGRAM 'seq 1'; --- should use local execution -BEGIN; - COPY citus_local_table(a) FROM PROGRAM 'seq 1'; -NOTICE: executing the copy locally for shard xxxxx - COPY citus_local_table(a) FROM PROGRAM 'seq 1'; -NOTICE: executing the copy locally for shard xxxxx -COMMIT; -COPY citus_local_table TO STDOUT; -1 \N -1 \N -1 \N -1 \N -COPY (SELECT * FROM citus_local_table) TO STDOUT; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -1 \N -1 \N -1 \N -1 \N -BEGIN; - COPY citus_local_table TO STDOUT; -1 \N -1 \N -1 \N -1 \N -COMMIT; -BEGIN; - COPY (SELECT * FROM citus_local_table) TO STDOUT; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table -1 \N -1 \N -1 \N -1 \N -COMMIT; --- truncate test tables for next test -TRUNCATE citus_local_table, reference_table, distributed_table; -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.citus_local_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_queries.reference_table_xxxxx CASCADE -BEGIN; - INSERT INTO citus_local_table VALUES (1), (2); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (1), (2) - SAVEPOINT sp1; - INSERT INTO citus_local_table VALUES (3), (4); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (3), (4) - ROLLBACK TO SAVEPOINT sp1; - SELECT * FROM citus_local_table ORDER BY 1,2; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table ORDER BY a, b - a | b ---------------------------------------------------------------------- - 1 | - 2 | -(2 rows) - - SAVEPOINT sp2; - INSERT INTO citus_local_table VALUES (3), (4); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (3), (4) - INSERT INTO distributed_table VALUES (3), (4); - ROLLBACK TO SAVEPOINT sp2; - SELECT * FROM citus_local_table ORDER BY 1,2; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table ORDER BY a, b - a | b ---------------------------------------------------------------------- - 1 | - 2 | -(2 rows) - - SELECT * FROM distributed_table ORDER BY 1,2; - a | b ---------------------------------------------------------------------- -(0 rows) - - SAVEPOINT sp3; - INSERT INTO citus_local_table VALUES (3), (2); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.citus_local_table_1509001 AS citus_table_alias (a) VALUES (3), (2) - INSERT INTO reference_table VALUES (3), (2); -NOTICE: executing the command locally: INSERT INTO citus_local_table_queries.reference_table_1509003 AS citus_table_alias (a) VALUES (3), (2) - ROLLBACK TO SAVEPOINT sp3; - SELECT * FROM citus_local_table ORDER BY 1,2; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.citus_local_table_1509001 citus_local_table ORDER BY a, b - a | b ---------------------------------------------------------------------- - 1 | - 2 | -(2 rows) - - SELECT * FROM reference_table ORDER BY 1,2; -NOTICE: executing the command locally: SELECT a, b FROM citus_local_table_queries.reference_table_1509003 reference_table ORDER BY a, b - a | b ---------------------------------------------------------------------- -(0 rows) - -COMMIT; --- cleanup at exit -DROP SCHEMA citus_local_table_queries CASCADE; -NOTICE: drop cascades to 14 other objects diff --git a/src/test/regress/expected/columnar_pg15.out b/src/test/regress/expected/columnar_pg15.out index 2ad95fcaf..62d2de2dc 100644 --- a/src/test/regress/expected/columnar_pg15.out +++ b/src/test/regress/expected/columnar_pg15.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE TABLE alter_am(i int); INSERT INTO alter_am SELECT generate_series(1,1000000); SELECT * FROM columnar.options WHERE relation = 'alter_am'::regclass; diff --git a/src/test/regress/expected/columnar_pg15_0.out b/src/test/regress/expected/columnar_pg15_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/columnar_pg15_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/columnar_vacuum_vs_insert.out b/src/test/regress/expected/columnar_vacuum_vs_insert.out index f229c4c71..0d88e8a04 100644 --- a/src/test/regress/expected/columnar_vacuum_vs_insert.out +++ b/src/test/regress/expected/columnar_vacuum_vs_insert.out @@ -55,7 +55,7 @@ step s1-commit: COMMIT; s2: INFO: vacuuming "public.test_vacuum_vs_insert" -s2: INFO: "test_vacuum_vs_insert": found 0 removable, 6 nonremovable row versions in 4 pages +s2: INFO: "public.test_vacuum_vs_insert": found 0 removable, 6 nonremovable row versions in 4 pages DETAIL: 0 dead row versions cannot be removed yet. step s2-vacuum-full: <... completed> step s2-select: diff --git a/src/test/regress/expected/coordinator_shouldhaveshards.out b/src/test/regress/expected/coordinator_shouldhaveshards.out index 047827dd8..6f24614ba 100644 --- a/src/test/regress/expected/coordinator_shouldhaveshards.out +++ b/src/test/regress/expected/coordinator_shouldhaveshards.out @@ -3,17 +3,6 @@ -- -- Test queries on a distributed table with shards on the coordinator -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA coordinator_shouldhaveshards; SET search_path TO coordinator_shouldhaveshards; SET citus.next_shard_id TO 1503000; diff --git a/src/test/regress/expected/coordinator_shouldhaveshards_0.out b/src/test/regress/expected/coordinator_shouldhaveshards_0.out deleted file mode 100644 index 00ccedb15..000000000 --- a/src/test/regress/expected/coordinator_shouldhaveshards_0.out +++ /dev/null @@ -1,1190 +0,0 @@ --- --- COORDINATOR_SHOULDHAVESHARDS --- --- Test queries on a distributed table with shards on the coordinator --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA coordinator_shouldhaveshards; -SET search_path TO coordinator_shouldhaveshards; -SET citus.next_shard_id TO 1503000; -SET citus.next_placement_id TO 1503000; --- idempotently add node to allow this test to run without add_coordinator -SET client_min_messages TO WARNING; -SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -RESET client_min_messages; -SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SET citus.shard_replication_factor TO 1; -CREATE TABLE test (x int, y int); -SELECT create_distributed_table('test','x', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) FROM pg_dist_shard JOIN pg_dist_placement USING (shardid) -WHERE logicalrelid = 'test'::regclass AND groupid = 0; - count ---------------------------------------------------------------------- - 2 -(1 row) - ---- enable logging to see which tasks are executed locally -SET client_min_messages TO LOG; -SET citus.log_local_commands TO ON; --- INSERT..SELECT with COPY under the covers -INSERT INTO test SELECT s,s FROM generate_series(2,100) s; -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx --- router queries execute locally -INSERT INTO test VALUES (1, 1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - --- multi-shard queries connect to localhost -SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 100 -(1 row) - -WITH a AS (SELECT * FROM test) SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 100 -(1 row) - --- multi-shard queries in transaction blocks execute locally -BEGIN; -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -END; -BEGIN; -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -END; --- INSERT..SELECT with re-partitioning after local execution -BEGIN; -INSERT INTO test VALUES (0,1000); -CREATE TABLE repart_test (x int primary key, y int); -SELECT create_distributed_table('repart_test','x', colocate_with := 'none'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503004, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.repart_test (x integer NOT NULL, y integer) USING heap');SELECT worker_apply_shard_ddl_command (1503004, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.repart_test OWNER TO postgres');SELECT worker_apply_shard_ddl_command (1503004, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.repart_test ADD CONSTRAINT repart_test_pkey PRIMARY KEY (x)') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503007, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.repart_test (x integer NOT NULL, y integer) USING heap');SELECT worker_apply_shard_ddl_command (1503007, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.repart_test OWNER TO postgres');SELECT worker_apply_shard_ddl_command (1503007, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.repart_test ADD CONSTRAINT repart_test_pkey PRIMARY KEY (x)') - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO repart_test (x, y) SELECT y, x FROM test; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503000_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503000_to','SELECT y AS x, x AS y FROM coordinator_shouldhaveshards.test_1503000 test WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503003_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503003_to','SELECT y AS x, x AS y FROM coordinator_shouldhaveshards.test_1503003 test WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.repart_test_1503004 AS citus_table_alias (x, y) SELECT x, y FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1503000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(x integer, y integer) -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.repart_test_1503007 AS citus_table_alias (x, y) SELECT x, y FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1503003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(x integer, y integer) -SELECT y FROM repart_test WHERE x = 1000; - y ---------------------------------------------------------------------- - 0 -(1 row) - -INSERT INTO repart_test (x, y) SELECT y, x FROM test ON CONFLICT (x) DO UPDATE SET y = -1; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503000_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503000_to','SELECT y AS x, x AS y FROM coordinator_shouldhaveshards.test_1503000 test WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503003_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503003_to','SELECT y AS x, x AS y FROM coordinator_shouldhaveshards.test_1503003 test WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.repart_test_1503004 AS citus_table_alias (x, y) SELECT x, y FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1503000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(x integer, y integer) ON CONFLICT(x) DO UPDATE SET y = '-1'::integer -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.repart_test_1503007 AS citus_table_alias (x, y) SELECT x, y FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1503003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(x integer, y integer) ON CONFLICT(x) DO UPDATE SET y = '-1'::integer -SELECT y FROM repart_test WHERE x = 1000; - y ---------------------------------------------------------------------- - -1 -(1 row) - -ROLLBACK; --- INSERT..SELECT with re-partitioning in EXPLAIN ANALYZE after local execution -BEGIN; -INSERT INTO test VALUES (0,1000); -EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) INSERT INTO test (x, y) SELECT y, x FROM test; -ERROR: EXPLAIN ANALYZE is currently not supported for INSERT ... SELECT commands with repartitioning -ROLLBACK; --- DDL connects to locahost -ALTER TABLE test ADD COLUMN z int; --- DDL after local execution -BEGIN; -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - -ALTER TABLE test DROP COLUMN z; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503000, 'coordinator_shouldhaveshards', 'ALTER TABLE test DROP COLUMN z;') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503003, 'coordinator_shouldhaveshards', 'ALTER TABLE test DROP COLUMN z;') -ROLLBACK; -BEGIN; -ALTER TABLE test DROP COLUMN z; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503000, 'coordinator_shouldhaveshards', 'ALTER TABLE test DROP COLUMN z;') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503003, 'coordinator_shouldhaveshards', 'ALTER TABLE test DROP COLUMN z;') -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - -END; -SET citus.shard_count TO 6; -SET citus.log_remote_commands TO OFF; -BEGIN; -SET citus.log_local_commands TO ON; -CREATE TABLE dist_table (a int); -INSERT INTO dist_table SELECT * FROM generate_series(1, 100); --- trigger local execution -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - --- this should be run locally -SELECT create_distributed_table('dist_table', 'a', colocate_with := 'none'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503008, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503008, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503011, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503011, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table OWNER TO postgres') -NOTICE: executing the copy locally for shard xxxxx -NOTICE: Copying data from local table... -NOTICE: executing the copy locally for shard xxxxx -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$coordinator_shouldhaveshards.dist_table$$) - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) FROM dist_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.dist_table_1503008 dist_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.dist_table_1503011 dist_table WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -ROLLBACK; -CREATE TABLE dist_table (a int); -INSERT INTO dist_table SELECT * FROM generate_series(1, 100); -BEGIN; -SET citus.log_local_commands TO ON; --- trigger local execution -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - --- this should be run locally -SELECT create_distributed_table('dist_table', 'a', colocate_with := 'none'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503014, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503014, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503017, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503017, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table OWNER TO postgres') -NOTICE: executing the copy locally for shard xxxxx -NOTICE: Copying data from local table... -NOTICE: executing the copy locally for shard xxxxx -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$coordinator_shouldhaveshards.dist_table$$) - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) FROM dist_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.dist_table_1503014 dist_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.dist_table_1503017 dist_table WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -ROLLBACK; --- repartition queries should work fine -SET citus.enable_repartition_joins TO ON; -SELECT count(*) FROM test t1, test t2 WHERE t1.x = t2.y; - count ---------------------------------------------------------------------- - 100 -(1 row) - -BEGIN; -SET citus.enable_unique_job_ids TO off; -SELECT count(*) FROM test t1, test t2 WHERE t1.x = t2.y; -NOTICE: executing the command locally: SELECT partition_index, 'repartition_26_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_26_1','SELECT x AS column1 FROM coordinator_shouldhaveshards.test_1503000 t1 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_26_4' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_26_4','SELECT x AS column1 FROM coordinator_shouldhaveshards.test_1503003 t1 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_27_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_27_1','SELECT y AS column1 FROM coordinator_shouldhaveshards.test_1503000 t2 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_27_4' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_27_4','SELECT y AS column1 FROM coordinator_shouldhaveshards.test_1503003 t2 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_1_2']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_2_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_3_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_4_2']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_1_2']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_2_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_3_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_4_2']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_1_5']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_2_5']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_3_5']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_26_4_5']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_1_5']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_2_5']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_3_5']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_27_4_5']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_26_1_2,repartition_26_2_2,repartition_26_3_2,repartition_26_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 integer) JOIN read_intermediate_results('{repartition_27_1_2,repartition_27_2_2,repartition_27_3_2,repartition_27_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 integer) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_26_1_5,repartition_26_2_5,repartition_26_3_5,repartition_26_4_5}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 integer) JOIN read_intermediate_results('{repartition_27_1_5,repartition_27_2_5,repartition_27_3_5,repartition_27_4_5}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 integer) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -END; -BEGIN; -SET citus.enable_repartition_joins TO ON; --- trigger local execution -SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM test t1, test t2 WHERE t1.x = t2.y; -NOTICE: executing the command locally: SELECT partition_index, 'repartition_30_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_30_1','SELECT x AS column1 FROM coordinator_shouldhaveshards.test_1503000 t1 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_30_4' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_30_4','SELECT x AS column1 FROM coordinator_shouldhaveshards.test_1503003 t1 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_31_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_31_1','SELECT y AS column1 FROM coordinator_shouldhaveshards.test_1503000 t2 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_31_4' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_31_4','SELECT y AS column1 FROM coordinator_shouldhaveshards.test_1503003 t2 WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_1_1']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_2_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_3_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_4_1']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_1_1']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_2_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_3_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_4_1']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_1_4']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_2_4']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_3_4']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_30_4_4']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_1_4']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_2_4']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_3_4']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_31_4_4']::text[],'localhost',57636) bytes -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_30_1_1,repartition_30_2_1,repartition_30_3_1,repartition_30_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 integer) JOIN read_intermediate_results('{repartition_31_1_1,repartition_31_2_1,repartition_31_3_1,repartition_31_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 integer) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_30_1_4,repartition_30_2_4,repartition_30_3_4,repartition_30_4_4}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 integer) JOIN read_intermediate_results('{repartition_31_1_4,repartition_31_2_4,repartition_31_3_4,repartition_31_4_4}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 integer) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -ROLLBACK; -CREATE TABLE ref (a int, b int); -SELECT create_reference_table('ref'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE local (x int, y int); -BEGIN; -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -SELECT * FROM ref JOIN local ON (a = x); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) - a | b | x | y ---------------------------------------------------------------------- -(0 rows) - -TRUNCATE ref; -NOTICE: executing the command locally: TRUNCATE TABLE coordinator_shouldhaveshards.ref_xxxxx CASCADE -ROLLBACK; -BEGIN; -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -TRUNCATE ref; -NOTICE: executing the command locally: TRUNCATE TABLE coordinator_shouldhaveshards.ref_xxxxx CASCADE -SELECT * FROM ref JOIN local ON (a = x); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) - a | b | x | y ---------------------------------------------------------------------- -(0 rows) - -ROLLBACK; -BEGIN; -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -INSERT INTO ref VALUES (1,2); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (1, 2) -INSERT INTO local VALUES (1,2); -SELECT * FROM ref JOIN local ON (a = x); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) - a | b | x | y ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 -(1 row) - -ROLLBACK; -BEGIN; -SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - --- we wont see the modifying cte in this query because we will use local execution and --- in postgres we wouldn't see this modifying cte, so it is consistent with postgres. -WITH a AS (SELECT count(*) FROM test), b AS (INSERT INTO local VALUES (3,2) RETURNING *), c AS (INSERT INTO ref VALUES (3,2) RETURNING *), d AS (SELECT count(*) FROM ref JOIN local ON (a = x)) SELECT * FROM a, b, c, d ORDER BY x,y,a,b; -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (3, 2) RETURNING a, b -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, NULL::integer AS y FROM (SELECT intermediate_result.x FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) -NOTICE: executing the command locally: SELECT a.count, b.x, b.y, c.a, c.b, d.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) a, (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) b, (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) c, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) d ORDER BY b.x, b.y, c.a, c.b - count | x | y | a | b | count ---------------------------------------------------------------------- - 100 | 3 | 2 | 3 | 2 | 0 -(1 row) - -TRUNCATE ref; -NOTICE: executing the command locally: TRUNCATE TABLE coordinator_shouldhaveshards.ref_xxxxx CASCADE -SELECT * FROM ref JOIN local ON (a = x); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) - a | b | x | y ---------------------------------------------------------------------- -(0 rows) - --- we wont see the modifying cte in this query because we will use local execution and --- in postgres we wouldn't see this modifying cte, so it is consistent with postgres. -WITH a AS (SELECT count(*) FROM test), b AS (INSERT INTO local VALUES (3,2) RETURNING *), c AS (INSERT INTO ref VALUES (3,2) RETURNING *), d AS (SELECT count(*) FROM ref JOIN local ON (a = x)) SELECT * FROM a, b, c, d ORDER BY x,y,a,b; -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (3, 2) RETURNING a, b -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, NULL::integer AS y FROM (SELECT intermediate_result.x FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) -NOTICE: executing the command locally: SELECT a.count, b.x, b.y, c.a, c.b, d.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) a, (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) b, (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) c, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) d ORDER BY b.x, b.y, c.a, c.b - count | x | y | a | b | count ---------------------------------------------------------------------- - 100 | 3 | 2 | 3 | 2 | 0 -(1 row) - -ROLLBACK; -BEGIN; --- we wont see the modifying cte in this query because we will use local execution and --- in postgres we wouldn't see this modifying cte, so it is consistent with postgres. -WITH a AS (SELECT count(*) FROM test), b AS (INSERT INTO local VALUES (3,2) RETURNING *), c AS (INSERT INTO ref VALUES (3,2) RETURNING *), d AS (SELECT count(*) FROM ref JOIN local ON (a = x)) SELECT * FROM a, b, c, d ORDER BY x,y,a,b; -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (3, 2) RETURNING a, b -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, NULL::integer AS y FROM (SELECT intermediate_result.x FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) -NOTICE: executing the command locally: SELECT a.count, b.x, b.y, c.a, c.b, d.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) a, (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) b, (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) c, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) d ORDER BY b.x, b.y, c.a, c.b - count | x | y | a | b | count ---------------------------------------------------------------------- - 100 | 3 | 2 | 3 | 2 | 0 -(1 row) - -ROLLBACK; -BEGIN; --- we wont see the modifying cte in this query because we will use local execution and --- in postgres we wouldn't see this modifying cte, so it is consistent with postgres. -WITH a AS (SELECT count(*) FROM test), b AS (INSERT INTO local VALUES (3,2) RETURNING *), c AS (INSERT INTO ref SELECT *,* FROM generate_series(1,10) RETURNING *), d AS (SELECT count(*) FROM ref JOIN local ON (a = x)) SELECT * FROM a, b, c, d ORDER BY x,y,a,b; -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_1503020'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, NULL::integer AS y FROM (SELECT intermediate_result.x FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) -NOTICE: executing the command locally: SELECT a.count, b.x, b.y, c.a, c.b, d.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) a, (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) b, (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) c, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) d ORDER BY b.x, b.y, c.a, c.b - count | x | y | a | b | count ---------------------------------------------------------------------- - 100 | 3 | 2 | 1 | 1 | 0 - 100 | 3 | 2 | 2 | 2 | 0 - 100 | 3 | 2 | 3 | 3 | 0 - 100 | 3 | 2 | 4 | 4 | 0 - 100 | 3 | 2 | 5 | 5 | 0 - 100 | 3 | 2 | 6 | 6 | 0 - 100 | 3 | 2 | 7 | 7 | 0 - 100 | 3 | 2 | 8 | 8 | 0 - 100 | 3 | 2 | 9 | 9 | 0 - 100 | 3 | 2 | 10 | 10 | 0 -(10 rows) - -ROLLBACK; --- same local table reference table tests, but outside a transaction block -INSERT INTO ref VALUES (1,2); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (1, 2) -INSERT INTO local VALUES (1,2); -SELECT * FROM ref JOIN local ON (a = x); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) - a | b | x | y ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 -(1 row) - --- we wont see the modifying cte in this query because we will use local execution and --- in postgres we wouldn't see this modifying cte, so it is consistent with postgres. -WITH a AS (SELECT count(*) FROM test), b AS (INSERT INTO local VALUES (3,2) RETURNING *), c AS (INSERT INTO ref VALUES (3,2) RETURNING *), d AS (SELECT count(*) FROM ref JOIN local ON (a = x)) SELECT * FROM a, b, c, d ORDER BY x,y,a,b; -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.ref_1503020 (a, b) VALUES (3, 2) RETURNING a, b -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, NULL::integer AS y FROM (SELECT intermediate_result.x FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) -NOTICE: executing the command locally: SELECT a.count, b.x, b.y, c.a, c.b, d.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) a, (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) b, (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) c, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) d ORDER BY b.x, b.y, c.a, c.b - count | x | y | a | b | count ---------------------------------------------------------------------- - 100 | 3 | 2 | 3 | 2 | 1 -(1 row) - --- joins between local tables and distributed tables are disallowed -CREATE TABLE dist_table(a int); -ERROR: relation "dist_table" already exists -SELECT create_distributed_table('dist_table', 'a'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$coordinator_shouldhaveshards.dist_table$$) - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO dist_table VALUES(1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.dist_table_1503021 (a) VALUES (1) -SELECT * FROM local JOIN dist_table ON (a = x) ORDER BY 1,2,3; - x | y | a ---------------------------------------------------------------------- - 1 | 2 | 1 - 1 | 2 | 1 - 3 | 2 | 3 -(3 rows) - -SELECT * FROM local JOIN dist_table ON (a = x) WHERE a = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT local.x, local.y, dist_table.a FROM ((SELECT local_1.x, local_1.y FROM (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) local_1) local JOIN coordinator_shouldhaveshards.dist_table_1503021 dist_table ON ((dist_table.a OPERATOR(pg_catalog.=) local.x))) WHERE (dist_table.a OPERATOR(pg_catalog.=) 1) ORDER BY local.x, local.y, dist_table.a - x | y | a ---------------------------------------------------------------------- - 1 | 2 | 1 - 1 | 2 | 1 -(2 rows) - --- intermediate results are allowed -WITH cte_1 AS (SELECT * FROM dist_table ORDER BY 1 LIMIT 1) -SELECT * FROM ref JOIN local ON (a = x) JOIN cte_1 ON (local.x = cte_1.a); -NOTICE: executing the command locally: SELECT a FROM coordinator_shouldhaveshards.dist_table_1503021 dist_table WHERE true ORDER BY a LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT a FROM coordinator_shouldhaveshards.dist_table_1503024 dist_table WHERE true ORDER BY a LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y, cte_1.a FROM ((coordinator_shouldhaveshards.ref_1503020 ref JOIN (SELECT local_1.x, local_1.y FROM (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) local_1) local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) JOIN (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer)) cte_1 ON ((local.x OPERATOR(pg_catalog.=) cte_1.a))) - a | b | x | y | a ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 | 1 -(1 row) - --- full router query with CTE and local -WITH cte_1 AS (SELECT * FROM ref LIMIT 1) -SELECT * FROM ref JOIN local ON (a = x) JOIN cte_1 ON (local.x = cte_1.a); -NOTICE: executing the command locally: SELECT ref.a, ref.b, local.x, local.y, cte_1.a, cte_1.b FROM ((coordinator_shouldhaveshards.ref_1503020 ref JOIN coordinator_shouldhaveshards.local ON ((ref.a OPERATOR(pg_catalog.=) local.x))) JOIN (SELECT ref_1.a, ref_1.b FROM coordinator_shouldhaveshards.ref_1503020 ref_1 LIMIT 1) cte_1 ON ((local.x OPERATOR(pg_catalog.=) cte_1.a))) - a | b | x | y | a | b ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 | 1 | 2 -(1 row) - -DROP TABLE dist_table; --- issue #3801 -SET citus.shard_replication_factor TO 2; -CREATE TABLE dist_table(a int); -SELECT create_distributed_table('dist_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -BEGIN; --- this will use perPlacementQueryStrings, make sure it works correctly with --- copying task -INSERT INTO dist_table SELECT a + 1 FROM dist_table; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503027_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503027_to','SELECT (a OPERATOR(pg_catalog.+) 1) AS a FROM coordinator_shouldhaveshards.dist_table_1503027 dist_table WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503029_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503029_to','SELECT (a OPERATOR(pg_catalog.+) 1) AS a FROM coordinator_shouldhaveshards.dist_table_1503029 dist_table WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503030_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503030_to','SELECT (a OPERATOR(pg_catalog.+) 1) AS a FROM coordinator_shouldhaveshards.dist_table_1503030 dist_table WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1503032_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1503032_to','SELECT (a OPERATOR(pg_catalog.+) 1) AS a FROM coordinator_shouldhaveshards.dist_table_1503032 dist_table WHERE true',0,'hash','{-2147483648,-1431655766,-715827884,-2,715827880,1431655762}'::text[],'{-1431655767,-715827885,-3,715827879,1431655761,2147483647}'::text[],true) WHERE rows_written > 0 -ROLLBACK; -SET citus.shard_replication_factor TO 1; -BEGIN; -SET citus.shard_replication_factor TO 2; -CREATE TABLE dist_table1(a int); --- this will use queryStringList, make sure it works correctly with --- copying task -SELECT create_distributed_table('dist_table1', 'a'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503033, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table1 (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503033, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table1 OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503035, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table1 (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503035, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table1 OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503036, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table1 (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503036, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table1 OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1503038, 'coordinator_shouldhaveshards', 'CREATE TABLE coordinator_shouldhaveshards.dist_table1 (a integer) USING heap');SELECT worker_apply_shard_ddl_command (1503038, 'coordinator_shouldhaveshards', 'ALTER TABLE coordinator_shouldhaveshards.dist_table1 OWNER TO postgres') - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -ROLLBACK; -CREATE table ref_table(x int, y int); --- this will be replicated to the coordinator because of add_coordinator test -SELECT create_reference_table('ref_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -TRUNCATE TABLE test; -BEGIN; -INSERT INTO test SELECT *, * FROM generate_series(1, 100); -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO ref_table SELECT *, * FROM generate_series(1, 100); -NOTICE: executing the copy locally for shard xxxxx -SELECT COUNT(*) FROM test JOIN ref_table USING(x); -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.test_1503000 test JOIN coordinator_shouldhaveshards.ref_table_1503039 ref_table ON ((test.x OPERATOR(pg_catalog.=) ref_table.x))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (coordinator_shouldhaveshards.test_1503003 test JOIN coordinator_shouldhaveshards.ref_table_1503039 ref_table ON ((test.x OPERATOR(pg_catalog.=) ref_table.x))) WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -ROLLBACK; --- writing to local file and remote intermediate files --- at the same time -INSERT INTO ref_table SELECT *, * FROM generate_series(1, 100); -NOTICE: executing the copy locally for shard xxxxx -CREATE UNIQUE INDEX test_x_unique ON test(x); -WITH cte_1 AS ( -INSERT INTO test SELECT sum(x), y FROM test GROUP BY y ON CONFLICT (x) DO UPDATE SET y = EXCLUDED.y + 1 RETURNING *) -SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT sum(x) AS x, y FROM coordinator_shouldhaveshards.test_1503000 test WHERE true GROUP BY y -NOTICE: executing the command locally: SELECT sum(x) AS x, y FROM coordinator_shouldhaveshards.test_1503003 test WHERE true GROUP BY y -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) cte_1 - count ---------------------------------------------------------------------- - 0 -(1 row) - -DROP INDEX test_x_unique; --- issue #4237: preventing empty placement creation on coordinator -CREATE TABLE test_append_table(a int); -SELECT create_distributed_table('test_append_table', 'a', 'append'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- this will fail since it will try to create an empty placement in the --- coordinator as well -SET citus.shard_replication_factor TO 3; -SELECT master_create_empty_shard('test_append_table'); -NOTICE: Creating placements for the append partitioned tables on the coordinator is not supported, skipping coordinator ... -ERROR: could only create 2 of 3 of required shard replicas --- this will create an empty shard with replicas in the two worker nodes -SET citus.shard_replication_factor TO 2; -SELECT 1 FROM master_create_empty_shard('test_append_table'); -NOTICE: Creating placements for the append partitioned tables on the coordinator is not supported, skipping coordinator ... - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- verify groupid is not 0 for each placement -SELECT COUNT(*) FROM pg_dist_placement p, pg_dist_shard s WHERE p.shardid = s.shardid AND s.logicalrelid = 'test_append_table'::regclass AND p.groupid > 0; - count ---------------------------------------------------------------------- - 2 -(1 row) - -SET citus.shard_replication_factor TO 1; --- test partitioned index creation with long name -CREATE TABLE test_index_creation1 -( - tenant_id integer NOT NULL, - timeperiod timestamp without time zone NOT NULL, - field1 integer NOT NULL, - inserted_utc timestamp without time zone NOT NULL DEFAULT now(), - PRIMARY KEY(tenant_id, timeperiod) -) PARTITION BY RANGE (timeperiod); -CREATE TABLE test_index_creation1_p2020_09_26 -PARTITION OF test_index_creation1 FOR VALUES FROM ('2020-09-26 00:00:00') TO ('2020-09-27 00:00:00'); -CREATE TABLE test_index_creation1_p2020_09_27 -PARTITION OF test_index_creation1 FOR VALUES FROM ('2020-09-27 00:00:00') TO ('2020-09-28 00:00:00'); -select create_distributed_table('test_index_creation1', 'tenant_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- should be able to create indexes with INCLUDE/WHERE -CREATE INDEX ix_test_index_creation5 ON test_index_creation1 - USING btree(tenant_id, timeperiod) - INCLUDE (field1) WHERE (tenant_id = 100); -NOTICE: executing the command locally: CREATE INDEX ix_test_index_creation5_1503042 ON coordinator_shouldhaveshards.test_index_creation1_1503042 USING btree (tenant_id ,timeperiod ) INCLUDE (field1 ) WHERE (tenant_id = 100) -NOTICE: executing the command locally: CREATE INDEX ix_test_index_creation5_1503045 ON coordinator_shouldhaveshards.test_index_creation1_1503045 USING btree (tenant_id ,timeperiod ) INCLUDE (field1 ) WHERE (tenant_id = 100) -NOTICE: executing the command locally: SELECT pg_catalog.citus_run_local_command($$SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503048', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503048');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503049', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503049');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503050', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503050');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503051', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503051');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503052', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503052');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503053', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503053');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503054', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503054');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503055', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503055');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503056', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503056');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503057', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503057');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503058', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503058');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503042'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503059', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503059')$$) -NOTICE: executing the command locally: SELECT pg_catalog.citus_run_local_command($$SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503048', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503048');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503049', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503049');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503050', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503050');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503051', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503051');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503052', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503052');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_26_1503053', 'test_index_creation1_p2020_09_2_tenant_id_time_6020e8f8_1503053');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503054', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503054');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503055', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503055');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503056', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503056');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503057', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503057');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503058', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503058');SELECT worker_fix_partition_shard_index_names('coordinator_shouldhaveshards.ix_test_index_creation5_1503045'::regclass, 'coordinator_shouldhaveshards.test_index_creation1_p2020_09_27_1503059', 'test_index_creation1_p2020_09__tenant_id_timep_624f7e94_1503059')$$) --- test if indexes are created -SELECT 1 AS created WHERE EXISTS(SELECT * FROM pg_indexes WHERE indexname LIKE '%test_index_creation%'); - created ---------------------------------------------------------------------- - 1 -(1 row) - --- test alter_distributed_table UDF -SET citus.shard_count TO 4; -CREATE TABLE adt_table (a INT, b INT); -CREATE TABLE adt_col (a INT UNIQUE, b INT); -CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); -SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_table_1503060 AS citus_table_alias (a, b) VALUES (1,2), (5,6) -INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_col_1503064 AS citus_table_alias (a, b) VALUES (5,6) -INSERT INTO adt_ref VALUES (3), (5); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_ref_1503068 AS citus_table_alias (a) VALUES (5) -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 4 - adt_ref | distributed | a | 4 - adt_table | distributed | a | 4 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref, adt_table -(1 row) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SET client_min_messages TO WARNING; -SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEFAULT; -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 6 - adt_ref | distributed | a | 6 - adt_table | distributed | a | 6 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref, adt_table -(1 row) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); -NOTICE: creating a new table for coordinator_shouldhaveshards.adt_table -NOTICE: moving the data of coordinator_shouldhaveshards.adt_table -NOTICE: dropping the old coordinator_shouldhaveshards.adt_table -NOTICE: renaming the new table to coordinator_shouldhaveshards.adt_table - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 6 - adt_ref | distributed | a | 6 - adt_table | distributed | b | 6 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref - adt_table -(2 rows) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SELECT * FROM adt_table ORDER BY 1; - a | b ---------------------------------------------------------------------- - 1 | 2 - 3 | 4 - 5 | 6 -(3 rows) - -SELECT * FROM adt_col ORDER BY 1; - a | b ---------------------------------------------------------------------- - 3 | 4 - 5 | 6 - 7 | 8 -(3 rows) - -SELECT * FROM adt_ref ORDER BY 1; - a ---------------------------------------------------------------------- - 3 - 5 -(2 rows) - -SET client_min_messages TO WARNING; -BEGIN; -INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; -SELECT alter_distributed_table('adt_table', distribution_column:='a'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT COUNT(*) FROM adt_table; - count ---------------------------------------------------------------------- - 1003 -(1 row) - -END; -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text = 'adt_table'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_table | distributed | a | 6 -(1 row) - -SET client_min_messages TO DEFAULT; --- issue 4508 table_1 and table_2 are used to test --- some edge cases around intermediate result pruning -CREATE TABLE table_1 (key int, value text); -SELECT create_distributed_table('table_1', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE table_2 (key int, value text); -SELECT create_distributed_table('table_2', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO table_1 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.table_1_1503102 AS citus_table_alias (key, value) VALUES (1,'1'::text) -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.table_1_1503105 AS citus_table_alias (key, value) VALUES (2,'2'::text) -INSERT INTO table_2 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.table_2_1503106 AS citus_table_alias (key, value) VALUES (1,'1'::text), (5,'5'::text) -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.table_2_1503109 AS citus_table_alias (key, value) VALUES (2,'2'::text) -SET citus.log_intermediate_results TO ON; -SET client_min_messages to debug1; -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -NOTICE: executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503102 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503105 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT count(*) AS count, worker_column_1 AS key, max(worker_column_2) AS worker_column_3 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503106 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1 -NOTICE: executing the command locally: SELECT count(*) AS count, worker_column_1 AS key, max(worker_column_2) AS worker_column_3 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503109 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1 - count | key ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -INSERT INTO table_1 SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -NOTICE: executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503102 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT key, value FROM coordinator_shouldhaveshards.table_1_1503105 table_1 WHERE true ORDER BY key, value DESC LIMIT '1'::bigint -DEBUG: Subplan XXX_2 will be written to local file -NOTICE: executing the command locally: SELECT count(*) AS auto_coerced_by_citus_0, (worker_column_1)::text AS auto_coerced_by_citus_1, worker_column_1 AS discarded_target_item_1, max(worker_column_2) AS worker_column_4 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503106 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1 -NOTICE: executing the command locally: SELECT count(*) AS auto_coerced_by_citus_0, (worker_column_1)::text AS auto_coerced_by_citus_1, worker_column_1 AS discarded_target_item_1, max(worker_column_2) AS worker_column_4 FROM (SELECT a.key AS worker_column_1, table_2.value AS worker_column_2 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN coordinator_shouldhaveshards.table_2_1503109 table_2(key, value) USING (key))) worker_subquery GROUP BY worker_column_1 -NOTICE: executing the command locally: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -NOTICE: executing the copy locally for shard xxxxx -WITH stats AS ( - SELECT count(key) m FROM table_1 -), -inserts AS ( - INSERT INTO table_2 - SELECT key, count(*) - FROM table_1 - WHERE key >= (SELECT m FROM stats) - GROUP BY key - HAVING count(*) <= (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM coordinator_shouldhaveshards.table_1 -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO coordinator_shouldhaveshards.table_2 (key, value) SELECT key, count(*) AS count FROM coordinator_shouldhaveshards.table_1 WHERE (key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2.key, table_2.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -NOTICE: executing the command locally: SELECT count(key) AS m FROM coordinator_shouldhaveshards.table_1_1503102 table_1 WHERE true -NOTICE: executing the command locally: SELECT count(key) AS m FROM coordinator_shouldhaveshards.table_1_1503105 table_1 WHERE true -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator -NOTICE: executing the command locally: SELECT worker_column_1 AS key, (count(*))::text AS value FROM (SELECT table_1.key AS worker_column_1 FROM coordinator_shouldhaveshards.table_1_1503102 table_1 WHERE (table_1.key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats))) worker_subquery GROUP BY worker_column_1 HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS key, (count(*))::text AS value FROM (SELECT table_1.key AS worker_column_1 FROM coordinator_shouldhaveshards.table_1_1503105 table_1 WHERE (table_1.key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats))) worker_subquery GROUP BY worker_column_1 HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT '1'::bigint -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts - count ---------------------------------------------------------------------- - 0 -(1 row) - --- a helper function which return true if the coordinated --- trannsaction uses 2PC -SET citus.enable_metadata_sync TO OFF; -CREATE OR REPLACE FUNCTION coordinated_transaction_should_use_2PC() -RETURNS BOOL LANGUAGE C STRICT VOLATILE AS 'citus', -$$coordinated_transaction_should_use_2PC$$; -RESET citus.enable_metadata_sync; --- a local SELECT followed by remote SELECTs --- does not trigger 2PC -BEGIN; - SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- -(0 rows) - - WITH cte_1 AS (SELECT y FROM test WHERE x = 1 LIMIT 5) SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - WITH cte_1 as (SELECT * FROM test LIMIT 5) SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; --- remote SELECTs followed by local SELECTs --- does not trigger 2PC -BEGIN; - SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - WITH cte_1 as (SELECT * FROM test LIMIT 5) SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- -(0 rows) - - WITH cte_1 AS (SELECT y FROM test WHERE x = 1 LIMIT 5) SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; --- a local SELECT followed by a remote Modify --- triggers 2PC -BEGIN; - SELECT y FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT y FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - y ---------------------------------------------------------------------- -(0 rows) - - UPDATE test SET y = y +1; -NOTICE: executing the command locally: UPDATE coordinator_shouldhaveshards.test_1503000 test SET y = (y OPERATOR(pg_catalog.+) 1) -NOTICE: executing the command locally: UPDATE coordinator_shouldhaveshards.test_1503003 test SET y = (y OPERATOR(pg_catalog.+) 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; --- a local modify followed by a remote SELECT --- triggers 2PC -BEGIN; - INSERT INTO test VALUES (1,1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) - SELECT count(*) FROM test; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503003 test WHERE true - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; --- a local modify followed by a remote MODIFY --- triggers 2PC -BEGIN; - INSERT INTO test VALUES (1,1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) - UPDATE test SET y = y +1; -NOTICE: executing the command locally: UPDATE coordinator_shouldhaveshards.test_1503000 test SET y = (y OPERATOR(pg_catalog.+) 1) -NOTICE: executing the command locally: UPDATE coordinator_shouldhaveshards.test_1503003 test SET y = (y OPERATOR(pg_catalog.+) 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; --- a local modify followed by a remote single shard MODIFY --- triggers 2PC -BEGIN; - INSERT INTO test VALUES (1,1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) - INSERT INTO test VALUES (3,3); - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; --- a remote single shard modify followed by a local single --- shard MODIFY triggers 2PC -BEGIN; - INSERT INTO test VALUES (3,3); - INSERT INTO test VALUES (1,1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; --- a remote single shard select followed by a local single --- shard MODIFY triggers 2PC. But, the transaction manager --- is smart enough to skip sending 2PC as the remote --- command is read only -BEGIN; - SELECT count(*) FROM test WHERE x = 3; - count ---------------------------------------------------------------------- - 2 -(1 row) - - INSERT INTO test VALUES (1,1); -NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.test_1503000 (x, y) VALUES (1, 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - - SET LOCAL citus.log_remote_commands TO ON; -COMMIT; -NOTICE: issuing COMMIT -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx --- a local single shard select followed by a remote single --- shard modify does not trigger 2PC -BEGIN; - SELECT count(*) FROM test WHERE x = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM coordinator_shouldhaveshards.test_1503000 test WHERE (x OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 5 -(1 row) - - INSERT INTO test VALUES (3,3); - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - - SET LOCAL citus.log_remote_commands TO ON; -COMMIT; -NOTICE: issuing COMMIT -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -RESET client_min_messages; -\set VERBOSITY terse -DROP TABLE ref_table; -NOTICE: executing the command locally: DROP TABLE IF EXISTS coordinator_shouldhaveshards.ref_table_xxxxx CASCADE -DELETE FROM test; -DROP TABLE test; -DROP TABLE dist_table; -DROP TABLE ref; -NOTICE: executing the command locally: DROP TABLE IF EXISTS coordinator_shouldhaveshards.ref_xxxxx CASCADE -DROP TABLE test_append_table; -DROP SCHEMA coordinator_shouldhaveshards CASCADE; -NOTICE: drop cascades to 20 other objects -SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', false); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 7af842e29..e558df2e2 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -1,17 +1,6 @@ -- -- CTE_INLINE -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA cte_inline; SET search_path TO cte_inline; SET citus.next_shard_id TO 1960000; diff --git a/src/test/regress/expected/cte_inline_0.out b/src/test/regress/expected/cte_inline_0.out deleted file mode 100644 index e5afa4ee3..000000000 --- a/src/test/regress/expected/cte_inline_0.out +++ /dev/null @@ -1,1489 +0,0 @@ --- --- CTE_INLINE --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA cte_inline; -SET search_path TO cte_inline; -SET citus.next_shard_id TO 1960000; -CREATE TABLE test_table (key int, value text, other_value jsonb); -SELECT create_distributed_table ('test_table', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'test' || i)) FROM generate_series (0, 100) i; -SET client_min_messages TO DEBUG; --- Citus should not inline this CTE because otherwise it cannot --- plan the query -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - * -FROM - test_table LEFT JOIN cte_1 USING (value) -ORDER BY 1 DESC LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test99 | 9 | {"f1": 99, "f2": 1782, "f3": "test99"} | 9 | {"f1": 99, "f2": 1782, "f3": "test99"} - test98 | 8 | {"f1": 98, "f2": 1764, "f3": "test98"} | 8 | {"f1": 98, "f2": 1764, "f3": "test98"} - test97 | 7 | {"f1": 97, "f2": 1746, "f3": "test97"} | 7 | {"f1": 97, "f2": 1746, "f3": "test97"} -(3 rows) - --- Should still not be inlined even if NOT MATERIALIZED is passed -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - * -FROM - test_table LEFT JOIN cte_1 USING (value) -ORDER BY 2 DESC LIMIT 1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.key DESC LIMIT 1 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 1 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test9 | 9 | {"f1": 9, "f2": 162, "f3": "test9"} | 9 | {"f1": 9, "f2": 162, "f3": "test9"} -(1 row) - --- the cte can be inlined because the unsupported --- part of the query (subquery in WHERE clause) --- doesn't access the cte -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 -WHERE - key IN ( - SELECT - (SELECT 1) - FROM - test_table WHERE key = 1 - ); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 -DEBUG: generating subplan XXX_1 for subquery SELECT (SELECT 1) FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- a similar query as the above, and this time the planning --- fails, but it fails because the subquery in WHERE clause --- cannot be planned by Citus -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 -WHERE - key IN ( - SELECT - key - FROM - test_table - FOR UPDATE - ); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. -ERROR: could not run distributed query with FOR UPDATE/SHARE commands -HINT: Consider using an equality filter on the distributed table's partition column. --- Citus does the inlining, the planning fails --- and retries without inlining, which works --- fine later via recursive planning -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT *, (SELECT 1) -FROM - (SELECT * - FROM cte_1) AS foo -ORDER BY 2 DESC LIMIT 1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 1 - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 -(1 row) - --- a little more complicated query tree --- Citus does the inlining, the planning fails --- and retries without inlining, which works -WITH top_cte AS - (SELECT * - FROM test_table) -SELECT count(*) -FROM top_cte, - (WITH cte_1 AS - (SELECT * - FROM test_table) SELECT *, (SELECT 1) - FROM - (SELECT * - FROM cte_1) AS foo) AS bar; -DEBUG: CTE top_cte is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) top_cte, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) bar(key, value, other_value, "?column?") -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10201 -(1 row) - --- CTE is used inside a subquery in WHERE clause --- the query wouldn't work by inlining, so Citus --- retries again via recursive planning, which --- works fine -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT count(*) -FROM test_table -WHERE KEY IN - (SELECT (SELECT 1) - FROM - (SELECT *, - random() - FROM - (SELECT * - FROM cte_1) AS foo) AS bar); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT (SELECT 1) FROM (SELECT foo.key, foo.value, foo.other_value, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo) bar -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- cte_1 is used inside another CTE, but still --- doesn't work when inlined because it is finally --- used in an unsupported query --- but still works fine because recursive planning --- kicks in -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT (SELECT 1) AS KEY FROM ( - WITH cte_2 AS (SELECT *, random() - FROM (SELECT *,random() FROM cte_1) as foo) -SELECT *, random() FROM cte_2) as bar ORDER BY 1 DESC LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT key, value, other_value, random, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value, random() AS random FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (SELECT 1) AS key FROM (SELECT cte_2.key, cte_2.value, cte_2.other_value, cte_2.random, cte_2.random_1 AS random, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random, intermediate_result.random_1 AS random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision, random_1 double precision)) cte_2(key, value, other_value, random, random_1)) bar(key, value, other_value, random, random_1, random_2) ORDER BY (SELECT 1) DESC LIMIT 3 -DEBUG: Creating router plan - key ---------------------------------------------------------------------- - 1 - 1 - 1 -(3 rows) - --- in this example, cte_2 can be inlined, because it is not used --- on any query that Citus cannot plan. However, cte_1 should not be --- inlined, because it is used with a subquery in target list -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (select * from test_table) -SELECT - count(*) -FROM - (SELECT *, (SELECT 1) FROM cte_1) as foo - JOIN - cte_2 - ON (true); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT cte_1.key, cte_1.value, cte_1.other_value, (SELECT 1) FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo(key, value, other_value, "?column?") JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON (true)) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10201 -(1 row) - --- unreferenced CTEs are just ignored --- by Citus/Postgres -WITH a AS (SELECT * FROM test_table) -SELECT - *, row_number() OVER () -FROM - test_table -WHERE - key = 1 -ORDER BY 3 DESC -LIMIT 5; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - key | value | other_value | row_number ---------------------------------------------------------------------- - 1 | test91 | {"f1": 91, "f2": 1638, "f3": "test91"} | 10 - 1 | test81 | {"f1": 81, "f2": 1458, "f3": "test81"} | 9 - 1 | test71 | {"f1": 71, "f2": 1278, "f3": "test71"} | 8 - 1 | test61 | {"f1": 61, "f2": 1098, "f3": "test61"} | 7 - 1 | test51 | {"f1": 51, "f2": 918, "f3": "test51"} | 6 -(5 rows) - --- router queries are affected by the distributed --- cte inlining -WITH a AS (SELECT * FROM test_table WHERE key = 1) -SELECT - *, (SELECT 1) -FROM - a -WHERE - key = 1 -ORDER BY 1 DESC -LIMIT 5; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - key | value | other_value | ?column? ---------------------------------------------------------------------- - 1 | test1 | {"f1": 1, "f2": 18, "f3": "test1"} | 1 - 1 | test11 | {"f1": 11, "f2": 198, "f3": "test11"} | 1 - 1 | test21 | {"f1": 21, "f2": 378, "f3": "test21"} | 1 - 1 | test31 | {"f1": 31, "f2": 558, "f3": "test31"} | 1 - 1 | test41 | {"f1": 41, "f2": 738, "f3": "test41"} | 1 -(5 rows) - --- non router queries are affected by the distributed --- cte inlining as well -WITH a AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - count ---------------------------------------------------------------------- - 10 -(1 row) - --- explicitely using NOT MATERIALIZED should result in the same -WITH a AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - count ---------------------------------------------------------------------- - 10 -(1 row) - --- using MATERIALIZED should cause inlining not to happen -WITH a AS MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 10 -(1 row) - --- EXPLAIN should show the difference between materialized an not materialized -EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Seq Scan on test_table_1960000 test_table - Filter: (key = 1) -(8 rows) - -EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Creating router plan - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on test_table_1960000 test_table - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Function Scan on read_intermediate_result intermediate_result - Filter: (key = 1) -(15 rows) - --- citus should not inline the CTE because it is used multiple times -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- NOT MATERIALIZED should cause the query to be inlined twice -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED -\set VERBOSITY terse -SELECT public.coordinator_plan_with_subplans($Q$ -EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -$Q$); -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) -DEBUG: Creating router plan - coordinator_plan_with_subplans ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Task Count: 1 -(5 rows) - -\set VERBOSITY default -EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - QUERY PLAN ---------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Hash Join - Hash Cond: (test_table.key = test_table_1.key) - -> Seq Scan on test_table_1960000 test_table - -> Hash - -> Seq Scan on test_table_1960000 test_table_1 -(12 rows) - --- ctes with volatile functions are not --- inlined -WITH cte_1 AS (SELECT *, random() FROM test_table) -SELECT - key, value -FROM - cte_1 -ORDER BY 2 DESC LIMIT 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 ORDER BY value DESC LIMIT 1 -DEBUG: Creating router plan - key | value ---------------------------------------------------------------------- - 9 | test99 -(1 row) - --- even with NOT MATERIALIZED volatile functions should not be inlined -WITH cte_1 AS NOT MATERIALIZED (SELECT *, random() FROM test_table) -SELECT - count(*) -FROM - cte_1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 101 -(1 row) - --- cte_1 should be able to inlined even if --- it is used one level below -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM -( - WITH ct2 AS (SELECT * FROM cte_1) - SELECT * FROM ct2 -) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE ct2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 101 -(1 row) - --- a similar query, but there is also --- one more cte, which relies on the previous --- CTE -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(DISTINCT key) -FROM -( - WITH cte_2 AS (SELECT * FROM cte_1), - cte_3 AS (SELECT * FROM cte_2) - SELECT * FROM cte_3 -) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: CTE cte_3 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- inlined CTE contains a reference to outer query --- should be fine (because we pushdown the whole query) -SELECT count(*) - FROM - (SELECT * - FROM test_table) AS test_table_cte - JOIN LATERAL - (WITH bar AS (SELECT * - FROM test_table - WHERE key = test_table_cte.key) - SELECT * - FROM - bar - LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; -DEBUG: CTE bar is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10331 -(1 row) - --- inlined CTE contains a reference to outer query --- should be fine (even if the recursive planning fails --- to recursively plan the query) -SELECT count(*) - FROM - (SELECT * - FROM test_table) AS test_table_cte - JOIN LATERAL - (WITH bar AS (SELECT * - FROM test_table - WHERE key = test_table_cte.key) - SELECT * - FROM - bar - LEFT JOIN test_table u2 ON u2.key = bar.value::int) AS foo ON TRUE; -DEBUG: CTE bar is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries --- inlined CTE can recursively planned later, that's the decision --- recursive planning makes --- LIMIT 5 in cte2 triggers recusrive planning, after cte inlining -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - * -FROM -( - WITH ct2 AS (SELECT * FROM cte_1 ORDER BY 1, 2, 3 LIMIT 5) - SELECT * FROM ct2 -) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 5; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE ct2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 ORDER BY key, value, other_value LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value FROM (SELECT ct2.key, ct2.value, ct2.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) ct2) foo ORDER BY key DESC, value DESC, other_value DESC LIMIT 5 -DEBUG: Creating router plan - key | value | other_value ---------------------------------------------------------------------- - 0 | test30 | {"f1": 30, "f2": 540, "f3": "test30"} - 0 | test20 | {"f1": 20, "f2": 360, "f3": "test20"} - 0 | test100 | {"f1": 100, "f2": 1800, "f3": "test100"} - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} -(5 rows) - --- all nested CTEs can be inlinied -WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) - SELECT * FROM cte_1) - SELECT * FROM cte_1 WHERE key >= 1) - SELECT * FROM cte_1 WHERE key >= 2) - SELECT * FROM cte_1 WHERE key >= 3) - SELECT * FROM cte_1 WHERE key >= 4) - SELECT * FROM cte_1 WHERE key >= 5) -SELECT * FROM cte_1 WHERE key = 1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 - count | key ---------------------------------------------------------------------- -(0 rows) - --- ctes can be inlined even if they are used --- in set operations -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -SELECT count(*) FROM ( -(SELECT * FROM cte_1 EXCEPT SELECT * FROM test_table) -UNION -(SELECT * FROM cte_2)) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_2 -DEBUG: Creating router plan -DEBUG: generating subplan XXX_4 for subquery (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) UNION SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) foo -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 101 -(1 row) - --- cte_1 is going to be inlined even inside another set operation -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) -(SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) -UNION -(SELECT *, 1 FROM cte_2) -ORDER BY 1,2; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ORDER BY 1, 2 -DEBUG: Creating router plan - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 - 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 -(3 rows) - --- cte_1 is safe to inline, even if because after inlining --- it'd be in a query tree where there is a query that is --- not supported by Citus unless recursively planned --- cte_2 is on another queryTree, should be fine -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -(SELECT *, (SELECT key FROM cte_1) FROM test_table) -UNION -(SELECT *, 1 FROM cte_2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) AS key FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.key_1 AS key FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, key_1 integer) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 -DEBUG: Creating router plan -ERROR: more than one row returned by a subquery used as an expression -CONTEXT: while executing command on localhost:xxxxx --- after inlining CTEs, the query becomes --- subquery pushdown with set operations -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -SELECT max(key) FROM -( - SELECT * FROM cte_1 - UNION - SELECT * FROM cte_2 -) as bar; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - max ---------------------------------------------------------------------- - 9 -(1 row) - --- cte LEFT JOIN subquery should only work --- when CTE is inlined, as Citus currently --- doesn't know how to handle intermediate --- results in the outer parts of outer --- queries -WITH cte AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte LEFT JOIN test_table USING (key); -DEBUG: CTE cte is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- the CTEs are very simple, so postgres --- can pull-up the subqueries after inlining --- the CTEs, and the query that we send to workers --- becomes a join between two tables -WITH cte_1 AS (SELECT key FROM test_table), - cte_2 AS (SELECT key FROM test_table) -SELECT - count(*) -FROM - cte_1 JOIN cte_2 USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- the following query is kind of interesting --- During INSERT .. SELECT via coordinator, --- Citus moves the CTEs into SELECT part, and plans/execute --- the SELECT separately. Thus, fist_table_cte can be inlined --- by Citus -- but not by Postgres -WITH fist_table_cte AS - (SELECT * FROM test_table) -INSERT INTO test_table - (key, value) - SELECT - key, value - FROM - fist_table_cte; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: CTE fist_table_cte is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'key' -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960001_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960002_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) --- the following INSERT..SELECT is even more interesting --- the CTE becomes pushdownable -INSERT INTO test_table -WITH fist_table_cte AS - (SELECT * FROM test_table) - SELECT - key, value - FROM - fist_table_cte; -DEBUG: CTE fist_table_cte is going to be inlined via distributed planning -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960000 test_table) fist_table_cte WHERE (key IS NOT NULL) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960001 test_table) fist_table_cte WHERE (key IS NOT NULL) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960002 test_table) fist_table_cte WHERE (key IS NOT NULL) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960003 test_table) fist_table_cte WHERE (key IS NOT NULL) --- update/delete/modifying ctes --- we don't support any cte inlining in modifications --- queries and modifying CTEs -WITH cte_1 AS (SELECT * FROM test_table) - DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) -DEBUG: Creating router plan --- NOT MATERIALIZED should not CTEs that are used in a modifying query, because --- we de still don't support it -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) - DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) -DEBUG: Creating router plan --- we don't inline CTEs if they are modifying CTEs -WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) -SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key -DEBUG: Creating router plan -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 ORDER BY key DESC LIMIT 3 -DEBUG: Creating router plan - key ---------------------------------------------------------------------- - 7 - 7 - 7 -(3 rows) - --- NOT MATERIALIZED should not affect modifying CTEs -WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) -SELECT count(*) FROM cte_1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 0) RETURNING key -DEBUG: Creating router plan -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 164 -(1 row) - --- cte with column aliases -SELECT * FROM test_table, -(WITH cte_1 (x,y) AS (SELECT * FROM test_table), - cte_2 (z,y) AS (SELECT value, other_value, key FROM test_table), - cte_3 (t,m) AS (SELECT z, y, key as cte_2_key FROM cte_2) - SELECT * FROM cte_2, cte_3) as bar -ORDER BY value, other_value, z, y, t, m, cte_2_key -LIMIT 5; -DEBUG: CTE cte_3 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT value, other_value, key FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 5 - key | value | other_value | z | y | key | t | m | cte_2_key ---------------------------------------------------------------------- - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test15 | {"f1": 15, "f2": 270, "f3": "test15"} | 5 -(5 rows) - --- cte used in HAVING subquery just works fine --- even if it is inlined -WITH cte_1 AS (SELECT max(key) as max FROM test_table) -SELECT - key, count(*) -FROM - test_table -GROUP BY - key -HAVING - (count(*) > (SELECT max FROM cte_1)) -ORDER BY 2 DESC, 1 DESC -LIMIT 5; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) AS count FROM cte_inline.test_table GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.>) (SELECT cte_1.max FROM (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1)) ORDER BY (count(*)) DESC, key DESC LIMIT 5 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 5 - key | count ---------------------------------------------------------------------- - 8 | 40 - 5 | 40 - 2 | 40 -(3 rows) - --- cte used in ORDER BY just works fine --- even if it is inlined -WITH cte_1 AS (SELECT max(key) as max FROM test_table) -SELECT - key -FROM - test_table JOIN cte_1 ON (key = max) -ORDER BY - cte_1.max -LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key FROM (cte_inline.test_table JOIN (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1 ON ((test_table.key OPERATOR(pg_catalog.=) cte_1.max))) ORDER BY cte_1.max LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - key ---------------------------------------------------------------------- - 8 - 8 - 8 -(3 rows) - -PREPARE inlined_cte_without_params AS - WITH cte_1 AS (SELECT count(*) FROM test_table GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE non_inlined_cte_without_params AS - WITH cte_1 AS (SELECT * FROM test_table) - SELECT - * - FROM - test_table LEFT JOIN cte_1 USING (value) ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; -PREPARE inlined_cte_has_parameter_on_non_dist_key(text) AS - WITH cte_1 AS (SELECT count(*) FROM test_table WHERE value = $1 GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE inlined_cte_has_parameter_on_dist_key(int) AS - WITH cte_1 AS (SELECT count(*) FROM test_table WHERE key > $1 GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE non_inlined_cte_has_parameter_on_dist_key(int) AS - WITH cte_1 AS (SELECT * FROM test_table where key > $1) - SELECT - * - FROM - test_table LEFT JOIN cte_1 USING (value) ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; -PREPARE retry_planning(int) AS - WITH cte_1 AS (SELECT * FROM test_table WHERE key > $1) - SELECT json_object_agg(DISTINCT key, value) FROM cte_1 ORDER BY max(key), min(value) DESC LIMIT 3; -EXECUTE inlined_cte_without_params; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE non_inlined_cte_without_params; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_without_params; - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_without_params; - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_without_params; - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_without_params; - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_without_params; - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test2'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test3'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test5'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test6'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 -(2 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 -(2 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 -(2 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_dist_key(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 -(1 row) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 4) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 5) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 6) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.value, test_table.key, test_table.other_value, cte_1.key, cte_1.other_value FROM (cte_inline.test_table LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 USING (value)) ORDER BY test_table.value DESC, test_table.key DESC, test_table.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - value | key | other_value | key | other_value ---------------------------------------------------------------------- - test98 | 8 | | 8 | - test98 | 8 | | 8 | - test98 | 8 | | 8 | -(3 rows) - -EXECUTE retry_planning(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -EXECUTE retry_planning(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -EXECUTE retry_planning(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -EXECUTE retry_planning(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -EXECUTE retry_planning(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -EXECUTE retry_planning(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - -WITH b AS (SELECT * FROM test_table) -SELECT count(*) FROM (SELECT key as x FROM test_table OFFSET 0) as ref LEFT JOIN b ON (ref.x = b.key); -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 -DEBUG: recursively planning right side of the left join since the outer side is a recurring rel -DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 4800 -(1 row) - --- this becomes a non-colocated subquery join --- because after the CTEs are inlined the joins --- become a non-colocated subquery join -WITH a AS (SELECT * FROM test_table), -b AS (SELECT * FROM test_table) -SELECT count(*) FROM a LEFT JOIN b ON (a.value = b.value); -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 480 -(1 row) - -WITH a AS (SELECT * FROM test_table OFFSET 0), -b AS (SELECT * FROM test_table) -SELECT min(a.key) FROM a LEFT JOIN b ON (a.value = b.value); -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 -DEBUG: recursively planning right side of the left join since the outer side is a recurring rel -DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) -DEBUG: Creating router plan - min ---------------------------------------------------------------------- - 2 -(1 row) - --- after both CTEs are inlined, this becomes non-colocated subquery join -WITH cte_1 AS (SELECT * FROM test_table), -cte_2 AS (SELECT * FROM test_table) -SELECT * FROM cte_1 JOIN cte_2 ON (cte_1.value > cte_2.value) ORDER BY 1,2,3,4,5,6 DESC LIMIT 3;; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - key | value | other_value | key | value | other_value ---------------------------------------------------------------------- - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | -(3 rows) - --- full join is only supported when both sides are --- recursively planned -WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), - cte_2 AS (SELECT value FROM test_table WHERE key > 3) -SELECT * FROM cte_1 FULL JOIN cte_2 USING (value) ORDER BY 1 DESC LIMIT 3;; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: recursively planning left side of the full join since the other side is a recurring rel -DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 -DEBUG: Creating router plan - value ---------------------------------------------------------------------- - test98 - test98 - test98 -(3 rows) - --- an unsupported agg. for multi-shard queries --- so CTE has to be recursively planned -WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) -SELECT json_object_agg(DISTINCT key, value) FROM cte_1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } -(1 row) - --- both cte_1 and cte_2 are going to be inlined. --- later, cte_2 is recursively planned since it doesn't have --- GROUP BY but aggragate in a subquery. --- this is an important example of being able to recursively plan --- "some" of the CTEs -WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), - cte_2 AS (SELECT max(value) as value FROM test_table WHERE key > 3) -SELECT count(*) FROM cte_1 JOIN cte_2 USING (value); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(value) AS value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 4 -(1 row) - --- prevent DROP CASCADE to give notices -SET client_min_messages TO ERROR; -DROP SCHEMA cte_inline CASCADE; diff --git a/src/test/regress/expected/detect_conn_close.out b/src/test/regress/expected/detect_conn_close.out index 60973de76..41f98ac6e 100644 --- a/src/test/regress/expected/detect_conn_close.out +++ b/src/test/regress/expected/detect_conn_close.out @@ -1,13 +1,6 @@ -- -- PG15+ test as WL_SOCKET_CLOSED exposed for PG15+ -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE SCHEMA socket_close; SET search_path TO socket_close; CREATE OR REPLACE FUNCTION kill_all_cached_internal_conns(gpid bigint) diff --git a/src/test/regress/expected/detect_conn_close_0.out b/src/test/regress/expected/detect_conn_close_0.out deleted file mode 100644 index 27e9787c6..000000000 --- a/src/test/regress/expected/detect_conn_close_0.out +++ /dev/null @@ -1,9 +0,0 @@ --- --- PG15+ test as WL_SOCKET_CLOSED exposed for PG15+ --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/grant_on_schema_propagation.out b/src/test/regress/expected/grant_on_schema_propagation.out index 77447c2dd..fc34ab416 100644 --- a/src/test/regress/expected/grant_on_schema_propagation.out +++ b/src/test/regress/expected/grant_on_schema_propagation.out @@ -1,16 +1,6 @@ -- -- GRANT_ON_SCHEMA_PROPAGATION -- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - -- test grants are propagated when the schema is CREATE SCHEMA dist_schema; CREATE TABLE dist_schema.dist_table (id int); diff --git a/src/test/regress/expected/grant_on_schema_propagation_0.out b/src/test/regress/expected/grant_on_schema_propagation_0.out deleted file mode 100644 index 9806a0dbd..000000000 --- a/src/test/regress/expected/grant_on_schema_propagation_0.out +++ /dev/null @@ -1,400 +0,0 @@ --- --- GRANT_ON_SCHEMA_PROPAGATION --- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - --- test grants are propagated when the schema is -CREATE SCHEMA dist_schema; -CREATE TABLE dist_schema.dist_table (id int); -CREATE SCHEMA another_dist_schema; -CREATE TABLE another_dist_schema.dist_table (id int); -SET citus.enable_ddl_propagation TO off; -CREATE SCHEMA non_dist_schema; -SET citus.enable_ddl_propagation TO on; --- create roles on all nodes -CREATE USER role_1; -CREATE USER role_2; -CREATE USER role_3; --- do some varying grants -GRANT USAGE, CREATE ON SCHEMA dist_schema TO role_1 WITH GRANT OPTION; -GRANT USAGE ON SCHEMA dist_schema TO role_2; -SET ROLE role_1; -GRANT USAGE ON SCHEMA dist_schema TO role_3 WITH GRANT OPTION; -GRANT CREATE ON SCHEMA dist_schema TO role_3; -GRANT CREATE, USAGE ON SCHEMA dist_schema TO PUBLIC; -RESET ROLE; -GRANT USAGE ON SCHEMA dist_schema TO PUBLIC; -SELECT create_distributed_table('dist_schema.dist_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('another_dist_schema.dist_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'dist_schema'; - nspname | nspacl ---------------------------------------------------------------------- - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U/postgres,role_3=U*C/role_1,=UC/role_1,=U/postgres} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'dist_schema'; - nspname | nspacl ---------------------------------------------------------------------- - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U/postgres,role_3=U*C/role_1,=UC/role_1,=U/postgres} -(1 row) - -\c - - - :master_port --- grant all permissions -GRANT ALL ON SCHEMA dist_schema, another_dist_schema, non_dist_schema TO role_1, role_2, role_3 WITH GRANT OPTION; -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/postgres,role_3=U*C*/postgres} - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/postgres,role_3=U*C/role_1,=UC/role_1,=U/postgres,role_3=U*C*/postgres} - non_dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/postgres,role_3=U*C*/postgres} -(3 rows) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/postgres,role_3=U*C*/postgres} - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/postgres,role_3=U*C/role_1,=UC/role_1,=U/postgres,role_3=U*C*/postgres} -(2 rows) - -\c - - - :master_port --- revoke all permissions -REVOKE ALL ON SCHEMA dist_schema, another_dist_schema, non_dist_schema FROM role_1, role_2, role_3, PUBLIC CASCADE; -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres} - dist_schema | {postgres=UC/postgres} - non_dist_schema | {postgres=UC/postgres} -(3 rows) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres} - dist_schema | {postgres=UC/postgres} -(2 rows) - -\c - - - :master_port --- grant with multiple permissions, roles and schemas -GRANT USAGE, CREATE ON SCHEMA dist_schema, another_dist_schema, non_dist_schema TO role_1, role_2, role_3; -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_1=UC/postgres,role_2=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_1=UC/postgres,role_2=UC/postgres,role_3=UC/postgres} - non_dist_schema | {postgres=UC/postgres,role_1=UC/postgres,role_2=UC/postgres,role_3=UC/postgres} -(3 rows) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_1=UC/postgres,role_2=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_1=UC/postgres,role_2=UC/postgres,role_3=UC/postgres} -(2 rows) - -\c - - - :master_port --- revoke with multiple permissions, roles and schemas -REVOKE USAGE, CREATE ON SCHEMA dist_schema, another_dist_schema, non_dist_schema FROM role_1, role_2; -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - non_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} -(3 rows) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_3=UC/postgres} -(2 rows) - -\c - - - :master_port --- grant with grant option -GRANT USAGE ON SCHEMA dist_schema TO role_1, role_3 WITH GRANT OPTION; -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_3=U*C/postgres,role_1=U*/postgres} -(2 rows) - -\c - - - :master_port --- revoke grant option for -REVOKE GRANT OPTION FOR USAGE ON SCHEMA dist_schema FROM role_3; -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_3=UC/postgres,role_1=U*/postgres} -(2 rows) - -\c - - - :master_port --- test current_user -SET citus.enable_alter_role_propagation TO ON; -ALTER ROLE role_1 SUPERUSER; -SET citus.enable_alter_role_propagation TO OFF; -SET ROLE role_1; --- this is only supported on citus enterprise where multiple users can be managed --- The output of the nspname select below will indicate if the create has been granted -GRANT CREATE ON SCHEMA dist_schema TO CURRENT_USER; -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname IN ('dist_schema', 'another_dist_schema', 'non_dist_schema') ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - another_dist_schema | {postgres=UC/postgres,role_3=UC/postgres} - dist_schema | {postgres=UC/postgres,role_3=UC/postgres,role_1=U*C/postgres} -(2 rows) - -\c - - - :master_port -RESET ROLE; -SET citus.enable_alter_role_propagation TO ON; -ALTER ROLE role_1 NOSUPERUSER; -SET citus.enable_alter_role_propagation TO OFF; -DROP TABLE dist_schema.dist_table, another_dist_schema.dist_table; -DROP SCHEMA dist_schema; -DROP SCHEMA another_dist_schema; -DROP SCHEMA non_dist_schema; --- test if the grantors are propagated correctly --- first remove one of the worker nodes -SET citus.shard_replication_factor TO 1; -SELECT master_remove_node('localhost', :worker_2_port); - master_remove_node ---------------------------------------------------------------------- - -(1 row) - --- create a new schema -CREATE SCHEMA grantor_schema; --- give cascading permissions -GRANT USAGE, CREATE ON SCHEMA grantor_schema TO role_1 WITH GRANT OPTION; -GRANT CREATE ON SCHEMA grantor_schema TO PUBLIC; -SET ROLE role_1; -GRANT USAGE ON SCHEMA grantor_schema TO role_2 WITH GRANT OPTION; -GRANT CREATE ON SCHEMA grantor_schema TO role_2; -GRANT USAGE, CREATE ON SCHEMA grantor_schema TO role_3; -GRANT CREATE, USAGE ON SCHEMA grantor_schema TO PUBLIC; -SET ROLE role_2; -GRANT USAGE ON SCHEMA grantor_schema TO role_3; -RESET ROLE; --- distribute the schema -CREATE TABLE grantor_schema.grantor_table (id INT); -SELECT create_distributed_table('grantor_schema.grantor_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- check if the grantors are propagated correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=U*C/role_1,role_3=UC/role_1,=UC/role_1,role_3=U/role_2} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=U*C/role_1,role_3=UC/role_1,=UC/role_1,role_3=U/role_2} -(1 row) - -\c - - - :master_port --- add the previously removed node -SELECT 1 FROM master_add_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- check if the grantors are propagated correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=U*C/role_1,role_3=UC/role_1,=UC/role_1,role_3=U/role_2} -(1 row) - -\c - - - :worker_2_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=U*C/role_1,role_3=UC/role_1,=UC/role_1,role_3=U/role_2} -(1 row) - -\c - - - :master_port --- revoke one of the permissions -REVOKE USAGE ON SCHEMA grantor_schema FROM role_1 CASCADE; --- check if revoke worked correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=C*/postgres,=C/postgres,role_2=C/role_1,role_3=C/role_1,=C/role_1} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=C*/postgres,=C/postgres,role_2=C/role_1,role_3=C/role_1,=C/role_1} -(1 row) - -\c - - - :master_port --- test if grantor propagates correctly on already distributed schemas -GRANT USAGE ON SCHEMA grantor_schema TO role_1 WITH GRANT OPTION; -SET ROLE role_1; -GRANT USAGE ON SCHEMA grantor_schema TO role_2; -GRANT USAGE ON SCHEMA grantor_schema TO role_3 WITH GRANT OPTION; -SET ROLE role_3; -GRANT USAGE ON SCHEMA grantor_schema TO role_2; -RESET ROLE; --- check the results -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=UC/role_1,role_3=U*C/role_1,=C/role_1,role_2=U/role_3} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'grantor_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - grantor_schema | {postgres=UC/postgres,role_1=U*C*/postgres,=C/postgres,role_2=UC/role_1,role_3=U*C/role_1,=C/role_1,role_2=U/role_3} -(1 row) - -\c - - - :master_port -DROP TABLE grantor_schema.grantor_table; -DROP SCHEMA grantor_schema CASCADE; --- test distributing the schema with another user -CREATE SCHEMA dist_schema; -GRANT ALL ON SCHEMA dist_schema TO role_1 WITH GRANT OPTION; -SET ROLE role_1; -GRANT ALL ON SCHEMA dist_schema TO role_2 WITH GRANT OPTION; -CREATE TABLE dist_schema.dist_table (id int); -SELECT create_distributed_table('dist_schema.dist_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'dist_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/role_1} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'dist_schema' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - dist_schema | {postgres=UC/postgres,role_1=U*C*/postgres,role_2=U*C*/role_1} -(1 row) - -\c - - - :master_port -DROP TABLE dist_schema.dist_table; -DROP SCHEMA dist_schema CASCADE; --- test grants on public schema --- first remove one of the worker nodes -SET citus.shard_replication_factor TO 1; -SELECT master_remove_node('localhost', :worker_2_port); - master_remove_node ---------------------------------------------------------------------- - -(1 row) - --- to avoid different output in PG15 -GRANT CREATE ON SCHEMA public TO public; --- distribute the public schema (it has to be distributed by now but just in case) -CREATE TABLE public_schema_table (id INT); -SELECT create_distributed_table('public_schema_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- give cascading permissions -GRANT USAGE, CREATE ON SCHEMA PUBLIC TO role_1 WITH GRANT OPTION; -SET ROLE role_1; -GRANT USAGE ON SCHEMA PUBLIC TO PUBLIC; -RESET ROLE; --- check if the grants are propagated correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres,role_1=U*C*/postgres,=U/role_1} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres,role_1=U*C*/postgres,=U/role_1} -(1 row) - -\c - - - :master_port --- add the previously removed node -SELECT 1 FROM master_add_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- check if the grants are propagated correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres,role_1=U*C*/postgres,=U/role_1} -(1 row) - -\c - - - :worker_2_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres,role_1=U*C*/postgres,=U/role_1} -(1 row) - -\c - - - :master_port --- revoke those new permissions -REVOKE CREATE, USAGE ON SCHEMA PUBLIC FROM role_1 CASCADE; --- check if the grants are propagated correctly -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres} -(1 row) - -\c - - - :worker_1_port -SELECT nspname, nspacl FROM pg_namespace WHERE nspname = 'public' ORDER BY nspname; - nspname | nspacl ---------------------------------------------------------------------- - public | {postgres=UC/postgres,=UC/postgres} -(1 row) - -\c - - - :master_port -DROP TABLE public_schema_table; -DROP ROLE role_1, role_2, role_3; diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out index 476aa8640..fbe85914f 100644 --- a/src/test/regress/expected/insert_select_repartition.out +++ b/src/test/regress/expected/insert_select_repartition.out @@ -1,17 +1,6 @@ -- -- INSERT_SELECT_REPARTITION -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - -- tests behaviour of INSERT INTO ... SELECT with repartitioning CREATE SCHEMA insert_select_repartition; SET search_path TO 'insert_select_repartition'; diff --git a/src/test/regress/expected/insert_select_repartition_0.out b/src/test/regress/expected/insert_select_repartition_0.out deleted file mode 100644 index 904bd215a..000000000 --- a/src/test/regress/expected/insert_select_repartition_0.out +++ /dev/null @@ -1,1334 +0,0 @@ --- --- INSERT_SELECT_REPARTITION --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - --- tests behaviour of INSERT INTO ... SELECT with repartitioning -CREATE SCHEMA insert_select_repartition; -SET search_path TO 'insert_select_repartition'; -SET citus.next_shard_id TO 4213581; -SET citus.shard_replication_factor TO 1; --- 4 shards, hash distributed. --- Negate distribution column value. -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT * FROM generate_series(1, 10); -CREATE TABLE target_table(a int); -SELECT create_distributed_table('target_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEBUG2; -INSERT INTO target_table SELECT -a FROM source_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an operator in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213585 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213583_to_0,repartitioned_results_xxxxx_from_4213584_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213586 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213582_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213587 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213581_to_2,repartitioned_results_xxxxx_from_4213582_to_2,repartitioned_results_xxxxx_from_4213584_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213588 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213581_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) -RESET client_min_messages; -SELECT * FROM target_table WHERE a=-1 OR a=-3 OR a=-7 ORDER BY a; - a ---------------------------------------------------------------------- - -7 - -3 - -1 -(3 rows) - -DROP TABLE source_table, target_table; --- --- range partitioning, composite distribution column --- -CREATE TYPE composite_key_type AS (f1 int, f2 text); --- source -CREATE TABLE source_table(f1 int, key composite_key_type, value int, mapped_key composite_key_type); -SELECT create_distributed_table('source_table', 'key', 'range'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CALL public.create_range_partitioned_shards('source_table', '{"(0,a)","(25,a)"}','{"(24,z)","(49,z)"}'); -INSERT INTO source_table VALUES (0, (0, 'a'), 1, (0, 'a')); -INSERT INTO source_table VALUES (1, (1, 'b'), 2, (26, 'b')); -INSERT INTO source_table VALUES (2, (2, 'c'), 3, (3, 'c')); -INSERT INTO source_table VALUES (3, (4, 'd'), 4, (27, 'd')); -INSERT INTO source_table VALUES (4, (30, 'e'), 5, (30, 'e')); -INSERT INTO source_table VALUES (5, (31, 'f'), 6, (31, 'f')); -INSERT INTO source_table VALUES (6, (32, 'g'), 50, (8, 'g')); --- target -CREATE TABLE target_table(f1 int DEFAULT 0, value int, key composite_key_type PRIMARY KEY); -SELECT create_distributed_table('target_table', 'key', 'range'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CALL public.create_range_partitioned_shards('target_table', '{"(0,a)","(25,a)"}','{"(24,z)","(49,z)"}'); -SET client_min_messages TO DEBUG2; -INSERT INTO target_table SELECT f1, value, mapped_key FROM source_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 2 with name 'key' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213591 AS citus_table_alias (f1, value, key) SELECT f1, value, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_0,repartitioned_results_xxxxx_from_4213590_to_0}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, value integer, key insert_select_repartition.composite_key_type) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213592 AS citus_table_alias (f1, value, key) SELECT f1, value, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_1,repartitioned_results_xxxxx_from_4213590_to_1}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, value integer, key insert_select_repartition.composite_key_type) -RESET client_min_messages; -SELECT * FROM target_table ORDER BY key; - f1 | value | key ---------------------------------------------------------------------- - 0 | 1 | (0,a) - 2 | 3 | (3,c) - 6 | 50 | (8,g) - 1 | 2 | (26,b) - 3 | 4 | (27,d) - 4 | 5 | (30,e) - 5 | 6 | (31,f) -(7 rows) - -SELECT * FROM target_table WHERE key = (26, 'b')::composite_key_type; - f1 | value | key ---------------------------------------------------------------------- - 1 | 2 | (26,b) -(1 row) - --- with explicit column names -TRUNCATE target_table; -SET client_min_messages TO DEBUG2; -INSERT INTO target_table(value, key) SELECT value, mapped_key FROM source_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 2 with name 'key' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213591 AS citus_table_alias (f1, value, key) SELECT f1, value, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_0,repartitioned_results_xxxxx_from_4213590_to_0}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, value integer, key insert_select_repartition.composite_key_type) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213592 AS citus_table_alias (f1, value, key) SELECT f1, value, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_1,repartitioned_results_xxxxx_from_4213590_to_1}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, value integer, key insert_select_repartition.composite_key_type) -RESET client_min_messages; -SELECT * FROM target_table ORDER BY key; - f1 | value | key ---------------------------------------------------------------------- - 0 | 1 | (0,a) - 0 | 3 | (3,c) - 0 | 50 | (8,g) - 0 | 2 | (26,b) - 0 | 4 | (27,d) - 0 | 5 | (30,e) - 0 | 6 | (31,f) -(7 rows) - --- missing value for a column -TRUNCATE target_table; -SET client_min_messages TO DEBUG2; -INSERT INTO target_table(key) SELECT mapped_key AS key_renamed FROM source_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 1 with name 'key' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213591 AS citus_table_alias (f1, key) SELECT f1, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_0,repartitioned_results_xxxxx_from_4213590_to_0}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, key insert_select_repartition.composite_key_type) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213592 AS citus_table_alias (f1, key) SELECT f1, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_1,repartitioned_results_xxxxx_from_4213590_to_1}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, key insert_select_repartition.composite_key_type) -RESET client_min_messages; -SELECT * FROM target_table ORDER BY key; - f1 | value | key ---------------------------------------------------------------------- - 0 | | (0,a) - 0 | | (3,c) - 0 | | (8,g) - 0 | | (26,b) - 0 | | (27,d) - 0 | | (30,e) - 0 | | (31,f) -(7 rows) - --- ON CONFLICT -SET client_min_messages TO DEBUG2; -INSERT INTO target_table(key) -SELECT mapped_key AS key_renamed FROM source_table -WHERE (mapped_key).f1 % 2 = 1 -ON CONFLICT (key) DO UPDATE SET f1=1; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 1 with name 'key' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213591 AS citus_table_alias (f1, key) SELECT f1, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_0}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, key insert_select_repartition.composite_key_type) ON CONFLICT(key) DO UPDATE SET f1 = 1 -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213592 AS citus_table_alias (f1, key) SELECT f1, key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213589_to_1,repartitioned_results_xxxxx_from_4213590_to_1}'::text[], 'text'::citus_copy_format) intermediate_result(f1 integer, key insert_select_repartition.composite_key_type) ON CONFLICT(key) DO UPDATE SET f1 = 1 -RESET client_min_messages; -SELECT * FROM target_table ORDER BY key; - f1 | value | key ---------------------------------------------------------------------- - 0 | | (0,a) - 1 | | (3,c) - 0 | | (8,g) - 0 | | (26,b) - 1 | | (27,d) - 0 | | (30,e) - 1 | | (31,f) -(7 rows) - --- missing value for distribution column -INSERT INTO target_table(value) SELECT value FROM source_table; -ERROR: the partition column of table insert_select_repartition.target_table should have a value -DROP TABLE source_table, target_table; --- different column types --- verifies that we add necessary casts, otherwise even shard routing won't --- work correctly and we will see 2 values for the same primary key. -CREATE TABLE target_table(col_1 int primary key, col_2 int); -SELECT create_distributed_table('target_table','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO target_table VALUES (1,2), (2,3), (3,4), (4,5), (5,6); -CREATE TABLE source_table(col_1 numeric, col_2 numeric, col_3 numeric); -SELECT create_distributed_table('source_table','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table VALUES (1,1,1), (3,3,3), (5,5,5); -SET client_min_messages TO DEBUG2; -INSERT INTO target_table -SELECT - col_1, col_2 -FROM - source_table -ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The data type of the target table's partition column should exactly match the data type of the corresponding simple column reference in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'col_1' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213593 AS citus_table_alias (col_1, col_2) SELECT col_1, col_2 FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213597_to_0,repartitioned_results_xxxxx_from_4213600_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer) ON CONFLICT(col_1) DO UPDATE SET col_2 = excluded.col_2 -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213594 AS citus_table_alias (col_1, col_2) SELECT col_1, col_2 FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213599_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer) ON CONFLICT(col_1) DO UPDATE SET col_2 = excluded.col_2 -RESET client_min_messages; -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 1 - 2 | 3 - 3 | 3 - 4 | 5 - 5 | 5 -(5 rows) - -DROP TABLE source_table, target_table; --- --- array coercion --- -SET citus.shard_count TO 3; -CREATE TABLE source_table(a int, mapped_key int, c float[]); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table VALUES (1, -1, ARRAY[1.1, 2.2, 3.3]), (2, -2, ARRAY[4.5, 5.8]), - (3, -3, ARRAY[]::float[]), (4, -4, ARRAY[3.3]); -SET citus.shard_count TO 2; -CREATE TABLE target_table(a int, b int[]); -SELECT create_distributed_table('target_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEBUG1; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - --- --- worker queries can have more columns than necessary. ExpandWorkerTargetEntry() --- might add additional columns to the target list. --- -TRUNCATE target_table; -\set VERBOSITY TERSE --- first verify that the SELECT query below fetches 3 projected columns from workers -SET citus.log_remote_commands TO true; SET client_min_messages TO DEBUG; - CREATE TABLE results AS SELECT max(-a), array_agg(mapped_key) FROM source_table GROUP BY a; -DEBUG: Router planner cannot handle multi-shard select queries -NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213601 source_table WHERE true GROUP BY a -NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213602 source_table WHERE true GROUP BY a -NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213603 source_table WHERE true GROUP BY a -RESET citus.log_remote_commands; RESET client_min_messages; -DROP TABLE results; --- now verify that we don't write the extra columns to the intermediate result files and --- insertion to the target works fine. -SET client_min_messages TO DEBUG1; -INSERT INTO target_table SELECT max(-a), array_agg(mapped_key) FROM source_table GROUP BY a; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {-4} - -3 | {-3} - -2 | {-2} - -1 | {-1} -(4 rows) - --- --- repartitioned INSERT/SELECT followed/preceded by other DML in same transaction --- --- case 1. followed by DELETE -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -DELETE FROM target_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- -(0 rows) - --- case 2. followed by UPDATE -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -UPDATE target_table SET b=array_append(b, a); -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3,-4} - -3 | {-3} - -2 | {4,6,-2} - -1 | {1,2,3,-1} -(4 rows) - --- case 3. followed by multi-row INSERT -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -INSERT INTO target_table VALUES (-5, ARRAY[10,11]), (-6, ARRAY[11,12]), (-7, ARRAY[999]); -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -7 | {999} - -6 | {11,12} - -5 | {10,11} - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(7 rows) - --- case 4. followed by distributed INSERT/SELECT -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -INSERT INTO target_table SELECT * FROM target_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -4 | {3} - -3 | {} - -3 | {} - -2 | {4,6} - -2 | {4,6} - -1 | {1,2,3} - -1 | {1,2,3} -(8 rows) - --- case 5. preceded by DELETE -TRUNCATE target_table; -BEGIN; -DELETE FROM target_table; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - --- case 6. preceded by UPDATE -TRUNCATE target_table; -BEGIN; -UPDATE target_table SET b=array_append(b, a); -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - --- case 7. preceded by multi-row INSERT -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table VALUES (-5, ARRAY[10,11]), (-6, ARRAY[11,12]), (-7, ARRAY[999]); -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -7 | {999} - -6 | {11,12} - -5 | {10,11} - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(7 rows) - --- case 8. preceded by distributed INSERT/SELECT -TRUNCATE target_table; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -BEGIN; -INSERT INTO target_table SELECT * FROM target_table; -INSERT INTO target_table SELECT mapped_key, c FROM source_table; -END; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -4 | {3} - -4 | {3} - -3 | {} - -3 | {} - -3 | {} - -2 | {4,6} - -2 | {4,6} - -2 | {4,6} - -1 | {1,2,3} - -1 | {1,2,3} - -1 | {1,2,3} -(12 rows) - --- --- repartitioned INSERT/SELECT with RETURNING --- -TRUNCATE target_table; -SET client_min_messages TO DEBUG1; -WITH c AS ( - INSERT INTO target_table - SELECT mapped_key, c FROM source_table - RETURNING *) -SELECT * FROM c ORDER by a; -DEBUG: generating subplan XXX_1 for CTE c: INSERT INTO insert_select_repartition.target_table (a, b) SELECT mapped_key, c FROM insert_select_repartition.source_table RETURNING target_table.a, target_table.b -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT a, b FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer[])) c ORDER BY a -DEBUG: performing repartitioned INSERT ... SELECT - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -RESET client_min_messages; --- --- in combination with CTEs --- -TRUNCATE target_table; -SET client_min_messages TO DEBUG1; -WITH t AS ( - SELECT mapped_key, a, c FROM source_table - WHERE a > floor(random()) -) -INSERT INTO target_table -SELECT mapped_key, c FROM t NATURAL JOIN source_table; -DEBUG: volatile functions are not allowed in distributed INSERT ... SELECT queries -DEBUG: generating subplan XXX_1 for CTE t: SELECT mapped_key, a, c FROM insert_select_repartition.source_table WHERE ((a)::double precision OPERATOR(pg_catalog.>) floor(random())) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT mapped_key AS a, auto_coerced_by_citus_1 AS b FROM (SELECT t.mapped_key, (t.c)::integer[] AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.mapped_key, intermediate_result.a, intermediate_result.c FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(mapped_key integer, a integer, c double precision[])) t JOIN insert_select_repartition.source_table USING (mapped_key, a, c))) citus_insert_select_subquery -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - -4 | {3} - -3 | {} - -2 | {4,6} - -1 | {1,2,3} -(4 rows) - -DROP TABLE source_table, target_table; --- --- The case where select query has a GROUP BY ... --- -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int, b int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET citus.shard_count TO 3; -CREATE TABLE target_table(a int, b int); -SELECT create_distributed_table('target_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT floor(i/4), i*i FROM generate_series(1, 20) i; -SET client_min_messages TO DEBUG1; -INSERT INTO target_table SELECT a, max(b) FROM source_table GROUP BY a; -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - 0 | 9 - 1 | 49 - 2 | 121 - 3 | 225 - 4 | 361 - 5 | 400 -(6 rows) - --- --- EXPLAIN output should specify repartitioned INSERT/SELECT --- -EXPLAIN INSERT INTO target_table SELECT a, max(b) FROM source_table GROUP BY a; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) (cost=0.00..0.00 rows=0 width=0) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=100000 width=8) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate (cost=43.90..45.90 rows=200 width=8) - Group Key: a - -> Seq Scan on source_table_4213606 source_table (cost=0.00..32.60 rows=2260 width=8) -(10 rows) - --- --- EXPLAIN ANALYZE is currently not supported --- -EXPLAIN ANALYZE INSERT INTO target_table SELECT a, max(b) FROM source_table GROUP BY a; -ERROR: EXPLAIN ANALYZE is currently not supported for INSERT ... SELECT commands with repartitioning --- --- Duplicate names in target list --- -TRUNCATE target_table; -SET client_min_messages TO DEBUG2; -INSERT INTO target_table - SELECT max(b), max(b) FROM source_table GROUP BY a; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213610 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213606_to_0,repartitioned_results_xxxxx_from_4213607_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213611 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213607_to_1,repartitioned_results_xxxxx_from_4213609_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213612 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213606_to_2,repartitioned_results_xxxxx_from_4213607_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - 9 | 9 - 49 | 49 - 121 | 121 - 225 | 225 - 361 | 361 - 400 | 400 -(6 rows) - --- --- Prepared INSERT/SELECT --- -TRUNCATE target_table; -PREPARE insert_plan(int, int) AS -INSERT INTO target_table - SELECT a, max(b) FROM source_table - WHERE a BETWEEN $1 AND $2 GROUP BY a; -SET client_min_messages TO DEBUG1; -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(0, 2); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan(2, 4); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT a, count(*), count(distinct b) distinct_values FROM target_table GROUP BY a ORDER BY a; - a | count | distinct_values ---------------------------------------------------------------------- - 0 | 6 | 1 - 1 | 6 | 1 - 2 | 12 | 1 - 3 | 6 | 1 - 4 | 6 | 1 -(5 rows) - -DEALLOCATE insert_plan; --- --- Prepared router INSERT/SELECT. We currently use pull to coordinator when the --- distributed query has a single task. --- -TRUNCATE target_table; -PREPARE insert_plan(int) AS -INSERT INTO target_table - SELECT a, max(b) FROM source_table - WHERE a=$1 GROUP BY a; -SET client_min_messages TO DEBUG1; -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -EXECUTE insert_plan(0); -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Collecting INSERT ... SELECT results on coordinator -RESET client_min_messages; -SELECT a, count(*), count(distinct b) distinct_values FROM target_table GROUP BY a ORDER BY a; - a | count | distinct_values ---------------------------------------------------------------------- - 0 | 7 | 1 -(1 row) - -DEALLOCATE insert_plan; --- --- Prepared INSERT/SELECT with no parameters. --- -TRUNCATE target_table; -PREPARE insert_plan AS -INSERT INTO target_table - SELECT a, max(b) FROM source_table - WHERE a BETWEEN 1 AND 2 GROUP BY a; -SELECT public.coordinator_plan($Q$ -EXPLAIN EXECUTE insert_plan; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) (cost=0.00..0.00 rows=0 width=0) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=100000 width=8) - Task Count: 4 -(4 rows) - -SET client_min_messages TO DEBUG1; -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -EXECUTE insert_plan; -DEBUG: performing repartitioned INSERT ... SELECT -RESET client_min_messages; -SELECT a, count(*), count(distinct b) distinct_values FROM target_table GROUP BY a ORDER BY a; - a | count | distinct_values ---------------------------------------------------------------------- - 1 | 7 | 1 - 2 | 7 | 1 -(2 rows) - -DEALLOCATE insert_plan; --- --- INSERT/SELECT in CTE --- -TRUNCATE target_table; -SET client_min_messages TO DEBUG2; -SET citus.enable_non_colocated_router_query_pushdown TO ON; -WITH r AS ( - INSERT INTO target_table SELECT * FROM source_table RETURNING * -) -INSERT INTO target_table SELECT source_table.a, max(source_table.b) FROM source_table NATURAL JOIN r GROUP BY source_table.a; -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: only SELECT, UPDATE, or DELETE common table expressions may be router planned -DEBUG: generating subplan XXX_1 for CTE r: INSERT INTO insert_select_repartition.target_table (a, b) SELECT a, b FROM insert_select_repartition.source_table RETURNING target_table.a, target_table.b -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT a, max AS b FROM (SELECT source_table.a, max(source_table.b) AS max FROM (insert_select_repartition.source_table JOIN (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) r USING (a, b)) GROUP BY source_table.a) citus_insert_select_subquery -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213610 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213606_to_0,repartitioned_results_xxxxx_from_4213607_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) RETURNING citus_table_alias.a, citus_table_alias.b -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213611 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213607_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) RETURNING citus_table_alias.a, citus_table_alias.b -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213612 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213609_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) RETURNING citus_table_alias.a, citus_table_alias.b -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213610 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213606_to_0,repartitioned_results_xxxxx_from_4213607_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213611 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213607_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213612 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213609_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -RESET citus.enable_non_colocated_router_query_pushdown; -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a, b; - a | b ---------------------------------------------------------------------- - 0 | 1 - 0 | 4 - 0 | 9 - 0 | 9 - 1 | 16 - 1 | 25 - 1 | 36 - 1 | 49 - 1 | 49 - 2 | 64 - 2 | 81 - 2 | 100 - 2 | 121 - 2 | 121 - 3 | 144 - 3 | 169 - 3 | 196 - 3 | 225 - 3 | 225 - 4 | 256 - 4 | 289 - 4 | 324 - 4 | 361 - 4 | 361 - 5 | 400 - 5 | 400 -(26 rows) - -DROP TABLE source_table, target_table; --- --- Constraint failure and rollback --- -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int, b int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT i, i * i FROM generate_series(1, 10) i; -UPDATE source_table SET b = NULL where b IN (9, 4); -SET citus.shard_replication_factor TO 2; -CREATE TABLE target_table(a int, b int not null); -SELECT create_distributed_table('target_table', 'a', 'range'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CALL public.create_range_partitioned_shards('target_table', '{0,3,6,9}','{2,5,8,50}'); -INSERT INTO target_table VALUES (11,9), (22,4); -EXPLAIN (costs off) INSERT INTO target_table SELECT * FROM source_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on source_table_4213613 source_table -(8 rows) - -EXPLAIN (costs off) INSERT INTO target_table SELECT * FROM source_table WHERE b IS NOT NULL; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on source_table_4213613 source_table - Filter: (b IS NOT NULL) -(9 rows) - -BEGIN; -SAVEPOINT s1; -INSERT INTO target_table SELECT * FROM source_table; -ERROR: null value in column "b" violates not-null constraint -ROLLBACK TO SAVEPOINT s1; -INSERT INTO target_table SELECT * FROM source_table WHERE b IS NOT NULL; -END; -SELECT * FROM target_table ORDER BY b; - a | b ---------------------------------------------------------------------- - 1 | 1 - 22 | 4 - 11 | 9 - 4 | 16 - 5 | 25 - 6 | 36 - 7 | 49 - 8 | 64 - 9 | 81 - 10 | 100 -(10 rows) - --- verify that values have been replicated to both replicas -SELECT * FROM run_command_on_placements('target_table', 'select count(*) from %s') ORDER BY shardid, nodeport; - nodename | nodeport | shardid | success | result ---------------------------------------------------------------------- - localhost | 57637 | 4213617 | t | 1 - localhost | 57638 | 4213617 | t | 1 - localhost | 57637 | 4213618 | t | 2 - localhost | 57638 | 4213618 | t | 2 - localhost | 57637 | 4213619 | t | 3 - localhost | 57638 | 4213619 | t | 3 - localhost | 57637 | 4213620 | t | 4 - localhost | 57638 | 4213620 | t | 4 -(8 rows) - --- --- Multiple casts in the SELECT query --- -TRUNCATE target_table; -SET client_min_messages TO DEBUG2; -INSERT INTO target_table SELECT 1.12, b::bigint FROM source_table WHERE b IS NOT NULL; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213617 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213613_to_0,repartitioned_results_xxxxx_from_4213614_to_0,repartitioned_results_xxxxx_from_4213615_to_0,repartitioned_results_xxxxx_from_4213616_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -RESET client_min_messages; -SELECT * FROM target_table ORDER BY a, b; - a | b ---------------------------------------------------------------------- - 1 | 1 - 1 | 16 - 1 | 25 - 1 | 36 - 1 | 49 - 1 | 64 - 1 | 81 - 1 | 100 -(8 rows) - --- --- ROLLBACK after out of range error --- -TRUNCATE target_table; -BEGIN; -INSERT INTO target_table SELECT a * 10, b FROM source_table WHERE b IS NOT NULL; -ERROR: could not find shard for partition column value -END; -SELECT max(result) FROM run_command_on_placements('target_table', 'select count(*) from %s'); - max ---------------------------------------------------------------------- - 0 -(1 row) - -DROP TABLE source_table, target_table; --- --- Range partitioned target's ranges doesn't cover the whole range --- -SET citus.shard_replication_factor TO 2; -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int, b int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT i, i * i FROM generate_series(1, 10) i; -SET citus.shard_replication_factor TO 2; -CREATE TABLE target_table(b int not null, a float); -SELECT create_distributed_table('target_table', 'a', 'range'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CALL public.create_range_partitioned_shards('target_table', '{0.0,3.5,6.5,9.5}','{2.9,5.9,8.9,50.0}'); -INSERT INTO target_table SELECT b, a+0.6 FROM source_table; -SELECT * FROM target_table ORDER BY a; - b | a ---------------------------------------------------------------------- - 1 | 1.6 - 4 | 2.6 - 9 | 3.6 - 16 | 4.6 - 25 | 5.6 - 36 | 6.6 - 49 | 7.6 - 64 | 8.6 - 81 | 9.6 - 100 | 10.6 -(10 rows) - --- verify that values have been replicated to both replicas, and that each --- replica has received correct number of rows -SELECT * FROM run_command_on_placements('target_table', 'select count(*) from %s') ORDER BY shardid, nodeport; - nodename | nodeport | shardid | success | result ---------------------------------------------------------------------- - localhost | 57637 | 4213625 | t | 2 - localhost | 57638 | 4213625 | t | 2 - localhost | 57637 | 4213626 | t | 3 - localhost | 57638 | 4213626 | t | 3 - localhost | 57637 | 4213627 | t | 3 - localhost | 57638 | 4213627 | t | 3 - localhost | 57637 | 4213628 | t | 2 - localhost | 57638 | 4213628 | t | 2 -(8 rows) - -DROP TABLE source_table, target_table; --- --- Select column names should be unique --- -SET citus.shard_replication_factor TO 1; -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int, b int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET citus.shard_count TO 3; -CREATE TABLE target_table(a int, b int, c int, d int, e int, f int); -SELECT create_distributed_table('target_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT i, i * i FROM generate_series(1, 10) i; -SET client_min_messages TO DEBUG2; -INSERT INTO target_table SELECT a AS aa, b AS aa, 1 AS aa, 2 AS aa FROM source_table; -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'a' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213633 AS citus_table_alias (a, b, c, d) SELECT a, b, c, d FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213629_to_0,repartitioned_results_xxxxx_from_4213630_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer, c integer, d integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213634 AS citus_table_alias (a, b, c, d) SELECT a, b, c, d FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213630_to_1,repartitioned_results_xxxxx_from_4213631_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer, c integer, d integer) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213635 AS citus_table_alias (a, b, c, d) SELECT a, b, c, d FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213632_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer, c integer, d integer) -RESET client_min_messages; -SELECT count(*) FROM target_table; - count ---------------------------------------------------------------------- - 10 -(1 row) - --- --- Disable repartitioned insert/select --- -TRUNCATE target_table; -SET citus.enable_repartitioned_insert_select TO OFF; -EXPLAIN (costs off) INSERT INTO target_table SELECT a AS aa, b AS aa, 1 AS aa, 2 AS aa FROM source_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on source_table_4213629 source_table -(8 rows) - -SET client_min_messages TO DEBUG2; -INSERT INTO target_table SELECT a AS aa, b AS aa, 1 AS aa, 2 AS aa FROM source_table; -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Collecting INSERT ... SELECT results on coordinator -RESET client_min_messages; -SELECT count(*) FROM target_table; - count ---------------------------------------------------------------------- - 10 -(1 row) - -SET citus.enable_repartitioned_insert_select TO ON; -EXPLAIN (costs off) INSERT INTO target_table SELECT a AS aa, b AS aa, 1 AS aa, 2 AS aa FROM source_table; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on source_table_4213629 source_table -(8 rows) - -DROP TABLE source_table, target_table; --- --- Don't use INSERT/SELECT repartition with repartition joins --- -create table test(x int, y int); -select create_distributed_table('test', 'x'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -set citus.enable_repartition_joins to true; -INSERT INTO test SELECT i, i FROM generate_series(1, 10) i; -EXPLAIN (costs off) INSERT INTO test(y, x) SELECT a.x, b.y FROM test a JOIN test b USING (y); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 3 - Merge Task Count: 6 - -> MapMergeJob - Map Task Count: 3 - Merge Task Count: 6 -(11 rows) - -SET client_min_messages TO DEBUG1; -INSERT INTO test(y, x) SELECT a.x, b.y FROM test a JOIN test b USING (y); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: Collecting INSERT ... SELECT results on coordinator -RESET client_min_messages; -SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 20 -(1 row) - -TRUNCATE test; -INSERT INTO test SELECT i, i FROM generate_series(1, 10) i; -EXPLAIN (costs off) INSERT INTO test SELECT a.* FROM test a JOIN test b USING (y); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 3 - Merge Task Count: 6 - -> MapMergeJob - Map Task Count: 3 - Merge Task Count: 6 -(11 rows) - -SET client_min_messages TO DEBUG1; -INSERT INTO test SELECT a.* FROM test a JOIN test b USING (y); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: Collecting INSERT ... SELECT results on coordinator -RESET client_min_messages; -SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 20 -(1 row) - --- --- In the following case we coerce some columns and move uncoerced versions to the --- end of SELECT list. The following case verifies that we rename those columns so --- we don't get "column reference is ambiguous" errors. --- -CREATE TABLE target_table( - c1 int, - c2 int, - c3 timestamp, - a int, - b int, - c int, - c4 int, - c5 int, - c6 int[], - cardinality int, - sum int, - PRIMARY KEY (c1, c2, c3, c4, c5, c6) -); -SET citus.shard_count TO 5; -SELECT create_distributed_table('target_table', 'c1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE source_table( - c1 int, - c2 int, - c3 date, - c4 int, - cardinality int, - sum int -); -SET citus.shard_count TO 4; -SELECT create_distributed_table('source_table', 'c1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE OR REPLACE FUNCTION dist_func(a int, b int) RETURNS int[] -AS $$ -BEGIN - RETURN array_fill(a, ARRAY[b]); -END; -$$ -LANGUAGE plpgsql STABLE; -SELECT create_distributed_function('dist_func(int, int)'); -NOTICE: procedure insert_select_repartition.dist_func is already distributed - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEBUG; -SET citus.enable_unique_job_ids TO off; -INSERT INTO source_table VALUES (1,2, '2020-02-02', 3, 4, 5); -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 -INSERT INTO source_table VALUES (1,2, '2020-02-02', 3, 4, 5); -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 -INSERT INTO source_table VALUES (3,4, '2020-02-02', 3, 4, 5); -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 3 -INSERT INTO target_table AS enriched(c1, c2, c3, c4, c5, c6, cardinality, sum) -SELECT c1, c2, c3, c4, -1::float AS c5, - dist_func(c1, 4) c6, - sum(cardinality), - sum(sum) -FROM source_table -GROUP BY c1, c2, c3, c4, c6 -ON CONFLICT(c1, c2, c3, c4, c5, c6) -DO UPDATE SET - cardinality = enriched.cardinality + excluded.cardinality, - sum = enriched.sum + excluded.sum; -DEBUG: INSERT target relation and all source relations of the SELECT must be colocated in distributed INSERT ... SELECT -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'c1' -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213639 AS enriched (c1, c2, c3, c4, c5, c6, cardinality, sum) SELECT c1, c2, c3, c4, c5, c6, cardinality, sum FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213644_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(c1 integer, c2 integer, c3 timestamp without time zone, c4 integer, c5 integer, c6 integer[], cardinality integer, sum integer) ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = (enriched.cardinality OPERATOR(pg_catalog.+) excluded.cardinality), sum = (enriched.sum OPERATOR(pg_catalog.+) excluded.sum) -DEBUG: distributed statement: INSERT INTO insert_select_repartition.target_table_4213641 AS enriched (c1, c2, c3, c4, c5, c6, cardinality, sum) SELECT c1, c2, c3, c4, c5, c6, cardinality, sum FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213645_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(c1 integer, c2 integer, c3 timestamp without time zone, c4 integer, c5 integer, c6 integer[], cardinality integer, sum integer) ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = (enriched.cardinality OPERATOR(pg_catalog.+) excluded.cardinality), sum = (enriched.sum OPERATOR(pg_catalog.+) excluded.sum) -RESET client_min_messages; -EXPLAIN (COSTS OFF) INSERT INTO target_table AS enriched(c1, c2, c3, c4, c5, c6, cardinality, sum) -SELECT c1, c2, c3, c4, -1::float AS c5, - dist_func(c1, 4) c6, - sum(cardinality), - sum(sum) -FROM source_table -GROUP BY c1, c2, c3, c4, c6 -ON CONFLICT(c1, c2, c3, c4, c5, c6) -DO UPDATE SET - cardinality = enriched.cardinality + excluded.cardinality, - sum = enriched.sum + excluded.sum; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Group Key: c1, c2, c3, c4, insert_select_repartition.dist_func(c1, 4) - -> Seq Scan on source_table_4213644 source_table -(10 rows) - --- verify that we don't report repartitioned insert/select for tables --- with sequences. See https://github.com/citusdata/citus/issues/3936 -create table table_with_sequences (x int, y int, z bigserial); -insert into table_with_sequences values (1,1); -select create_distributed_table('table_with_sequences','x'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -explain (costs off) insert into table_with_sequences select y, x from table_with_sequences; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on table_with_sequences_4213648 table_with_sequences -(8 rows) - --- verify that we don't report repartitioned insert/select for tables --- with user-defined sequences. -CREATE SEQUENCE user_defined_sequence; -create table table_with_user_sequences (x int, y int, z bigint default nextval('user_defined_sequence')); -insert into table_with_user_sequences values (1,1); -select create_distributed_table('table_with_user_sequences','x'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -explain (costs off) insert into table_with_user_sequences select y, x from table_with_user_sequences; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on table_with_user_sequences_4213652 table_with_user_sequences -(8 rows) - -CREATE TABLE dist_table_1(id int); -SELECT create_distributed_table('dist_table_1','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE dist_table_2(id int); -SELECT create_distributed_table('dist_table_2','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- verify that insert select with union can be repartitioned. We cannot push down the query --- since UNION clause has no FROM clause at top level query. -SELECT public.coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_1(id) SELECT id FROM dist_table_1 UNION SELECT id FROM dist_table_2; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(4 rows) - --- clean-up -SET client_min_messages TO WARNING; -DROP SCHEMA insert_select_repartition CASCADE; diff --git a/src/test/regress/expected/intermediate_result_pruning.out b/src/test/regress/expected/intermediate_result_pruning.out index 5262ebc79..6caeab91b 100644 --- a/src/test/regress/expected/intermediate_result_pruning.out +++ b/src/test/regress/expected/intermediate_result_pruning.out @@ -1,17 +1,6 @@ -- -- INTERMEDIATE_RESULT_PRUNING -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA intermediate_result_pruning; SET search_path TO intermediate_result_pruning; SET citus.log_intermediate_results TO TRUE; diff --git a/src/test/regress/expected/intermediate_result_pruning_0.out b/src/test/regress/expected/intermediate_result_pruning_0.out deleted file mode 100644 index ae1247545..000000000 --- a/src/test/regress/expected/intermediate_result_pruning_0.out +++ /dev/null @@ -1,1077 +0,0 @@ --- --- INTERMEDIATE_RESULT_PRUNING --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA intermediate_result_pruning; -SET search_path TO intermediate_result_pruning; -SET citus.log_intermediate_results TO TRUE; -SET citus.shard_count TO 4; -SET citus.next_shard_id TO 1480000; -SET citus.shard_replication_factor = 1; -CREATE TABLE table_1 (key int, value text); -SELECT create_distributed_table('table_1', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE table_2 (key int, value text); -SELECT create_distributed_table('table_2', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE table_3 (key int, value text); -SELECT create_distributed_table('table_3', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE ref_table (key int, value text); -SELECT create_reference_table('ref_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - --- load some data -INSERT INTO table_1 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); -INSERT INTO table_2 VALUES (3, '3'), (4, '4'), (5, '5'), (6, '6'); -INSERT INTO table_3 VALUES (3, '3'), (4, '4'), (5, '5'), (6, '6'); -INSERT INTO ref_table VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'); --- see which workers are hit for intermediate results -SET client_min_messages TO DEBUG1; --- a very basic case, where the intermediate result --- should go to both workers -WITH some_values_1 AS MATERIALIZED - (SELECT key FROM table_1 WHERE value IN ('3', '4')) -SELECT - count(*) -FROM - some_values_1 JOIN table_2 USING (key); -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 2 -(1 row) - --- a very basic case, where the intermediate result --- should only go to one worker because the final query is a router --- we use random() to prevent postgres inline the CTE(s) -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')) -SELECT - count(*) -FROM - some_values_1 JOIN table_2 USING (key) WHERE table_2.key = 1; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 1) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- a similar query, but with a reference table now --- given that reference tables are replicated to all nodes --- we have to broadcast to all nodes -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')) -SELECT - count(*) -FROM - some_values_1 JOIN ref_table USING (key); -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.ref_table USING (key)) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 2 -(1 row) - --- a similar query as above, but this time use the CTE inside --- another CTE -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1) -SELECT - count(*) -FROM - some_values_2 JOIN table_2 USING (key) WHERE table_2.key = 1; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT key, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 1) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the second CTE does a join with a distributed table --- and the final query is a router query -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key)) -SELECT - count(*) -FROM - some_values_2 JOIN table_2 USING (key) WHERE table_2.key = 3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 3) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 1 -(1 row) - --- the first CTE is used both within second CTE and the final query --- the second CTE does a join with a distributed table --- and the final query is a router query -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key)) -SELECT - count(*) -FROM - (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key) WHERE table_2.key = 3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 3) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 1 -(1 row) - --- the first CTE is used both within second CTE and the final query --- the second CTE does a join with a distributed table but a router query on a worker --- and the final query is another router query on another worker -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE table_2.key = 1) -SELECT - count(*) -FROM - (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key) WHERE table_2.key = 3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 3) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the first CTE is used both within second CTE and the final query --- the second CTE does a join with a distributed table but a router query on a worker --- and the final query is a router query on the same worker, so the first result is only --- broadcasted to a single node -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE table_2.key = 1) -SELECT - count(*) -FROM - (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key) WHERE table_2.key = 1; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 1) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the same query with the above, but the final query is hitting all shards -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key)) -SELECT - count(*) -FROM - (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key) WHERE table_2.key != 3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.<>) 3) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 1 -(1 row) - --- even if we add a filter on the first query and make it a router query, --- the first intermediate result still hits all workers because of the final --- join is hitting all workers -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE table_2.key = 3) -SELECT - count(*) -FROM - (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key) WHERE table_2.key != 3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 3) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.<>) 3) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the reference table is joined with a distributed table and an intermediate --- result, but the distributed table hits all shards, so the intermediate --- result is sent to all nodes -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM ref_table WHERE value IN ('3', '4')) -SELECT - count(*) -FROM - (some_values_1 JOIN ref_table USING (key)) JOIN table_2 USING (key); -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.ref_table WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.ref_table USING (key)) JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 2 -(1 row) - --- similar query as above, but this time the whole query is a router --- query, so no intermediate results -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM ref_table WHERE value IN ('3', '4')) -SELECT - count(*) -FROM - (some_values_1 JOIN ref_table USING (key)) JOIN table_2 USING (key) WHERE table_2.key = 1; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- now, the second CTE has a single shard join with a distributed table --- so the first CTE should only be broadcasted to that node --- since the final query doesn't have a join, it should simply be broadcasted --- to one node -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE key = 1) -SELECT - count(*) -FROM - some_values_2; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the same query inlined inside a CTE, and the final query has a --- join with a distributed table -WITH top_cte as MATERIALIZED ( - WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE key = 1) - SELECT - DISTINCT key - FROM - some_values_2 -) -SELECT - count(*) -FROM - top_cte JOIN table_2 USING (key); -DEBUG: generating subplan XXX_1 for CTE top_cte: WITH some_values_1 AS MATERIALIZED (SELECT table_1.key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (table_1.value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text]))), some_values_2 AS MATERIALIZED (SELECT some_values_1.key, random() AS random FROM (some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1)) SELECT DISTINCT key FROM some_values_2 -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT DISTINCT key FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) top_cte JOIN intermediate_result_pruning.table_2 USING (key)) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- very much the same query, but this time the top query is also a router query --- on a single worker, so all intermediate results only hit a single node -WITH top_cte as MATERIALIZED ( - WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE key = 1) - SELECT - DISTINCT key - FROM - some_values_2 -) -SELECT - count(*) -FROM - top_cte JOIN table_2 USING (key) WHERE table_2.key = 2; -DEBUG: generating subplan XXX_1 for CTE top_cte: WITH some_values_1 AS MATERIALIZED (SELECT table_1.key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (table_1.value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text]))), some_values_2 AS MATERIALIZED (SELECT some_values_1.key, random() AS random FROM (some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1)) SELECT DISTINCT key FROM some_values_2 -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT DISTINCT key FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) top_cte JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (table_2.key OPERATOR(pg_catalog.=) 2) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- some_values_1 is first used by a single shard-query, and than with a multi-shard --- CTE, finally a cartesian product join -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1 JOIN table_2 USING (key) WHERE key = 1), - some_values_3 AS MATERIALIZED - (SELECT key FROM (some_values_2 JOIN table_2 USING (key)) JOIN some_values_1 USING (key)) -SELECT * FROM some_values_3 JOIN ref_table ON (true); -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT some_values_1.key, random() AS random FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 JOIN intermediate_result_pruning.table_2 USING (key)) WHERE (some_values_1.key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_3 for CTE some_values_3: SELECT some_values_2.key FROM (((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN intermediate_result_pruning.table_2 USING (key)) JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT some_values_3.key, ref_table.key, ref_table.value FROM ((SELECT intermediate_result.key FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) some_values_3 JOIN intermediate_result_pruning.ref_table ON (true)) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx - key | key | value ---------------------------------------------------------------------- -(0 rows) - --- join on intermediate results, so should only --- go to a single node -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM table_2 WHERE value IN ('3', '4')) -SELECT count(*) FROM some_values_2 JOIN some_values_1 USING (key); -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT key, random() AS random FROM intermediate_result_pruning.table_2 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 2 -(1 row) - --- same query with WHERE false make sure that we're not broken --- for such edge cases -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM table_2 WHERE value IN ('3', '4')) -SELECT count(*) FROM some_values_2 JOIN some_values_1 USING (key) WHERE false; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_2: SELECT key, random() AS random FROM intermediate_result_pruning.table_2 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_2 JOIN (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 USING (key)) WHERE false -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- do not use some_values_2 at all, so only 2 intermediate results are --- broadcasted -WITH some_values_1 AS MATERIALIZED - (SELECT key, random() FROM table_1 WHERE value IN ('3', '4')), - some_values_2 AS MATERIALIZED - (SELECT key, random() FROM some_values_1), - some_values_3 AS MATERIALIZED - (SELECT key, random() FROM some_values_1) -SELECT - count(*) -FROM - some_values_3; -DEBUG: generating subplan XXX_1 for CTE some_values_1: SELECT key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (value OPERATOR(pg_catalog.=) ANY (ARRAY['3'::text, '4'::text])) -DEBUG: generating subplan XXX_2 for CTE some_values_3: SELECT key, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) some_values_3 -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 2 -(1 row) - --- lets have some deeper intermediate results --- the inner most two results and the final query (which contains only intermediate results) --- hitting single worker, others hitting all workers --- (see below query where all intermediate results hit a single node) -SELECT count(*) FROM -( - SELECT avg(min::int) FROM - ( - SELECT min(table_1.value) FROM - ( - SELECT avg(value::int) as avg_ev_type FROM - ( - SELECT max(value) as mx_val_1 FROM - ( - SELECT avg(value::int) as avg FROM - ( - SELECT cnt FROM - ( - SELECT count(*) as cnt, value - FROM table_1 - WHERE key = 1 - GROUP BY value - ) as level_1, table_1 - WHERE table_1.key = level_1.cnt AND key = 3 - ) as level_2, table_2 - WHERE table_2.key = level_2.cnt AND key = 5 - GROUP BY level_2.cnt - ) as level_3, table_1 - WHERE value::numeric = level_3.avg AND key = 6 - GROUP BY level_3.avg - ) as level_4, table_2 - WHERE level_4.mx_val_1::int = table_2.key - GROUP BY level_4.mx_val_1 - ) as level_5, table_1 - WHERE level_5.avg_ev_type = table_1.key AND key > 111 - GROUP BY level_5.avg_ev_type - ) as level_6, table_1 WHERE table_1.key::int = level_6.min::int - GROUP BY table_1.value -) as bar; -DEBUG: generating subplan XXX_1 for subquery SELECT count(*) AS cnt, value FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) GROUP BY value -DEBUG: generating subplan XXX_2 for subquery SELECT avg((table_2.value)::integer) AS avg FROM (SELECT level_1.cnt FROM (SELECT intermediate_result.cnt, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint, value text)) level_1, intermediate_result_pruning.table_1 WHERE ((table_1.key OPERATOR(pg_catalog.=) level_1.cnt) AND (table_1.key OPERATOR(pg_catalog.=) 3))) level_2, intermediate_result_pruning.table_2 WHERE ((table_2.key OPERATOR(pg_catalog.=) level_2.cnt) AND (table_2.key OPERATOR(pg_catalog.=) 5)) GROUP BY level_2.cnt -DEBUG: generating subplan XXX_3 for subquery SELECT max(table_1.value) AS mx_val_1 FROM (SELECT intermediate_result.avg FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(avg numeric)) level_3, intermediate_result_pruning.table_1 WHERE (((table_1.value)::numeric OPERATOR(pg_catalog.=) level_3.avg) AND (table_1.key OPERATOR(pg_catalog.=) 6)) GROUP BY level_3.avg -DEBUG: generating subplan XXX_4 for subquery SELECT avg((table_2.value)::integer) AS avg_ev_type FROM (SELECT intermediate_result.mx_val_1 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(mx_val_1 text)) level_4, intermediate_result_pruning.table_2 WHERE ((level_4.mx_val_1)::integer OPERATOR(pg_catalog.=) table_2.key) GROUP BY level_4.mx_val_1 -DEBUG: generating subplan XXX_5 for subquery SELECT min(table_1.value) AS min FROM (SELECT intermediate_result.avg_ev_type FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(avg_ev_type numeric)) level_5, intermediate_result_pruning.table_1 WHERE ((level_5.avg_ev_type OPERATOR(pg_catalog.=) (table_1.key)::numeric) AND (table_1.key OPERATOR(pg_catalog.>) 111)) GROUP BY level_5.avg_ev_type -DEBUG: generating subplan XXX_6 for subquery SELECT avg((level_6.min)::integer) AS avg FROM (SELECT intermediate_result.min FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(min text)) level_6, intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) (level_6.min)::integer) GROUP BY table_1.value -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.avg FROM read_intermediate_result('XXX_6'::text, 'binary'::citus_copy_format) intermediate_result(avg numeric)) bar -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_4 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_4 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_5 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_5 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_6 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- the same query where all intermediate results hits one --- worker because each and every query is a router query -- but on different nodes -SELECT count(*) FROM -( - SELECT avg(min::int) FROM - ( - SELECT min(table_1.value) FROM - ( - SELECT avg(value::int) as avg_ev_type FROM - ( - SELECT max(value) as mx_val_1 FROM - ( - SELECT avg(value::int) as avg FROM - ( - SELECT cnt FROM - ( - SELECT count(*) as cnt, value - FROM table_1 - WHERE key = 1 - GROUP BY value - ) as level_1, table_1 - WHERE table_1.key = level_1.cnt AND key = 3 - ) as level_2, table_2 - WHERE table_2.key = level_2.cnt AND key = 5 - GROUP BY level_2.cnt - ) as level_3, table_1 - WHERE value::numeric = level_3.avg AND key = 6 - GROUP BY level_3.avg - ) as level_4, table_2 - WHERE level_4.mx_val_1::int = table_2.key AND table_2.key = 1 - GROUP BY level_4.mx_val_1 - ) as level_5, table_1 - WHERE level_5.avg_ev_type = table_1.key AND key = 111 - GROUP BY level_5.avg_ev_type - ) as level_6, table_1 - WHERE table_1.key::int = level_6.min::int AND table_1.key = 4 - GROUP BY table_1.value -) as bar; -DEBUG: generating subplan XXX_1 for subquery SELECT count(*) AS cnt, value FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) GROUP BY value -DEBUG: generating subplan XXX_2 for subquery SELECT avg((table_2.value)::integer) AS avg FROM (SELECT level_1.cnt FROM (SELECT intermediate_result.cnt, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint, value text)) level_1, intermediate_result_pruning.table_1 WHERE ((table_1.key OPERATOR(pg_catalog.=) level_1.cnt) AND (table_1.key OPERATOR(pg_catalog.=) 3))) level_2, intermediate_result_pruning.table_2 WHERE ((table_2.key OPERATOR(pg_catalog.=) level_2.cnt) AND (table_2.key OPERATOR(pg_catalog.=) 5)) GROUP BY level_2.cnt -DEBUG: generating subplan XXX_3 for subquery SELECT max(table_1.value) AS mx_val_1 FROM (SELECT intermediate_result.avg FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(avg numeric)) level_3, intermediate_result_pruning.table_1 WHERE (((table_1.value)::numeric OPERATOR(pg_catalog.=) level_3.avg) AND (table_1.key OPERATOR(pg_catalog.=) 6)) GROUP BY level_3.avg -DEBUG: generating subplan XXX_4 for subquery SELECT avg((table_2.value)::integer) AS avg_ev_type FROM (SELECT intermediate_result.mx_val_1 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(mx_val_1 text)) level_4, intermediate_result_pruning.table_2 WHERE (((level_4.mx_val_1)::integer OPERATOR(pg_catalog.=) table_2.key) AND (table_2.key OPERATOR(pg_catalog.=) 1)) GROUP BY level_4.mx_val_1 -DEBUG: generating subplan XXX_5 for subquery SELECT min(table_1.value) AS min FROM (SELECT intermediate_result.avg_ev_type FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(avg_ev_type numeric)) level_5, intermediate_result_pruning.table_1 WHERE ((level_5.avg_ev_type OPERATOR(pg_catalog.=) (table_1.key)::numeric) AND (table_1.key OPERATOR(pg_catalog.=) 111)) GROUP BY level_5.avg_ev_type -DEBUG: generating subplan XXX_6 for subquery SELECT avg((level_6.min)::integer) AS avg FROM (SELECT intermediate_result.min FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(min text)) level_6, intermediate_result_pruning.table_1 WHERE ((table_1.key OPERATOR(pg_catalog.=) (level_6.min)::integer) AND (table_1.key OPERATOR(pg_catalog.=) 4)) GROUP BY table_1.value -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.avg FROM read_intermediate_result('XXX_6'::text, 'binary'::citus_copy_format) intermediate_result(avg numeric)) bar -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_4 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_5 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_6 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- sanity checks for set operations --- the intermediate results should just hit a single worker -(SELECT key FROM table_1 WHERE key = 1) -INTERSECT -(SELECT key FROM table_1 WHERE key = 2); -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_2 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer) INTERSECT SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - key ---------------------------------------------------------------------- -(0 rows) - --- the intermediate results should just hit a single worker -WITH cte_1 AS MATERIALIZED -( - (SELECT key FROM table_1 WHERE key = 1) - INTERSECT - (SELECT key FROM table_1 WHERE key = 2) -), -cte_2 AS MATERIALIZED -( - (SELECT key FROM table_1 WHERE key = 3) - INTERSECT - (SELECT key FROM table_1 WHERE key = 4) -) -SELECT * FROM cte_1 - UNION -SELECT * FROM cte_2; -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 1) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 2) -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_2 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer) INTERSECT SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer) -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 3) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 4) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 UNION SELECT cte_2.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - key ---------------------------------------------------------------------- -(0 rows) - --- one final test with SET operations, where --- we join the results with distributed tables --- so cte_1 should hit all workers, but still the --- others should hit single worker each -WITH cte_1 AS MATERIALIZED -( - (SELECT key FROM table_1 WHERE key = 1) - INTERSECT - (SELECT key FROM table_1 WHERE key = 2) -), -cte_2 AS MATERIALIZED -( - SELECT count(*) FROM table_1 JOIN cte_1 USING (key) -) -SELECT * FROM cte_2; -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 1) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 2) -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_2 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer) INTERSECT SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer) -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT count(*) AS count FROM (intermediate_result_pruning.table_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 USING (key)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count FROM (SELECT intermediate_result.count FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) cte_2 -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file - count ---------------------------------------------------------------------- - 0 -(1 row) - --- sanity checks for non-colocated subquery joins --- the recursively planned subquery (bar) should hit all --- nodes -SELECT - count(*) -FROM - (SELECT key, random() FROM table_1) as foo, - (SELECT key, random() FROM table_2) as bar -WHERE - foo.key != bar.key; -DEBUG: generating subplan XXX_1 for subquery SELECT key, random() AS random FROM intermediate_result_pruning.table_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT table_1.key, random() AS random FROM intermediate_result_pruning.table_1) foo, (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) bar WHERE (foo.key OPERATOR(pg_catalog.<>) bar.key) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 14 -(1 row) - --- the recursively planned subquery (bar) should hit one --- node because foo goes to a single node -SELECT - count(*) -FROM - (SELECT key, random() FROM table_1 WHERE key = 1) as foo, - (SELECT key, random() FROM table_2) as bar -WHERE - foo.key != bar.key; -DEBUG: generating subplan XXX_1 for subquery SELECT key, random() AS random FROM intermediate_result_pruning.table_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT table_1.key, random() AS random FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 1)) foo, (SELECT intermediate_result.key, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, random double precision)) bar WHERE (foo.key OPERATOR(pg_catalog.<>) bar.key) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 4 -(1 row) - --- sanity checks for modification queries --- select_data goes to a single node, because it is used in another subquery --- raw_data is also the final router query, so hits a single shard --- however, the subquery in WHERE clause of the DELETE query is broadcasted to all --- nodes -BEGIN; -WITH select_data AS MATERIALIZED ( - SELECT * FROM table_1 -), -raw_data AS MATERIALIZED ( - DELETE FROM table_2 WHERE key >= (SELECT min(key) FROM select_data WHERE key > 1) RETURNING * -) -SELECT * FROM raw_data; -DEBUG: generating subplan XXX_1 for CTE select_data: SELECT key, value FROM intermediate_result_pruning.table_1 -DEBUG: generating subplan XXX_2 for CTE raw_data: DELETE FROM intermediate_result_pruning.table_2 WHERE (key OPERATOR(pg_catalog.>=) (SELECT min(select_data.key) AS min FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) select_data WHERE (select_data.key OPERATOR(pg_catalog.>) 1))) RETURNING key, value -DEBUG: generating subplan XXX_1 for subquery SELECT min(key) AS min FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) select_data WHERE (key OPERATOR(pg_catalog.>) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM intermediate_result_pruning.table_2 WHERE (key OPERATOR(pg_catalog.>=) (SELECT intermediate_result.min FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(min integer))) RETURNING key, value -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) raw_data -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - key | value ---------------------------------------------------------------------- - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 -(4 rows) - -ROLLBACK; --- select_data goes to a single node, because it is used in another subquery --- raw_data is also the final router query, so hits a single shard --- however, the subquery in WHERE clause of the DELETE query is broadcasted to all --- nodes -BEGIN; -WITH select_data AS MATERIALIZED ( - SELECT * FROM table_1 -), -raw_data AS MATERIALIZED ( - DELETE FROM table_2 WHERE value::int >= (SELECT min(key) FROM select_data WHERE key > 1 + random()) RETURNING * -) -SELECT * FROM raw_data; -DEBUG: generating subplan XXX_1 for CTE select_data: SELECT key, value FROM intermediate_result_pruning.table_1 -DEBUG: generating subplan XXX_2 for CTE raw_data: DELETE FROM intermediate_result_pruning.table_2 WHERE ((value)::integer OPERATOR(pg_catalog.>=) (SELECT min(select_data.key) AS min FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) select_data WHERE ((select_data.key)::double precision OPERATOR(pg_catalog.>) ((1)::double precision OPERATOR(pg_catalog.+) random())))) RETURNING key, value -DEBUG: generating subplan XXX_1 for subquery SELECT min(key) AS min FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) select_data WHERE ((key)::double precision OPERATOR(pg_catalog.>) ((1)::double precision OPERATOR(pg_catalog.+) random())) -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM intermediate_result_pruning.table_2 WHERE ((value)::integer OPERATOR(pg_catalog.>=) (SELECT intermediate_result.min FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(min integer))) RETURNING key, value -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) raw_data -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - key | value ---------------------------------------------------------------------- - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 -(4 rows) - -ROLLBACK; --- now, we need only two intermediate results as the subquery in WHERE clause is --- router plannable -BEGIN; -WITH select_data AS MATERIALIZED ( - SELECT * FROM table_1 -), -raw_data AS MATERIALIZED ( - DELETE FROM table_2 WHERE value::int >= (SELECT min(key) FROM table_1 WHERE key > random()) AND key = 6 RETURNING * -) -SELECT * FROM raw_data; -DEBUG: generating subplan XXX_1 for CTE raw_data: DELETE FROM intermediate_result_pruning.table_2 WHERE (((value)::integer OPERATOR(pg_catalog.>=) (SELECT min(table_1.key) AS min FROM intermediate_result_pruning.table_1 WHERE ((table_1.key)::double precision OPERATOR(pg_catalog.>) random()))) AND (key OPERATOR(pg_catalog.=) 6)) RETURNING key, value -DEBUG: generating subplan XXX_1 for subquery SELECT min(key) AS min FROM intermediate_result_pruning.table_1 WHERE ((key)::double precision OPERATOR(pg_catalog.>) random()) -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM intermediate_result_pruning.table_2 WHERE (((value)::integer OPERATOR(pg_catalog.>=) (SELECT intermediate_result.min FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(min integer))) AND (key OPERATOR(pg_catalog.=) 6)) RETURNING key, value -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) raw_data -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - key | value ---------------------------------------------------------------------- - 6 | 6 -(1 row) - -ROLLBACK; --- test with INSERT SELECT via coordinator --- INSERT .. SELECT via coordinator that doesn't have any intermediate results --- We use offset 1 to make sure the result needs to be pulled to the coordinator, offset 0 would be optimized away -INSERT INTO table_1 - SELECT * FROM table_2 OFFSET 1; -DEBUG: cannot push down this subquery -DETAIL: Offset clause is currently unsupported when a subquery references a column from another query -DEBUG: Collecting INSERT ... SELECT results on coordinator --- INSERT .. SELECT via coordinator which has intermediate result, --- and can be pruned to a single worker because the final query is on --- single shard via filter in key -INSERT INTO table_1 - SELECT * FROM table_2 where value IN (SELECT value FROM table_1 WHERE random() > 1) AND key = 1; -DEBUG: volatile functions are not allowed in distributed INSERT ... SELECT queries -DEBUG: generating subplan XXX_1 for subquery SELECT value FROM intermediate_result_pruning.table_1 WHERE (random() OPERATOR(pg_catalog.>) (1)::double precision) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM intermediate_result_pruning.table_2 WHERE ((value OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text))) AND (key OPERATOR(pg_catalog.=) 1)) -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx --- a similar query, with more complex subquery -INSERT INTO table_1 - SELECT * FROM table_2 where key = 1 AND - value::int IN - (WITH cte_1 AS MATERIALIZED - ( - (SELECT key FROM table_1 WHERE key = 1) - INTERSECT - (SELECT key FROM table_1 WHERE key = 2) - ), - cte_2 AS MATERIALIZED - ( - (SELECT key FROM table_1 WHERE key = 3) - INTERSECT - (SELECT key FROM table_1 WHERE key = 4) - ) - SELECT * FROM cte_1 - UNION - SELECT * FROM cte_2); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 1) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 2) -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_2 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer) INTERSECT SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer) -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 3) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 4) -DEBUG: generating subplan XXX_3 for subquery SELECT cte_1.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 UNION SELECT cte_2.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM intermediate_result_pruning.table_2 WHERE ((key OPERATOR(pg_catalog.=) 1) AND ((value)::integer OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.key FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer)))) -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx --- same query, cte is on the FROM clause --- and this time the final query (and top-level intermediate result) --- hits all the shards because table_2.key != 1 -INSERT INTO table_1 - SELECT table_2.* FROM table_2, - (WITH cte_1 AS MATERIALIZED - ( - (SELECT key FROM table_1 WHERE key = 1) - INTERSECT - (SELECT key FROM table_1 WHERE key = 2) - ), - cte_2 AS MATERIALIZED - ( - (SELECT key FROM table_1 WHERE key = 3) - INTERSECT - (SELECT key FROM table_1 WHERE key = 4) - ) - SELECT * FROM cte_1 - UNION - SELECT * FROM cte_2 - ) foo - where table_2.key != 1 AND - foo.key = table_2.value::int; -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 1) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 2) -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: generating subplan XXX_2 for subquery SELECT key FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.=) 2) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer) INTERSECT SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer) -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 3) INTERSECT SELECT table_1.key FROM intermediate_result_pruning.table_1 WHERE (table_1.key OPERATOR(pg_catalog.=) 4) -DEBUG: generating subplan XXX_3 for subquery SELECT cte_1.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 UNION SELECT cte_2.key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table_2.key, table_2.value FROM intermediate_result_pruning.table_2, (SELECT intermediate_result.key FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) foo WHERE ((table_2.key OPERATOR(pg_catalog.<>) 1) AND (foo.key OPERATOR(pg_catalog.=) (table_2.value)::integer)) -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx --- append partitioned/heap-type --- do not print out 'building index pg_toast_xxxxx_index' messages -SET client_min_messages TO DEFAULT; -CREATE TABLE range_partitioned(range_column text, data int); -SET client_min_messages TO DEBUG1; -SELECT create_distributed_table('range_partitioned', 'range_column', 'range'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT master_create_empty_shard('range_partitioned'); - master_create_empty_shard ---------------------------------------------------------------------- - 1480013 -(1 row) - -SELECT master_create_empty_shard('range_partitioned'); - master_create_empty_shard ---------------------------------------------------------------------- - 1480014 -(1 row) - -SELECT master_create_empty_shard('range_partitioned'); - master_create_empty_shard ---------------------------------------------------------------------- - 1480015 -(1 row) - -SELECT master_create_empty_shard('range_partitioned'); - master_create_empty_shard ---------------------------------------------------------------------- - 1480016 -(1 row) - -SELECT master_create_empty_shard('range_partitioned'); - master_create_empty_shard ---------------------------------------------------------------------- - 1480017 -(1 row) - -UPDATE pg_dist_shard SET shardminvalue = 'A', shardmaxvalue = 'D' WHERE shardid = 1480013; -UPDATE pg_dist_shard SET shardminvalue = 'D', shardmaxvalue = 'G' WHERE shardid = 1480014; -UPDATE pg_dist_shard SET shardminvalue = 'G', shardmaxvalue = 'K' WHERE shardid = 1480015; -UPDATE pg_dist_shard SET shardminvalue = 'K', shardmaxvalue = 'O' WHERE shardid = 1480016; -UPDATE pg_dist_shard SET shardminvalue = 'O', shardmaxvalue = 'Z' WHERE shardid = 1480017; --- final query goes to a single shard -SELECT - count(*) -FROM - range_partitioned -WHERE - range_column = 'A' AND - data IN (SELECT data FROM range_partitioned); -DEBUG: generating subplan XXX_1 for subquery SELECT data FROM intermediate_result_pruning.range_partitioned -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM intermediate_result_pruning.range_partitioned WHERE ((range_column OPERATOR(pg_catalog.=) 'A'::text) AND (data OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.data FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(data integer)))) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- final query goes to three shards, so multiple workers -SELECT - count(*) -FROM - range_partitioned -WHERE - range_column >= 'A' AND range_column <= 'K' AND - data IN (SELECT data FROM range_partitioned); -DEBUG: generating subplan XXX_1 for subquery SELECT data FROM intermediate_result_pruning.range_partitioned -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM intermediate_result_pruning.range_partitioned WHERE ((range_column OPERATOR(pg_catalog.>=) 'A'::text) AND (range_column OPERATOR(pg_catalog.<=) 'K'::text) AND (data OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.data FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(data integer)))) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- two shards, both of which are on the first node -WITH some_data AS ( - SELECT data FROM range_partitioned -) -SELECT - count(*) -FROM - range_partitioned -WHERE - range_column IN ('A', 'E') AND - range_partitioned.data IN (SELECT data FROM some_data); -DEBUG: CTE some_data is going to be inlined via distributed planning -DEBUG: generating subplan XXX_1 for subquery SELECT data FROM (SELECT range_partitioned.data FROM intermediate_result_pruning.range_partitioned) some_data -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM intermediate_result_pruning.range_partitioned WHERE ((range_column OPERATOR(pg_catalog.=) ANY (ARRAY['A'::text, 'E'::text])) AND (data OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.data FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(data integer)))) -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - --- test case for issue #3556 -CREATE TABLE accounts (id text PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "accounts_pkey" for table "accounts" -CREATE TABLE stats (account_id text PRIMARY KEY, spent int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "stats_pkey" for table "stats" -SELECT create_distributed_table('accounts', 'id', colocate_with => 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('stats', 'account_id', colocate_with => 'accounts'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO accounts (id) VALUES ('foo'); -INSERT INTO stats (account_id, spent) VALUES ('foo', 100); -SELECT * -FROM -( - WITH accounts_cte AS MATERIALIZED ( - SELECT id AS account_id - FROM accounts - ), - joined_stats_cte_1 AS MATERIALIZED ( - SELECT spent, account_id - FROM stats - INNER JOIN accounts_cte USING (account_id) - ), - joined_stats_cte_2 AS MATERIALIZED ( - SELECT spent, account_id - FROM joined_stats_cte_1 - INNER JOIN accounts_cte USING (account_id) - ) - SELECT SUM(spent) OVER (PARTITION BY coalesce(account_id, NULL)) - FROM accounts_cte - INNER JOIN joined_stats_cte_2 USING (account_id) -) inner_query; -DEBUG: generating subplan XXX_1 for CTE accounts_cte: SELECT id AS account_id FROM intermediate_result_pruning.accounts -DEBUG: generating subplan XXX_2 for CTE joined_stats_cte_1: SELECT stats.spent, stats.account_id FROM (intermediate_result_pruning.stats JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte USING (account_id)) -DEBUG: generating subplan XXX_3 for CTE joined_stats_cte_2: SELECT joined_stats_cte_1.spent, joined_stats_cte_1.account_id FROM ((SELECT intermediate_result.spent, intermediate_result.account_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(spent integer, account_id text)) joined_stats_cte_1 JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte USING (account_id)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT sum FROM (SELECT sum(joined_stats_cte_2.spent) OVER (PARTITION BY COALESCE(accounts_cte.account_id, NULL::text)) AS sum FROM ((SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte JOIN (SELECT intermediate_result.spent, intermediate_result.account_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(spent integer, account_id text)) joined_stats_cte_2 USING (account_id))) inner_query -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Subplan XXX_3 will be written to local file - sum ---------------------------------------------------------------------- - 100 -(1 row) - --- confirm that the pruning works well when using round-robin as well -SET citus.task_assignment_policy to 'round-robin'; -SELECT * -FROM -( - WITH accounts_cte AS MATERIALIZED ( - SELECT id AS account_id - FROM accounts - ), - joined_stats_cte_1 AS MATERIALIZED ( - SELECT spent, account_id - FROM stats - INNER JOIN accounts_cte USING (account_id) - ), - joined_stats_cte_2 AS MATERIALIZED ( - SELECT spent, account_id - FROM joined_stats_cte_1 - INNER JOIN accounts_cte USING (account_id) - ) - SELECT SUM(spent) OVER (PARTITION BY coalesce(account_id, NULL)) - FROM accounts_cte - INNER JOIN joined_stats_cte_2 USING (account_id) -) inner_query; -DEBUG: generating subplan XXX_1 for CTE accounts_cte: SELECT id AS account_id FROM intermediate_result_pruning.accounts -DEBUG: generating subplan XXX_2 for CTE joined_stats_cte_1: SELECT stats.spent, stats.account_id FROM (intermediate_result_pruning.stats JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte USING (account_id)) -DEBUG: generating subplan XXX_3 for CTE joined_stats_cte_2: SELECT joined_stats_cte_1.spent, joined_stats_cte_1.account_id FROM ((SELECT intermediate_result.spent, intermediate_result.account_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(spent integer, account_id text)) joined_stats_cte_1 JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte USING (account_id)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT sum FROM (SELECT sum(joined_stats_cte_2.spent) OVER (PARTITION BY COALESCE(accounts_cte.account_id, NULL::text)) AS sum FROM ((SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte JOIN (SELECT intermediate_result.spent, intermediate_result.account_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(spent integer, account_id text)) joined_stats_cte_2 USING (account_id))) inner_query -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_3 will be sent to localhost:xxxxx - sum ---------------------------------------------------------------------- - 100 -(1 row) - -RESET citus.task_assignment_policy; --- Insert..select is planned differently, make sure we have results everywhere. --- We put the insert..select in a CTE here to prevent the CTE from being moved --- into the select, which would follow the regular code path for select. -WITH stats AS MATERIALIZED ( - SELECT count(key) m FROM table_3 -), -inserts AS MATERIALIZED ( - INSERT INTO table_2 - SELECT key, count(*) - FROM table_1 - WHERE key > (SELECT m FROM stats) - GROUP BY key - HAVING count(*) < (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM intermediate_result_pruning.table_3 -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO intermediate_result_pruning.table_2 (key, value) SELECT key, count(*) AS count FROM intermediate_result_pruning.table_1 WHERE (key OPERATOR(pg_catalog.>) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2.key, table_2.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator - count ---------------------------------------------------------------------- - 1 -(1 row) - -SET citus.task_assignment_policy to DEFAULT; -SET client_min_messages TO DEFAULT; -DROP TABLE table_1, table_2, table_3, ref_table, accounts, stats, range_partitioned; -DROP SCHEMA intermediate_result_pruning; diff --git a/src/test/regress/expected/issue_5248.out b/src/test/regress/expected/issue_5248.out index db1ae26c7..d5946089f 100644 --- a/src/test/regress/expected/issue_5248.out +++ b/src/test/regress/expected/issue_5248.out @@ -1,19 +1,11 @@ -- -- ISSUE_5248 -- --- This test file has an alternative output because of the change in the --- backup modes of Postgres. Specifically, there is a renaming --- issue: pg_stop_backup PRE PG15 vs pg_backup_stop PG15+ --- The alternative output can be deleted when we drop support for PG14 --- CREATE SCHEMA issue_5248; SET search_path TO issue_5248; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; SET citus.next_shard_id TO 3013000; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset create table countries( id serial primary key , name text @@ -219,12 +211,8 @@ FROM ( ( SELECT utc_offset FROM pg_catalog.pg_timezone_names limit 1 offset 4) limit 91) AS subq_3 -\if :server_version_ge_15 WHERE pg_catalog.pg_backup_stop() > cast(NULL AS record) limit 100; ERROR: cannot push down subquery on the target list DETAIL: Subqueries in the SELECT part of the query can only be pushed down if they happen before aggregates and window functions -\else -WHERE pg_catalog.pg_stop_backup() > cast(NULL AS pg_lsn) limit 100; -\endif SET client_min_messages TO WARNING; DROP SCHEMA issue_5248 CASCADE; diff --git a/src/test/regress/expected/issue_5248_0.out b/src/test/regress/expected/issue_5248_0.out deleted file mode 100644 index d7fe8020c..000000000 --- a/src/test/regress/expected/issue_5248_0.out +++ /dev/null @@ -1,230 +0,0 @@ --- --- ISSUE_5248 --- --- This test file has an alternative output because of the change in the --- backup modes of Postgres. Specifically, there is a renaming --- issue: pg_stop_backup PRE PG15 vs pg_backup_stop PG15+ --- The alternative output can be deleted when we drop support for PG14 --- -CREATE SCHEMA issue_5248; -SET search_path TO issue_5248; -SET citus.shard_count TO 4; -SET citus.shard_replication_factor TO 1; -SET citus.next_shard_id TO 3013000; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -create table countries( - id serial primary key - , name text - , code varchar(2) collate "C" unique -); -insert into countries(name, code) select 'country-'||i, i::text from generate_series(10,99) i; -select create_reference_table('countries'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$issue_5248.countries$$) - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -create table orgs ( - id bigserial primary key - , name text - , created_at timestamptz default now() -); -select create_distributed_table('orgs', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -create table users ( - id bigserial - , org_id bigint references orgs(id) - , name text - , created_at timestamptz default now() - , country_id int -- references countries(id) - , score bigint generated always as (id + country_id) stored - , primary key (org_id, id) -); -select create_distributed_table('users', 'org_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -alter table users add constraint fk_user_country foreign key (country_id) references countries(id); -create table orders ( - id bigserial - , org_id bigint references orgs(id) - , user_id bigint - , price int - , info jsonb - , primary key (org_id, id) - , foreign key (org_id, user_id) references users(org_id, id) -); -select create_distributed_table('orders', 'org_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -create table events ( - id bigserial not null - , user_id bigint not null - , org_id bigint not null - , event_time timestamp not null default now() - , event_type int not null default 0 - , payload jsonb - , primary key (user_id, id) -); -create index event_time_idx on events using BRIN (event_time); -create index event_json_idx on events using gin(payload); -select create_distributed_table('events', 'user_id'); -- on purpose don't colocate on correctly on org_id - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -create table local_data( - id bigserial primary key - , val int default ( (random()*100)::int ) -); -insert into orgs(id, name) select i,'org-'||i from generate_series(1,10) i; -insert into users(id, name, org_id, country_id) select i,'user-'||i, i+1, (i%90)+1 from generate_series(1,5) i; -insert into orders(id, org_id, user_id, price) select i, ((i+1))+1 , i+1, i/100 from generate_series(1,2) i; -insert into events(id, org_id, user_id, event_type) select i, ((i+1))+1 , i+1, i/100 from generate_series(1,10) i; -insert into local_data(id) select generate_series(1,10); -/* - * Test that we don't get a crash. See #5248. - */ -SELECT subq_3.c15 AS c0, - subq_3.c0 AS c1, - subq_3.c15 AS c2, - subq_0.c1 AS c3, - pg_catalog.String_agg( Cast( - ( - SELECT tgargs - FROM pg_catalog.pg_trigger limit 1 offset 1) AS BYTEA), Cast( - ( - SELECT minimum_value - FROM columnar.chunk limit 1 offset 5) AS BYTEA)) OVER (partition BY subq_3.c10 ORDER BY subq_3.c12,subq_0.c2) AS c4, - subq_0.c1 AS c5 -FROM ( - SELECT ref_1.address AS c0, - ref_1.error AS c1, - sample_0.NAME AS c2, - sample_2.trftosql AS c3 - FROM pg_catalog.pg_statio_all_sequences AS ref_0 - INNER JOIN pg_catalog.pg_hba_file_rules AS ref_1 - ON (( - SELECT pg_catalog.Max(aggnumdirectargs) - FROM pg_catalog.pg_aggregate) <= ref_0.blks_hit) - INNER JOIN countries AS sample_0 TABLESAMPLE system (6.4) - INNER JOIN local_data AS sample_1 TABLESAMPLE bernoulli (8) - ON (( - true) - OR ( - sample_0.NAME IS NOT NULL)) - INNER JOIN pg_catalog.pg_transform AS sample_2 TABLESAMPLE bernoulli (1.2) - INNER JOIN pg_catalog.pg_language AS ref_2 - ON (( - SELECT shard_cost_function - FROM pg_catalog.pg_dist_rebalance_strategy limit 1 offset 1) IS NULL) - RIGHT JOIN pg_catalog.pg_index AS sample_3 TABLESAMPLE system (0.3) - ON (( - cast(NULL AS bpchar) ~<=~ cast(NULL AS bpchar)) - OR (( - EXISTS - ( - SELECT sample_3.indnkeyatts AS c0, - sample_2.trflang AS c1, - sample_2.trftype AS c2 - FROM pg_catalog.pg_statistic_ext AS sample_4 TABLESAMPLE bernoulli (8.6) - WHERE sample_2.trftype IS NOT NULL)) - AND ( - false))) - ON ( - EXISTS - ( - SELECT sample_0.id AS c0, - sample_3.indisprimary AS c1 - FROM orgs AS sample_5 TABLESAMPLE system (5.3) - WHERE false)) - ON ( - cast(NULL AS float8) > - ( - SELECT pg_catalog.avg(enumsortorder) - FROM pg_catalog.pg_enum) ) - WHERE cast(COALESCE( - CASE - WHEN ref_1.auth_method ~>=~ ref_1.auth_method THEN cast(NULL AS path) - ELSE cast(NULL AS path) - END , cast(NULL AS path)) AS path) = cast(NULL AS path)) AS subq_0, - lateral - ( - SELECT - ( - SELECT pg_catalog.stddev(total_time) - FROM pg_catalog.pg_stat_user_functions) AS c0, - subq_0.c1 AS c1, - subq_2.c0 AS c2, - subq_0.c2 AS c3, - subq_0.c0 AS c4, - cast(COALESCE(subq_2.c0, subq_2.c0) AS text) AS c5, - subq_2.c0 AS c6, - subq_2.c1 AS c7, - subq_2.c1 AS c8, - subq_2.c1 AS c9, - subq_0.c3 AS c10, - pg_catalog.pg_stat_get_db_temp_files( cast( - ( - SELECT objoid - FROM pg_catalog.pg_description limit 1 offset 1) AS oid)) AS c11, - subq_0.c3 AS c12, - subq_2.c1 AS c13, - subq_0.c0 AS c14, - subq_0.c3 AS c15, - subq_0.c3 AS c16, - subq_0.c1 AS c17, - subq_0.c2 AS c18 - FROM ( - SELECT subq_1.c2 AS c0, - subq_0.c3 AS c1 - FROM information_schema.element_types AS ref_3, - lateral - ( - SELECT subq_0.c1 AS c0, - sample_6.info AS c1, - subq_0.c2 AS c2, - subq_0.c3 AS c3, - sample_6.user_id AS c5, - ref_3.collation_name AS c6 - FROM orders AS sample_6 TABLESAMPLE system (3.8) - WHERE sample_6.price = sample_6.org_id limit 58) AS subq_1 - WHERE ( - subq_1.c2 <= subq_0.c2) - AND ( - cast(NULL AS line) ?-| cast(NULL AS line)) limit 59) AS subq_2 - WHERE cast(COALESCE(pg_catalog.age( cast( - ( - SELECT pg_catalog.max(event_time) - FROM events) AS "timestamp")), - ( - SELECT write_lag - FROM pg_catalog.pg_stat_replication limit 1 offset 3) ) AS "interval") > - ( - SELECT utc_offset - FROM pg_catalog.pg_timezone_names limit 1 offset 4) limit 91) AS subq_3 -\if :server_version_ge_15 -WHERE pg_catalog.pg_backup_stop() > cast(NULL AS record) limit 100; -\else -WHERE pg_catalog.pg_stop_backup() > cast(NULL AS pg_lsn) limit 100; -ERROR: cannot push down subquery on the target list -DETAIL: Subqueries in the SELECT part of the query can only be pushed down if they happen before aggregates and window functions -\endif -SET client_min_messages TO WARNING; -DROP SCHEMA issue_5248 CASCADE; diff --git a/src/test/regress/expected/local_shard_execution.out b/src/test/regress/expected/local_shard_execution.out index 58293a2d6..2b1fa3c0b 100644 --- a/src/test/regress/expected/local_shard_execution.out +++ b/src/test/regress/expected/local_shard_execution.out @@ -1,17 +1,6 @@ -- -- LOCAL_SHARD_EXECUTION -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA local_shard_execution; SET search_path TO local_shard_execution; SET citus.shard_count TO 4; diff --git a/src/test/regress/expected/local_shard_execution_0.out b/src/test/regress/expected/local_shard_execution_0.out deleted file mode 100644 index 948941aad..000000000 --- a/src/test/regress/expected/local_shard_execution_0.out +++ /dev/null @@ -1,3302 +0,0 @@ --- --- LOCAL_SHARD_EXECUTION --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA local_shard_execution; -SET search_path TO local_shard_execution; -SET citus.shard_count TO 4; -SET citus.shard_replication_factor TO 1; -SET citus.next_shard_id TO 1470000; -CREATE TABLE reference_table (key int PRIMARY KEY); -SELECT create_reference_table('reference_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE distributed_table (key int PRIMARY KEY , value text, age bigint CHECK (age > 10), FOREIGN KEY (key) REFERENCES reference_table(key) ON DELETE CASCADE); -SELECT create_distributed_table('distributed_table','key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE second_distributed_table (key int PRIMARY KEY , value text, FOREIGN KEY (key) REFERENCES distributed_table(key) ON DELETE CASCADE); -SELECT create_distributed_table('second_distributed_table','key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- ingest some data to enable some tests with data -INSERT INTO reference_table VALUES (1); -INSERT INTO distributed_table VALUES (1, '1', 20); -INSERT INTO second_distributed_table VALUES (1, '1'); --- a simple test for -CREATE TABLE collections_list ( - key bigserial, - ser bigserial, - ts timestamptz, - collection_id integer, - value numeric, - PRIMARY KEY(key, collection_id) -) PARTITION BY LIST (collection_id ); -SELECT create_distributed_table('collections_list', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE collections_list_0 - PARTITION OF collections_list (key, ser, ts, collection_id, value) - FOR VALUES IN ( 0 ); --- create a volatile function that returns the local node id -CREATE OR REPLACE FUNCTION get_local_node_id_volatile() -RETURNS INT AS $$ -DECLARE localGroupId int; -BEGIN - SELECT groupid INTO localGroupId FROM pg_dist_local_group; - RETURN localGroupId; -END; $$ language plpgsql VOLATILE; -SELECT create_distributed_function('get_local_node_id_volatile()'); -NOTICE: procedure local_shard_execution.get_local_node_id_volatile is already distributed -DETAIL: Citus distributes procedures with CREATE [PROCEDURE|FUNCTION|AGGREGATE] commands - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - --- test case for issue #3556 -CREATE TABLE accounts (id text PRIMARY KEY); -CREATE TABLE stats (account_id text PRIMARY KEY, spent int); -SELECT create_distributed_table('accounts', 'id', colocate_with => 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('stats', 'account_id', colocate_with => 'accounts'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO accounts (id) VALUES ('foo'); -INSERT INTO stats (account_id, spent) VALUES ('foo', 100); -CREATE TABLE abcd(a int, b int, c int, d int); -SELECT create_distributed_table('abcd', 'b'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO abcd VALUES (1,2,3,4); -INSERT INTO abcd VALUES (2,3,4,5); -INSERT INTO abcd VALUES (3,4,5,6); -ALTER TABLE abcd DROP COLUMN a; --- connection worker and get ready for the tests -\c - - - :worker_1_port -SET search_path TO local_shard_execution; -SET citus.enable_unique_job_ids TO off; --- returns true of the distribution key filter --- on the distributed tables (e.g., WHERE key = 1), we'll hit a shard --- placement which is local to this not -SET citus.enable_metadata_sync TO OFF; -CREATE OR REPLACE FUNCTION shard_of_distribution_column_is_local(dist_key int) RETURNS bool AS $$ - - DECLARE shard_is_local BOOLEAN := FALSE; - - BEGIN - - WITH local_shard_ids AS (SELECT get_shard_id_for_distribution_column('local_shard_execution.distributed_table', dist_key)), - all_local_shard_ids_on_node AS (SELECT shardid FROM pg_dist_placement WHERE groupid IN (SELECT groupid FROM pg_dist_local_group)) - SELECT - true INTO shard_is_local - FROM - local_shard_ids - WHERE - get_shard_id_for_distribution_column IN (SELECT * FROM all_local_shard_ids_on_node); - - IF shard_is_local IS NULL THEN - shard_is_local = FALSE; - END IF; - - RETURN shard_is_local; - END; -$$ LANGUAGE plpgsql; -RESET citus.enable_metadata_sync; --- test case for issue #3556 -SET citus.log_intermediate_results TO TRUE; -SET client_min_messages TO DEBUG1; -SELECT * -FROM -( - WITH accounts_cte AS ( - SELECT id AS account_id - FROM accounts - ), - joined_stats_cte_1 AS ( - SELECT spent, account_id - FROM stats - INNER JOIN accounts_cte USING (account_id) - ), - joined_stats_cte_2 AS ( - SELECT spent, account_id - FROM joined_stats_cte_1 - INNER JOIN accounts_cte USING (account_id) - ) - SELECT SUM(spent) OVER (PARTITION BY coalesce(account_id, NULL)) - FROM accounts_cte - INNER JOIN joined_stats_cte_2 USING (account_id) -) inner_query; -DEBUG: CTE joined_stats_cte_1 is going to be inlined via distributed planning -DEBUG: CTE joined_stats_cte_2 is going to be inlined via distributed planning -DEBUG: generating subplan XXX_1 for CTE accounts_cte: SELECT id AS account_id FROM local_shard_execution.accounts -DEBUG: generating subplan XXX_2 for subquery SELECT sum(joined_stats_cte_2.spent) OVER (PARTITION BY COALESCE(accounts_cte.account_id, NULL::text)) AS sum FROM ((SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte JOIN (SELECT joined_stats_cte_1.spent, joined_stats_cte_1.account_id FROM ((SELECT stats.spent, stats.account_id FROM (local_shard_execution.stats JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte_2 USING (account_id))) joined_stats_cte_1 JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte_1 USING (account_id))) joined_stats_cte_2 USING (account_id)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT sum FROM (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint)) inner_query -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file - sum ---------------------------------------------------------------------- - 100 -(1 row) - -SET citus.log_intermediate_results TO DEFAULT; -SET client_min_messages TO DEFAULT; --- pick some example values that reside on the shards locally and remote --- distribution key values of 1,6, 500 and 701 are LOCAL to shards, --- we'll use these values in the tests -SELECT shard_of_distribution_column_is_local(1); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - t -(1 row) - -SELECT shard_of_distribution_column_is_local(6); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - t -(1 row) - -SELECT shard_of_distribution_column_is_local(500); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - t -(1 row) - -SELECT shard_of_distribution_column_is_local(701); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - t -(1 row) - --- distribution key values of 11 and 12 are REMOTE to shards -SELECT shard_of_distribution_column_is_local(11); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - f -(1 row) - -SELECT shard_of_distribution_column_is_local(12); - shard_of_distribution_column_is_local ---------------------------------------------------------------------- - f -(1 row) - ---- enable logging to see which tasks are executed locally -SET citus.log_local_commands TO ON; --- first, make sure that local execution works fine --- with simple queries that are not in transcation blocks -SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - --- multiple tasks both of which are local should NOT use local execution --- because local execution means executing the tasks locally, so the executor --- favors parallel execution even if everyting is local to node -SELECT count(*) FROM distributed_table WHERE key IN (1,6); - count ---------------------------------------------------------------------- - 1 -(1 row) - --- queries that hit any remote shards should NOT use local execution -SELECT count(*) FROM distributed_table WHERE key IN (1,11); - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM distributed_table; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- modifications also follow the same rules -INSERT INTO reference_table VALUES (1) ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 AS citus_table_alias (key) VALUES (1) ON CONFLICT DO NOTHING -INSERT INTO distributed_table VALUES (1, '1', 21) ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '1'::text, 21) ON CONFLICT DO NOTHING --- local query -DELETE FROM distributed_table WHERE key = 1 AND age = 21; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((key OPERATOR(pg_catalog.=) 1) AND (age OPERATOR(pg_catalog.=) 21)) --- hitting multiple shards, so should be a distributed execution -DELETE FROM distributed_table WHERE age = 21; --- although both shards are local, the executor choose the parallel execution --- over the wire because as noted above local execution is sequential -DELETE FROM second_distributed_table WHERE key IN (1,6); --- similarly, any multi-shard query just follows distributed execution -DELETE FROM second_distributed_table; --- load some more data for the following tests -INSERT INTO second_distributed_table VALUES (1, '1'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.second_distributed_table_1470005 (key, value) VALUES (1, '1'::text) --- INSERT .. SELECT hitting a single single (co-located) shard(s) should --- be executed locally -INSERT INTO distributed_table -SELECT - distributed_table.* -FROM - distributed_table, second_distributed_table -WHERE - distributed_table.key = 1 and distributed_table.key=second_distributed_table.key -ON CONFLICT(key) DO UPDATE SET value = '22' -RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution.distributed_table_1470001 distributed_table, local_shard_execution.second_distributed_table_1470005 second_distributed_table WHERE (((distributed_table.key OPERATOR(pg_catalog.=) 1) AND (distributed_table.key OPERATOR(pg_catalog.=) second_distributed_table.key)) AND (distributed_table.key IS NOT NULL)) ON CONFLICT(key) DO UPDATE SET value = '22'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - --- INSERT .. SELECT hitting multi-shards should go thourgh distributed execution -INSERT INTO distributed_table -SELECT - distributed_table.* -FROM - distributed_table, second_distributed_table -WHERE - distributed_table.key != 1 and distributed_table.key=second_distributed_table.key -ON CONFLICT(key) DO UPDATE SET value = '22' -RETURNING *; - key | value | age ---------------------------------------------------------------------- -(0 rows) - --- INSERT..SELECT via coordinator consists of two steps, select + COPY --- that's why it is disallowed to use local execution even if the SELECT --- can be executed locally -INSERT INTO distributed_table SELECT sum(key), value FROM distributed_table WHERE key = 1 GROUP BY value ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: SELECT int4(sum(key)) AS key, value FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) GROUP BY value -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_result('insert_select_XXX_1470001'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text) ON CONFLICT DO NOTHING -INSERT INTO distributed_table SELECT 1, '1',15 FROM distributed_table WHERE key = 2 LIMIT 1 ON CONFLICT DO NOTHING; --- sanity check: multi-shard INSERT..SELECT pushdown goes through distributed execution -INSERT INTO distributed_table SELECT * FROM distributed_table ON CONFLICT DO NOTHING; --- Ensure tuple data in explain analyze output is the same on all PG versions -SET citus.enable_binary_protocol = TRUE; --- EXPLAIN for local execution just works fine --- though going through distributed execution -EXPLAIN (COSTS OFF) SELECT * FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Index Scan using distributed_table_pkey_1470001 on distributed_table_1470001 distributed_table - Index Cond: (key = 1) - Filter: (age = 20) -(8 rows) - -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - Task Count: 1 - Tuple data received from nodes: 14 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 14 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Index Scan using distributed_table_pkey_1470001 on distributed_table_1470001 distributed_table (actual rows=1 loops=1) - Index Cond: (key = 1) - Filter: (age = 20) -(10 rows) - -EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF) -WITH r AS ( SELECT GREATEST(random(), 2) z,* FROM distributed_table) -SELECT 1 FROM r WHERE z < 3; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - -> Distributed Subplan XXX_1 - Intermediate Data Size: 40 bytes - Result destination: Write locally - -> Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - Task Count: 4 - Tuple data received from nodes: 22 bytes - Tasks Shown: One of 4 - -> Task - Tuple data received from node: 22 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on distributed_table_1470001 distributed_table (actual rows=1 loops=1) - Task Count: 1 - Tuple data received from nodes: 4 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 4 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) - Filter: (z < '3'::double precision) -(20 rows) - -EXPLAIN (COSTS OFF) DELETE FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Delete on distributed_table_1470001 distributed_table - -> Index Scan using distributed_table_pkey_1470001 on distributed_table_1470001 distributed_table - Index Cond: (key = 1) - Filter: (age = 20) -(9 rows) - -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) DELETE FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=0 loops=1) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Delete on distributed_table_1470001 distributed_table (actual rows=0 loops=1) - -> Index Scan using distributed_table_pkey_1470001 on distributed_table_1470001 distributed_table (actual rows=1 loops=1) - Index Cond: (key = 1) - Filter: (age = 20) - Trigger for constraint second_distributed_table_key_fkey_1470005: calls=1 -(10 rows) - --- show that EXPLAIN ANALYZE deleted the row and cascades deletes -SELECT * FROM distributed_table WHERE key = 1 AND age = 20 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((key OPERATOR(pg_catalog.=) 1) AND (age OPERATOR(pg_catalog.=) 20)) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM second_distributed_table WHERE key = 1 ORDER BY 1,2; -NOTICE: executing the command locally: SELECT key, value FROM local_shard_execution.second_distributed_table_1470005 second_distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value - key | value ---------------------------------------------------------------------- -(0 rows) - --- Put rows back for other tests -INSERT INTO distributed_table VALUES (1, '22', 20); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 (key, value, age) VALUES (1, '22'::text, 20) -INSERT INTO second_distributed_table VALUES (1, '1'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.second_distributed_table_1470005 (key, value) VALUES (1, '1'::text) -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW abcd_view AS SELECT * FROM abcd; -RESET citus.enable_ddl_propagation; -SELECT * FROM abcd first join abcd second on first.b = second.b ORDER BY 1,2,3,4; - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -BEGIN; -SELECT * FROM abcd first join abcd second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution.abcd_1470025 first JOIN local_shard_execution.abcd_1470025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution.abcd_1470027 first JOIN local_shard_execution.abcd_1470027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd_view first join abcd_view second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution.abcd_1470025 abcd JOIN local_shard_execution.abcd_1470025 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution.abcd_1470027 abcd JOIN local_shard_execution.abcd_1470027 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first full join abcd second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution.abcd_1470025 first FULL JOIN local_shard_execution.abcd_1470025 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution.abcd_1470027 first FULL JOIN local_shard_execution.abcd_1470027 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first join abcd second USING(b) ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution.abcd_1470025 first JOIN local_shard_execution.abcd_1470025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution.abcd_1470027 first JOIN local_shard_execution.abcd_1470027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true - b | c | d | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 3 | 4 - 3 | 4 | 5 | 4 | 5 - 4 | 5 | 6 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first join abcd second USING(b) join abcd third on first.b=third.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution.abcd_1470025 first JOIN local_shard_execution.abcd_1470025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution.abcd_1470025 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution.abcd_1470027 first JOIN local_shard_execution.abcd_1470027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution.abcd_1470027 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true - b | c | d | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; --- copy always happens via distributed execution irrespective of the --- shards that are accessed -COPY reference_table FROM STDIN; -COPY distributed_table FROM STDIN WITH CSV; -COPY second_distributed_table FROM STDIN WITH CSV; --- the behaviour in transaction blocks is the following: - -- (a) Unless the first query is a local query, always use distributed execution. - -- (b) If the executor has used local execution, it has to use local execution - -- for the remaining of the transaction block. If that's not possible, the - -- executor has to error out --- rollback should be able to rollback local execution -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - -ROLLBACK; --- make sure that the value is rollbacked -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - --- rollback should be able to rollback both the local and distributed executions -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table - -- DELETE should cascade, and we should not see any rows - SELECT count(*) FROM second_distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.second_distributed_table_1470005 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.second_distributed_table_1470007 second_distributed_table WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - -ROLLBACK; --- make sure that everything is rollbacked -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - -SELECT count(*) FROM second_distributed_table; - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT * FROM second_distributed_table ORDER BY 1; - key | value ---------------------------------------------------------------------- - 1 | 1 - 6 | '6' -(2 rows) - --- very simple examples, an SELECTs should see the modifications --- that has done before -BEGIN; - -- INSERT is executed locally - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '23' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '23'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- since the INSERT is executed locally, the SELECT should also be - -- executed locally and see the changes - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- multi-shard SELECTs are now forced to use local execution on - -- the shards that reside on this node - SELECT * FROM distributed_table WHERE value = '23' ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- similarly, multi-shard modifications should use local exection - -- on the shards that reside on this node - DELETE FROM distributed_table WHERE value = '23'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - -- make sure that the value is deleted - SELECT * FROM distributed_table WHERE value = '23' ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - key | value | age ---------------------------------------------------------------------- -(0 rows) - -COMMIT; --- make sure that we've committed everything -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - --- if we start with a distributed execution, we should keep --- using that and never switch back to local execution -BEGIN; - DELETE FROM distributed_table WHERE value = '11'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) - -- although this command could have been executed - -- locally, it is not going to be executed locally - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - - -- but we can still execute parallel queries, even if - -- they are utility commands - TRUNCATE distributed_table CASCADE; -NOTICE: truncate cascades to table "second_distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE - -- TRUNCATE cascaded into second_distributed_table - SELECT count(*) FROM second_distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.second_distributed_table_1470005 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.second_distributed_table_1470007 second_distributed_table WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - -ROLLBACK; --- load some data so that foreign keys won't complain with the next tests -INSERT INTO reference_table SELECT i FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx --- show that cascading foreign keys just works fine with local execution -BEGIN; - INSERT INTO reference_table VALUES (701); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (701) - INSERT INTO distributed_table VALUES (701, '701', 701); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 (key, value, age) VALUES (701, '701'::text, 701) - INSERT INTO second_distributed_table VALUES (701, '701'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.second_distributed_table_1470005 (key, value) VALUES (701, '701'::text) - DELETE FROM reference_table WHERE key = 701; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table WHERE (key OPERATOR(pg_catalog.=) 701) - SELECT count(*) FROM distributed_table WHERE key = 701; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 701) - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT count(*) FROM second_distributed_table WHERE key = 701; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.second_distributed_table_1470005 second_distributed_table WHERE (key OPERATOR(pg_catalog.=) 701) - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- multi-shard commands should also see the changes - SELECT count(*) FROM distributed_table WHERE key > 700; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- we can still do multi-shard commands - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table -ROLLBACK; --- multiple queries hitting different shards can be executed locally -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT count(*) FROM distributed_table WHERE key = 6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT count(*) FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - count ---------------------------------------------------------------------- - 0 -(1 row) - -ROLLBACK; --- a local query followed by TRUNCATE command can be executed locally -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - TRUNCATE distributed_table CASCADE; -NOTICE: truncate cascades to table "second_distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -ROLLBACK; --- a local query is followed by an INSERT..SELECT via the coordinator -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - INSERT INTO distributed_table (key) SELECT i FROM generate_series(1,1) i; -NOTICE: executing the copy locally for shard xxxxx -ROLLBACK; -BEGIN; -SET citus.enable_repartition_joins TO ON; -SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT count(*) FROM distributed_table d1 join distributed_table d2 using(age); -NOTICE: executing the command locally: SELECT partition_index, 'repartition_70_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_70_1','SELECT age AS column1 FROM local_shard_execution.distributed_table_1470001 d1 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_70_3' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_70_3','SELECT age AS column1 FROM local_shard_execution.distributed_table_1470003 d1 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_71_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_71_1','SELECT age AS column1 FROM local_shard_execution.distributed_table_1470001 d2 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_71_3' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_71_3','SELECT age AS column1 FROM local_shard_execution.distributed_table_1470003 d2 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_1_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_2_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_3_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_4_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_1_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_2_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_3_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_4_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_1_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_2_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_3_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_4_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_1_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_2_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_3_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_71_4_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_70_1_0,repartition_70_2_0,repartition_70_3_0,repartition_70_4_0}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_71_1_0,repartition_71_2_0,repartition_71_3_0,repartition_71_4_0}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_70_1_1,repartition_70_2_1,repartition_70_3_1,repartition_70_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_71_1_1,repartition_71_2_1,repartition_71_3_1,repartition_71_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_70_1_2,repartition_70_2_2,repartition_70_3_2,repartition_70_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_71_1_2,repartition_71_2_2,repartition_71_3_2,repartition_71_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_70_1_3,repartition_70_2_3,repartition_70_3_3,repartition_70_4_3}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_71_1_3,repartition_71_2_3,repartition_71_3_3,repartition_71_4_3}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -ROLLBACK; --- a local query is followed by an INSERT..SELECT with re-partitioning -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 1 -(1 row) - - INSERT INTO reference_table (key) SELECT -key FROM distributed_table; -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true -NOTICE: executing the copy locally for shard xxxxx - INSERT INTO distributed_table (key) SELECT -key FROM distributed_table; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1470001_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1470001_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1470003_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1470003_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key) SELECT key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1470003_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer) - SELECT count(*) FROM distributed_table WHERE key = -6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) '-6'::integer) - count ---------------------------------------------------------------------- - 1 -(1 row) - -ROLLBACK; -INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 -(1 row) - -BEGIN; - DELETE FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - EXPLAIN ANALYZE DELETE FROM distributed_table WHERE key = 1; -ERROR: cannot execute command because a local execution has accessed a placement in the transaction -DETAIL: Some parallel commands cannot be executed if a previous command has already been executed locally -HINT: Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;" -ROLLBACK; -BEGIN; - INSERT INTO distributed_table VALUES (11, '111',29) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; - key | value | age ---------------------------------------------------------------------- - 11 | 29 | 121 -(1 row) - - -- this is already disallowed on the nodes, adding it in case we - -- support DDLs from the worker nodes in the future - ALTER TABLE distributed_table ADD COLUMN x INT; -ERROR: operation is not allowed on this node -HINT: Connect to the coordinator and run it again. -ROLLBACK; -BEGIN; - INSERT INTO distributed_table VALUES (11, '111',29) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; - key | value | age ---------------------------------------------------------------------- - 11 | 29 | 121 -(1 row) - - -- this is already disallowed because VACUUM cannot be executed in tx block - -- adding in case this is supported some day - VACUUM second_distributed_table; -ERROR: VACUUM cannot run inside a transaction block -ROLLBACK; --- make sure that functions can use local execution -SET citus.enable_metadata_sync TO OFF; -CREATE OR REPLACE PROCEDURE only_local_execution() AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = 1; - DELETE FROM distributed_table WHERE key = 1; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution(); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function only_local_execution() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "DELETE FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution() line XX at SQL statement --- insert a row that we need in the next tests -INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text --- make sure that functions can use local execution -CREATE OR REPLACE PROCEDURE only_local_execution_with_function_evaluation() AS $$ - DECLARE nodeId INT; - BEGIN - -- fast path router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table WHERE key = 1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - - -- regular router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = 1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_function_evaluation(); -NOTICE: executing the command locally: SELECT local_shard_execution.get_local_node_id_volatile() AS get_local_node_id_volatile FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution_with_function_evaluation() line XX at SQL statement -NOTICE: executing the command locally: SELECT local_shard_execution.get_local_node_id_volatile() AS get_local_node_id_volatile FROM (local_shard_execution.distributed_table_1470001 d1(key, value, age) JOIN local_shard_execution.distributed_table_1470001 d2(key, value, age) USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = 1" -PL/pgSQL function only_local_execution_with_function_evaluation() line XX at SQL statement -CREATE OR REPLACE PROCEDURE only_local_execution_with_params(int) AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES ($1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = $1; - DELETE FROM distributed_table WHERE key = $1; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_params(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES ($1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "DELETE FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -CREATE OR REPLACE PROCEDURE only_local_execution_with_function_evaluation_param(int) AS $$ - DECLARE nodeId INT; - BEGIN - -- fast path router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table WHERE key = $1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - - -- regular router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = $1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_function_evaluation_param(1); -NOTICE: executing the command locally: SELECT local_shard_execution.get_local_node_id_volatile() AS get_local_node_id_volatile FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_function_evaluation_param(integer) line XX at SQL statement -NOTICE: executing the command locally: SELECT local_shard_execution.get_local_node_id_volatile() AS get_local_node_id_volatile FROM (local_shard_execution.distributed_table_1470001 d1(key, value, age) JOIN local_shard_execution.distributed_table_1470001 d2(key, value, age) USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) $1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = $1" -PL/pgSQL function only_local_execution_with_function_evaluation_param(integer) line XX at SQL statement -CREATE OR REPLACE PROCEDURE local_execution_followed_by_dist() AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = 1; - DELETE FROM distributed_table; - SELECT count(*) INTO cnt FROM distributed_table; - END; -$$ LANGUAGE plpgsql; -RESET citus.enable_metadata_sync; -CALL local_execution_followed_by_dist(); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = 1" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement --- test CTEs, including modifying CTEs -WITH local_insert AS (INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *), -distributed_local_mixed AS (SELECT * FROM reference_table WHERE key IN (SELECT key FROM local_insert)) -SELECT * FROM local_insert, distributed_local_mixed; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: SELECT key FROM local_shard_execution.reference_table_1470000 reference_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT local_insert.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert)) -NOTICE: executing the command locally: SELECT local_insert.key, local_insert.value, local_insert.age, distributed_local_mixed.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) distributed_local_mixed - key | value | age | key ---------------------------------------------------------------------- - 1 | 11 | 21 | 1 -(1 row) - --- since we start with parallel execution, we do not switch back to local execution in the --- latter CTEs -WITH distributed_local_mixed AS (SELECT * FROM distributed_table), -local_insert AS (INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *) -SELECT * FROM local_insert, distributed_local_mixed ORDER BY 1,2,3,4,5; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution.distributed_table_1470001 distributed_table) distributed_local_mixed) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution.distributed_table_1470003 distributed_table) distributed_local_mixed) worker_subquery - key | value | age | key | value | age ---------------------------------------------------------------------- - 1 | 29 | 21 | 1 | 11 | 21 -(1 row) - --- router CTE pushdown -WITH all_data AS (SELECT * FROM distributed_table WHERE key = 1) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.key AND distributed_table.key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table, (SELECT distributed_table_1.key, distributed_table_1.value, distributed_table_1.age FROM local_shard_execution.distributed_table_1470001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) - count ---------------------------------------------------------------------- - 1 -(1 row) - -INSERT INTO reference_table VALUES (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (2) -INSERT INTO distributed_table VALUES (2, '29', 29); -INSERT INTO second_distributed_table VALUES (2, '29'); --- single shard that is not a local query followed by a local query -WITH all_data AS (SELECT * FROM second_distributed_table WHERE key = 2) -SELECT - distributed_table.key -FROM - distributed_table, all_data -WHERE - distributed_table.value = all_data.value AND distributed_table.key = 1 -ORDER BY - 1 DESC; -NOTICE: executing the command locally: SELECT distributed_table.key FROM local_shard_execution.distributed_table_1470001 distributed_table, (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) all_data WHERE ((distributed_table.value OPERATOR(pg_catalog.=) all_data.value) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) ORDER BY distributed_table.key DESC - key ---------------------------------------------------------------------- - 1 -(1 row) - --- multi-shard CTE is followed by a query which could be executed locally, but --- since the query started with a parallel query, it doesn't use local execution --- note that if we allow Postgres to inline the CTE (e.g., not have the EXISTS --- subquery), then it'd pushdown the filters and the query becomes single-shard, --- locally executable query -WITH all_data AS (SELECT * FROM distributed_table) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.key AND distributed_table.key = 1 - AND EXISTS (SELECT * FROM all_data); -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 1) AND (EXISTS (SELECT all_data_1.key, all_data_1.value, all_data_1.age FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) all_data_1))) - count ---------------------------------------------------------------------- - 1 -(1 row) - --- in pg12, the following CTE can be inlined, still the query becomes --- a subquery that needs to be recursively planned and a parallel --- query, so do not use local execution -WITH all_data AS (SELECT age FROM distributed_table) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.age AND distributed_table.key = 1; -NOTICE: executing the command locally: SELECT age FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT age FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table, (SELECT intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(age bigint)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.age) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) - count ---------------------------------------------------------------------- - 0 -(1 row) - --- get ready for the next commands -TRUNCATE reference_table, distributed_table, second_distributed_table; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.reference_table_xxxxx CASCADE -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE --- local execution of returning of reference tables -INSERT INTO reference_table VALUES (1),(2),(3),(4),(5),(6) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 AS citus_table_alias (key) VALUES (1), (2), (3), (4), (5), (6) RETURNING citus_table_alias.key - key ---------------------------------------------------------------------- - 1 - 2 - 3 - 4 - 5 - 6 -(6 rows) - --- local execution of multi-row INSERTs -INSERT INTO distributed_table VALUES (1, '11',21), (5,'55',22) ON CONFLICT(key) DO UPDATE SET value = (EXCLUDED.value::int + 1)::text RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'11'::text,'21'::bigint), (5,'55'::text,'22'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 - 5 | 55 | 22 -(2 rows) - --- distributed execution of multi-rows INSERTs, where executor --- is smart enough to execute local tasks via local execution -INSERT INTO distributed_table VALUES (1, '11',21), (2,'22',22), (3,'33',33), (4,'44',44),(5,'55',55) ON CONFLICT(key) DO UPDATE SET value = (EXCLUDED.value::int + 1)::text RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'11'::text,'21'::bigint), (5,'55'::text,'55'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 12 | 21 - 2 | 22 | 22 - 3 | 33 | 33 - 4 | 44 | 44 - 5 | 56 | 22 -(5 rows) - -PREPARE local_prepare_no_param AS SELECT count(*) FROM distributed_table WHERE key = 1; -PREPARE local_prepare_no_param_subquery AS -SELECT DISTINCT trim(value) FROM ( - SELECT value FROM distributed_table - WHERE - key IN (1, 6, 500, 701) - AND (select 2) > random() - order by 1 - limit 2 - ) t; -PREPARE local_prepare_param (int) AS SELECT count(*) FROM distributed_table WHERE key = $1; -PREPARE remote_prepare_param (int) AS SELECT count(*) FROM distributed_table WHERE key != $1; -BEGIN; - -- 8 local execution without params - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - -- 8 local execution without params and some subqueries - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - -- 8 local executions with params - EXECUTE local_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(5); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 5) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(5); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 5) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- followed by a non-local execution - EXECUTE remote_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) - count ---------------------------------------------------------------------- - 4 -(1 row) - -COMMIT; -PREPARE local_insert_prepare_no_param AS INSERT INTO distributed_table VALUES (1+0*random(), '11',21::int) ON CONFLICT(key) DO UPDATE SET value = '29' || '28' RETURNING *, key + 1, value || '30', age * 15; -PREPARE local_insert_prepare_param (int) AS INSERT INTO distributed_table VALUES ($1+0*random(), '11',21::int) ON CONFLICT(key) DO UPDATE SET value = '29' || '28' RETURNING *, key + 1, value || '30', age * 15; -BEGIN; - -- 8 local execution without params - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - -- 8 local executions with params - EXECUTE local_insert_prepare_param(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 5 | 2928 | 22 | 6 | 292830 | 330 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 11 | 21 | 7 | 1130 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 5 | 2928 | 22 | 6 | 292830 | 330 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - -- followed by a non-local execution - EXECUTE remote_prepare_param(2); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) - count ---------------------------------------------------------------------- - 5 -(1 row) - -COMMIT; -PREPARE local_multi_row_insert_prepare_no_param AS - INSERT INTO distributed_table VALUES (1,'55', 21), (5,'15',33) ON CONFLICT (key) WHERE key > 3 and key < 4 DO UPDATE SET value = '88' || EXCLUDED.value; -PREPARE local_multi_row_insert_prepare_no_param_multi_shard AS - INSERT INTO distributed_table VALUES (6,'55', 21), (5,'15',33) ON CONFLICT (key) WHERE key > 3 AND key < 4 DO UPDATE SET value = '88' || EXCLUDED.value;; -PREPARE local_multi_row_insert_prepare_params(int,int) AS - INSERT INTO distributed_table VALUES ($1,'55', 21), ($2,'15',33) ON CONFLICT (key) WHERE key > 3 and key < 4 DO UPDATE SET value = '88' || EXCLUDED.value;; -INSERT INTO reference_table VALUES (11); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (11) -BEGIN; - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(6,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint), (1,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint), (1,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - -- one task is remote - EXECUTE local_multi_row_insert_prepare_params(5,11); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -ROLLBACK; --- make sure that we still get results if we switch off local execution -PREPARE ref_count_prepare AS SELECT count(*) FROM reference_table; -EXECUTE ref_count_prepare; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.reference_table_1470000 reference_table - count ---------------------------------------------------------------------- - 7 -(1 row) - -EXECUTE ref_count_prepare; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.reference_table_1470000 reference_table - count ---------------------------------------------------------------------- - 7 -(1 row) - -EXECUTE ref_count_prepare; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.reference_table_1470000 reference_table - count ---------------------------------------------------------------------- - 7 -(1 row) - -EXECUTE ref_count_prepare; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.reference_table_1470000 reference_table - count ---------------------------------------------------------------------- - 7 -(1 row) - -EXECUTE ref_count_prepare; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.reference_table_1470000 reference_table - count ---------------------------------------------------------------------- - 7 -(1 row) - -SET citus.enable_local_execution TO off; -EXECUTE ref_count_prepare; - count ---------------------------------------------------------------------- - 7 -(1 row) - -RESET citus.enable_local_execution; --- failures of local execution should rollback both the --- local execution and remote executions --- fail on a local execution -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 100 | 21 -(1 row) - - UPDATE distributed_table SET value = '200'; -NOTICE: executing the command locally: UPDATE local_shard_execution.distributed_table_1470001 distributed_table SET value = '200'::text -NOTICE: executing the command locally: UPDATE local_shard_execution.distributed_table_1470003 distributed_table SET value = '200'::text - INSERT INTO distributed_table VALUES (1, '100',21) ON CONFLICT(key) DO UPDATE SET value = (1 / (100.0 - EXCLUDED.value::int))::text RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '100'::text, 21) ON CONFLICT(key) DO UPDATE SET value = (((1)::numeric OPERATOR(pg_catalog./) (100.0 OPERATOR(pg_catalog.-) ((excluded.value)::integer)::numeric)))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -ERROR: division by zero -ROLLBACK; --- we've rollbacked everything -SELECT count(*) FROM distributed_table WHERE value = '200'; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- RETURNING should just work fine for reference tables -INSERT INTO reference_table VALUES (500) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - -DELETE FROM reference_table WHERE key = 500 RETURNING *; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table WHERE (key OPERATOR(pg_catalog.=) 500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - --- should be able to skip local execution even if in a sequential mode of execution -BEGIN; - SET LOCAL citus.multi_shard_modify_mode TO sequential ; - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 -(1 row) - -ROLLBACK; --- sequential execution should just work fine after a local execution -BEGIN; - SET citus.multi_shard_modify_mode TO sequential ; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 100 | 21 -(1 row) - - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table -ROLLBACK; --- load some data so that foreign keys won't complain with the next tests -TRUNCATE reference_table CASCADE; -NOTICE: truncate cascades to table "distributed_table" -NOTICE: truncate cascades to table "second_distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.reference_table_xxxxx CASCADE -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -INSERT INTO reference_table SELECT i FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO distributed_table SELECT i, i::text, i % 10 + 25 FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx --- show that both local, and mixed local-distributed executions --- calculate rows processed correctly -BEGIN; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - DELETE FROM distributed_table WHERE value != '123123213123213'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -ROLLBACK; -BEGIN; - DELETE FROM reference_table WHERE key = 500 RETURNING *; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table WHERE (key OPERATOR(pg_catalog.=) 500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - - DELETE FROM reference_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table -ROLLBACK; -BEGIN; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - -ROLLBACK; -BEGIN; - SET LOCAL client_min_messages TO INFO; - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true - count ---------------------------------------------------------------------- - 101 -(1 row) - - SET LOCAL client_min_messages TO LOG; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -ROLLBACK; --- probably not a realistic case since views are not very --- well supported with MX -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW v_local_query_execution AS -SELECT * FROM distributed_table WHERE key = 500; -RESET citus.enable_ddl_propagation; -SELECT * FROM v_local_query_execution; -NOTICE: executing the command locally: SELECT key, value, age FROM (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (distributed_table.key OPERATOR(pg_catalog.=) 500)) v_local_query_execution - key | value | age ---------------------------------------------------------------------- - 500 | 500 | 25 -(1 row) - --- similar test, but this time the view itself is a non-local --- query, but the query on the view is local -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW v_local_query_execution_2 AS -SELECT * FROM distributed_table; -RESET citus.enable_ddl_propagation; -SELECT * FROM v_local_query_execution_2 WHERE key = 500; -NOTICE: executing the command locally: SELECT key, value, age FROM (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution.distributed_table_1470003 distributed_table) v_local_query_execution_2 WHERE (key OPERATOR(pg_catalog.=) 500) - key | value | age ---------------------------------------------------------------------- - 500 | 500 | 25 -(1 row) - --- even if we switch from remote execution -> local execution, --- we are able to use remote execution after rollback -BEGIN; - SAVEPOINT my_savepoint; - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true - count ---------------------------------------------------------------------- - 101 -(1 row) - - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - ROLLBACK TO SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -COMMIT; --- even if we switch from local execution -> remote execution, --- we are able to use local execution after rollback -BEGIN; - SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true - count ---------------------------------------------------------------------- - 100 -(1 row) - - ROLLBACK TO SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -COMMIT; --- sanity check: local execution on partitions -INSERT INTO collections_list (collection_id) VALUES (0) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470011 (key, ser, collection_id) VALUES ('3940649673949185'::bigint, '3940649673949185'::bigint, 0) RETURNING key, ser, ts, collection_id, value - key | ser | ts | collection_id | value ---------------------------------------------------------------------- - 3940649673949185 | 3940649673949185 | | 0 | -(1 row) - -BEGIN; - INSERT INTO collections_list (key, collection_id) VALUES (1,0); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470009 (key, ser, collection_id) VALUES ('1'::bigint, '3940649673949186'::bigint, 0) - SELECT count(*) FROM collections_list_0 WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.collections_list_0_1470013 collections_list_0 WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT count(*) FROM collections_list; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.collections_list_1470009 collections_list WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.collections_list_1470011 collections_list WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - - SELECT * FROM collections_list ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution.collections_list_1470009 collections_list WHERE true -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution.collections_list_1470011 collections_list WHERE true - key | ser | ts | collection_id | value ---------------------------------------------------------------------- - 1 | 3940649673949186 | | 0 | - 3940649673949185 | 3940649673949185 | | 0 | -(2 rows) - -COMMIT; -TRUNCATE collections_list; --- make sure that even if local execution is used, the sequence values --- are generated locally -SET citus.enable_ddl_propagation TO OFF; -ALTER SEQUENCE collections_list_key_seq NO MINVALUE NO MAXVALUE; -RESET citus.enable_ddl_propagation; -PREPARE serial_prepared_local AS INSERT INTO collections_list (collection_id) VALUES (0) RETURNING key, ser; -SELECT setval('collections_list_key_seq', 4); - setval ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470009 (key, ser, collection_id) VALUES ('5'::bigint, '3940649673949187'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 5 | 3940649673949187 -(1 row) - -SELECT setval('collections_list_key_seq', 5); - setval ---------------------------------------------------------------------- - 5 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470011 (key, ser, collection_id) VALUES ('6'::bigint, '3940649673949188'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 6 | 3940649673949188 -(1 row) - -SELECT setval('collections_list_key_seq', 499); - setval ---------------------------------------------------------------------- - 499 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470011 (key, ser, collection_id) VALUES ('500'::bigint, '3940649673949189'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 500 | 3940649673949189 -(1 row) - -SELECT setval('collections_list_key_seq', 700); - setval ---------------------------------------------------------------------- - 700 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470009 (key, ser, collection_id) VALUES ('701'::bigint, '3940649673949190'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 701 | 3940649673949190 -(1 row) - -SELECT setval('collections_list_key_seq', 708); - setval ---------------------------------------------------------------------- - 708 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470011 (key, ser, collection_id) VALUES ('709'::bigint, '3940649673949191'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 709 | 3940649673949191 -(1 row) - -SELECT setval('collections_list_key_seq', 709); - setval ---------------------------------------------------------------------- - 709 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470009 (key, ser, collection_id) VALUES ('710'::bigint, '3940649673949192'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 710 | 3940649673949192 -(1 row) - --- get ready for the next executions -DELETE FROM collections_list WHERE key IN (5,6); -SELECT setval('collections_list_key_seq', 4); - setval ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470009 (key, ser, collection_id) VALUES ('5'::bigint, '3940649673949193'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 5 | 3940649673949193 -(1 row) - -SELECT setval('collections_list_key_seq', 5); - setval ---------------------------------------------------------------------- - 5 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.collections_list_1470011 (key, ser, collection_id) VALUES ('6'::bigint, '3940649673949194'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 6 | 3940649673949194 -(1 row) - --- and, one remote test -SELECT setval('collections_list_key_seq', 10); - setval ---------------------------------------------------------------------- - 10 -(1 row) - -EXECUTE serial_prepared_local; - key | ser ---------------------------------------------------------------------- - 11 | 3940649673949195 -(1 row) - --- the final queries for the following CTEs are going to happen on the intermediate results only --- one of them will be executed remotely, and the other is locally --- Citus currently doesn't allow using task_assignment_policy for intermediate results -WITH distributed_local_mixed AS (INSERT INTO reference_table VALUES (1000) RETURNING *) SELECT * FROM distributed_local_mixed; -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (1000) RETURNING key -NOTICE: executing the command locally: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) distributed_local_mixed - key ---------------------------------------------------------------------- - 1000 -(1 row) - --- clean the table for the next tests -SET search_path TO local_shard_execution; -TRUNCATE distributed_table CASCADE; -NOTICE: truncate cascades to table "second_distributed_table" --- load some data on a remote shard -INSERT INTO reference_table (key) VALUES (1), (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 AS citus_table_alias (key) VALUES (1), (2) -INSERT INTO distributed_table (key) VALUES (2); -BEGIN; - -- local execution followed by a distributed query - INSERT INTO distributed_table (key) VALUES (1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.distributed_table_1470001 (key) VALUES (1) - DELETE FROM distributed_table RETURNING key; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470001 distributed_table RETURNING key -NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table RETURNING key - key ---------------------------------------------------------------------- - 1 - 2 -(2 rows) - -COMMIT; --- a similar test with a reference table -TRUNCATE reference_table CASCADE; -NOTICE: truncate cascades to table "distributed_table" -NOTICE: truncate cascades to table "second_distributed_table" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.reference_table_xxxxx CASCADE -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.distributed_table_xxxxx CASCADE -NOTICE: truncate cascades to table "second_distributed_table_xxxxx" -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution.second_distributed_table_xxxxx CASCADE --- load some data on a remote shard -INSERT INTO reference_table (key) VALUES (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (2) -BEGIN; - -- local execution followed by a distributed query - INSERT INTO reference_table (key) VALUES (1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution.reference_table_1470000 (key) VALUES (1) - DELETE FROM reference_table RETURNING key; -NOTICE: executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table RETURNING key - key ---------------------------------------------------------------------- - 1 - 2 -(2 rows) - -COMMIT; --- however complex the query, local execution can handle -SET client_min_messages TO LOG; -SET citus.log_local_commands TO ON; -WITH cte_1 AS - (SELECT * - FROM - (WITH cte_1 AS - (SELECT * - FROM distributed_table - WHERE key = 1) SELECT * - FROM cte_1) AS foo) -SELECT count(*) -FROM cte_1 -JOIN distributed_table USING (key) -WHERE distributed_table.key = 1 - AND distributed_table.key IN - (SELECT key - FROM distributed_table - WHERE key = 1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM ((SELECT foo.key, foo.value, foo.age FROM (SELECT cte_1_1.key, cte_1_1.value, cte_1_1.age FROM (SELECT distributed_table_1.key, distributed_table_1.value, distributed_table_1.age FROM local_shard_execution.distributed_table_1470001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)) cte_1_1) foo) cte_1 JOIN local_shard_execution.distributed_table_1470001 distributed_table(key, value, age) USING (key)) WHERE ((distributed_table.key OPERATOR(pg_catalog.=) 1) AND (distributed_table.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table_1.key FROM local_shard_execution.distributed_table_1470001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)))) - count ---------------------------------------------------------------------- - 0 -(1 row) - -RESET client_min_messages; -RESET citus.log_local_commands; -\c - - - :master_port -SET search_path TO local_shard_execution; -SET citus.next_shard_id TO 1480000; --- test both local and remote execution with custom type -SET citus.shard_replication_factor TO 1; -CREATE TYPE invite_resp AS ENUM ('yes', 'no', 'maybe'); -CREATE TABLE event_responses ( - event_id int, - user_id int, - response invite_resp, - primary key (event_id, user_id) -); -SELECT create_distributed_table('event_responses', 'event_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO event_responses VALUES (1, 1, 'yes'), (2, 2, 'yes'), (3, 3, 'no'), (4, 4, 'no'); -CREATE TABLE event_responses_no_pkey ( - event_id int, - user_id int, - response invite_resp -); -SELECT create_distributed_table('event_responses_no_pkey', 'event_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE OR REPLACE FUNCTION regular_func(p invite_resp) -RETURNS int AS $$ -DECLARE - q1Result INT; - q2Result INT; - q3Result INT; -BEGIN -SELECT count(*) INTO q1Result FROM event_responses WHERE response = $1; -SELECT count(*) INTO q2Result FROM event_responses e1 LEFT JOIN event_responses e2 USING (event_id) WHERE e2.response = $1; -SELECT count(*) INTO q3Result FROM (SELECT * FROM event_responses WHERE response = $1 LIMIT 5) as foo; -RETURN q3Result+q2Result+q1Result; -END; -$$ LANGUAGE plpgsql; -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -CREATE OR REPLACE PROCEDURE regular_procedure(p invite_resp) -AS $$ -BEGIN -PERFORM * FROM event_responses WHERE response = $1 ORDER BY 1 DESC, 2 DESC, 3 DESC; -PERFORM * FROM event_responses e1 LEFT JOIN event_responses e2 USING (event_id) WHERE e2.response = $1 ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; -PERFORM * FROM (SELECT * FROM event_responses WHERE response = $1 LIMIT 5) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC; -END; -$$ LANGUAGE plpgsql; -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -PREPARE multi_shard_no_dist_key(invite_resp) AS select * from event_responses where response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -PREPARE multi_shard_with_dist_key(int, invite_resp) AS select * from event_responses where event_id > $1 AND response = $2::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -PREPARE query_pushdown_no_dist_key(invite_resp) AS select * from event_responses e1 LEFT JOIN event_responses e2 USING(event_id) where e1.response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC LIMIT 1; -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -PREPARE insert_select_via_coord(invite_resp) AS INSERT INTO event_responses SELECT * FROM event_responses where response = $1::invite_resp LIMIT 1 ON CONFLICT (event_id, user_id) DO NOTHING ; -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -PREPARE insert_select_pushdown(invite_resp) AS INSERT INTO event_responses SELECT * FROM event_responses where response = $1::invite_resp ON CONFLICT (event_id, user_id) DO NOTHING; -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -PREPARE router_select_with_no_dist_key_filter(invite_resp) AS select * from event_responses where event_id = 1 AND response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - --- rest of the tests assume the table is empty -TRUNCATE event_responses; -CREATE OR REPLACE PROCEDURE register_for_event(p_event_id int, p_user_id int, p_choice invite_resp) -LANGUAGE plpgsql -SET search_path TO local_shard_execution -AS $fn$ -BEGIN - INSERT INTO event_responses VALUES (p_event_id, p_user_id, p_choice) - ON CONFLICT (event_id, user_id) - DO UPDATE SET response = EXCLUDED.response; - - PERFORM count(*) FROM event_responses WHERE event_id = p_event_id; - - PERFORM count(*) FROM event_responses WHERE event_id = p_event_id AND false; - - UPDATE event_responses SET response = p_choice WHERE event_id = p_event_id; - -END; -$fn$; -SELECT create_distributed_function('register_for_event(int,int,invite_resp)', 'p_event_id', 'event_responses'); - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - --- call 8 times to make sure it works after the 5th time(postgres binds values after the 5th time and Citus 2nd time) --- after 6th, the local execution caches the local plans and uses it --- execute it both locally and remotely -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -\c - - - :worker_2_port -SET search_path TO local_shard_execution; -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); --- values 16, 17 and 19 hits the same --- shard, so we're re-using the same cached --- plans per statement across different distribution --- key values -CALL register_for_event(17, 1, 'yes'); -CALL register_for_event(19, 1, 'yes'); -CALL register_for_event(17, 1, 'yes'); -CALL register_for_event(19, 1, 'yes'); --- should work fine if the logs are enabled -\set VERBOSITY terse -SET citus.log_local_commands TO ON; -SET client_min_messages TO DEBUG2; -CALL register_for_event(19, 1, 'yes'); -DEBUG: not pushing down procedure to the same node -NOTICE: executing the command locally: INSERT INTO local_shard_execution.event_responses_1480001 AS citus_table_alias (event_id, user_id, response) VALUES (19, 1, 'yes'::local_shard_execution.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.event_responses_1480001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 19) -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT NULL::integer AS event_id, NULL::integer AS user_id, NULL::local_shard_execution.invite_resp AS response WHERE false) event_responses(event_id, user_id, response) WHERE ((event_id OPERATOR(pg_catalog.=) 19) AND false) -NOTICE: executing the command locally: UPDATE local_shard_execution.event_responses_1480001 event_responses SET response = 'yes'::local_shard_execution.invite_resp WHERE (event_id OPERATOR(pg_catalog.=) 19) --- should be fine even if no parameters exists in the query -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.event_responses_1480001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 16) - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.event_responses_1480001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 16) - count ---------------------------------------------------------------------- - 1 -(1 row) - -UPDATE event_responses SET response = 'no' WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: UPDATE local_shard_execution.event_responses_1480001 event_responses SET response = 'no'::local_shard_execution.invite_resp WHERE (event_id OPERATOR(pg_catalog.=) 16) -INSERT INTO event_responses VALUES (16, 666, 'maybe') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: INSERT INTO local_shard_execution.event_responses_1480001 AS citus_table_alias (event_id, user_id, response) VALUES (16, 666, 'maybe'::local_shard_execution.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response RETURNING citus_table_alias.event_id, citus_table_alias.user_id, citus_table_alias.response - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe -(1 row) - --- multi row INSERTs hitting the same shard -INSERT INTO event_responses VALUES (16, 666, 'maybe'), (17, 777, 'no') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -NOTICE: executing the command locally: INSERT INTO local_shard_execution.event_responses_1480001 AS citus_table_alias (event_id, user_id, response) VALUES (16,666,'maybe'::local_shard_execution.invite_resp), (17,777,'no'::local_shard_execution.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response RETURNING citus_table_alias.event_id, citus_table_alias.user_id, citus_table_alias.response - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe - 17 | 777 | no -(2 rows) - --- now, similar tests with some settings changed -SET citus.enable_local_execution TO false; -SET citus.enable_fast_path_router_planner TO false; -CALL register_for_event(19, 1, 'yes'); -DEBUG: not pushing down procedure to the same node --- should be fine even if no parameters exists in the query -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - count ---------------------------------------------------------------------- - 2 -(1 row) - -UPDATE event_responses SET response = 'no' WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -INSERT INTO event_responses VALUES (16, 666, 'maybe') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe -(1 row) - --- multi row INSERTs hitting the same shard -INSERT INTO event_responses VALUES (16, 666, 'maybe'), (17, 777, 'no') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe - 17 | 777 | no -(2 rows) - --- set back to sane settings -RESET citus.enable_local_execution; -RESET citus.enable_fast_path_router_planner; --- we'll test some 2PC states -SET citus.enable_metadata_sync TO OFF; --- coordinated_transaction_should_use_2PC prints the internal --- state for 2PC decision on Citus. However, even if 2PC is decided, --- we may not necessarily use 2PC over a connection unless it does --- a modification -CREATE OR REPLACE FUNCTION coordinated_transaction_should_use_2PC() -RETURNS BOOL LANGUAGE C STRICT VOLATILE AS 'citus', -$$coordinated_transaction_should_use_2PC$$; --- make tests consistent -SET citus.max_adaptive_executor_pool_size TO 1; -RESET citus.enable_metadata_sync; -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - -SET citus.log_remote_commands TO ON; --- we use event_id = 2 for local execution and event_id = 1 for reemote execution ---show it here, if anything changes here, all the tests below might be broken --- we prefer this to avoid excessive logging below -SELECT * FROM event_responses_no_pkey WHERE event_id = 2; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 2 -NOTICE: executing the command locally: SELECT event_id, user_id, response FROM local_shard_execution.event_responses_no_pkey_1480007 event_responses_no_pkey WHERE (event_id OPERATOR(pg_catalog.=) 2) - event_id | user_id | response ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM event_responses_no_pkey WHERE event_id = 1; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 1 -NOTICE: issuing SELECT event_id, user_id, response FROM local_shard_execution.event_responses_no_pkey_1480004 event_responses_no_pkey WHERE (event_id OPERATOR(pg_catalog.=) 1) - event_id | user_id | response ---------------------------------------------------------------------- -(0 rows) - -RESET citus.log_remote_commands; -RESET citus.log_local_commands; -RESET client_min_messages; --- single shard local command without transaction block does set the --- internal state for 2PC, but does not require any actual entries -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *) -SELECT coordinated_transaction_should_use_2PC() FROM cte_1; - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- two local commands without transaction block set the internal 2PC state --- but does not use remotely -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *), - cte_2 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - t -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local modification followed by another single shard --- local modification sets the 2PC state, but does not use remotely -BEGIN; - INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - - INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local modification followed by a single shard --- remote modification uses 2PC because multiple nodes involved --- in the modification -BEGIN; - INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local modification followed by a single shard --- remote modification uses 2PC even if it is not in an explicit --- tx block as multiple nodes involved in the modification -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *), - cte_2 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - t -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a single shard --- local modification uses 2PC as multiple nodes involved --- in the modification -BEGIN; - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a single shard --- local modification uses 2PC even if it is not in an explicit --- tx block -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *), - cte_2 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - t -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local SELECT command without transaction block does not set the --- internal state for 2PC -WITH cte_1 AS (SELECT * FROM event_responses_no_pkey WHERE event_id = 2) -SELECT coordinated_transaction_should_use_2PC() FROM cte_1; -ERROR: The transaction is not a coordinated transaction -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- two local SELECT commands without transaction block does not set the internal 2PC state --- and does not use remotely -WITH cte_1 AS (SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2), - cte_2 AS (SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2) -SELECT count(*) FROM cte_1, cte_2; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- two local SELECT commands without transaction block does not set the internal 2PC state --- and does not use remotely -BEGIN; - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 9 -(1 row) - - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 9 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- a local SELECT followed by a remote SELECT does not require to --- use actual 2PC -BEGIN; - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 9 -(1 row) - - SELECT count(*) FROM event_responses_no_pkey; - count ---------------------------------------------------------------------- - 13 -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local SELECT followed by a single shard --- remote modification does not use 2PC, because only a single --- machine involved in the modification -BEGIN; - SELECT * FROM event_responses_no_pkey WHERE event_id = 2; - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes - 2 | 2 | yes -(9 rows) - - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local SELECT followed by a single shard --- remote modification does not use 2PC, because only a single --- machine involved in the modification -WITH cte_1 AS (SELECT * FROM event_responses_no_pkey WHERE event_id = 2), - cte_2 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a single shard --- local SELECT does not use 2PC, because only a single --- machine involved in the modification -BEGIN; - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 9 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a single shard --- local SELECT does not use 2PC, because only a single --- machine involved in the modification -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *), - cte_2 AS (SELECT * FROM event_responses_no_pkey WHERE event_id = 2) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- multi shard local SELECT command without transaction block does not set the --- internal state for 2PC -WITH cte_1 AS (SELECT count(*) FROM event_responses_no_pkey) -SELECT coordinated_transaction_should_use_2PC() FROM cte_1; - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- two multi-shard SELECT commands without transaction block does not set the internal 2PC state --- and does not use remotely -WITH cte_1 AS (SELECT count(*) FROM event_responses_no_pkey), - cte_2 AS (SELECT count(*) FROM event_responses_no_pkey) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- two multi-shard SELECT commands without transaction block does not set the internal 2PC state --- and does not use remotely -BEGIN; - SELECT count(*) FROM event_responses_no_pkey; - count ---------------------------------------------------------------------- - 17 -(1 row) - - SELECT count(*) FROM event_responses_no_pkey; - count ---------------------------------------------------------------------- - 17 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- multi-shard shard SELECT followed by a single shard --- remote modification does not use 2PC, because only a single --- machine involved in the modification -BEGIN; - SELECT count(*) FROM event_responses_no_pkey; - count ---------------------------------------------------------------------- - 17 -(1 row) - - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- multi shard SELECT followed by a single shard --- remote single shard modification does not use 2PC, because only a single --- machine involved in the modification -WITH cte_1 AS (SELECT count(*) FROM event_responses_no_pkey), - cte_2 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a multi shard --- SELECT does not use 2PC, because only a single --- machine involved in the modification -BEGIN; - INSERT INTO event_responses_no_pkey VALUES (1, 2, 'yes') RETURNING *; - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 2 | yes -(1 row) - - SELECT count(*) FROM event_responses_no_pkey; - count ---------------------------------------------------------------------- - 20 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard remote modification followed by a multi shard --- SELECT does not use 2PC, because only a single --- machine involved in the modification -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (1, 1, 'yes') RETURNING *), - cte_2 AS (SELECT count(*) FROM event_responses_no_pkey) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - f -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- single shard local modification followed by remote multi-shard --- modification uses 2PC as multiple nodes are involved in modifications -WITH cte_1 AS (INSERT INTO event_responses_no_pkey VALUES (2, 2, 'yes') RETURNING *), - cte_2 AS (UPDATE event_responses_no_pkey SET user_id = 1000 RETURNING *) -SELECT bool_or(coordinated_transaction_should_use_2PC()) FROM cte_1, cte_2; - bool_or ---------------------------------------------------------------------- - t -(1 row) - -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- a local SELECT followed by a remote multi-shard UPDATE requires to --- use actual 2PC as multiple nodes are involved in modifications -BEGIN; - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 10 -(1 row) - - UPDATE event_responses_no_pkey SET user_id = 1; -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- a local SELECT followed by a remote single-shard UPDATE does not require to --- use actual 2PC. This is because a single node is involved in modification -BEGIN; - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 10 -(1 row) - - UPDATE event_responses_no_pkey SET user_id = 1 WHERE event_id = 1; -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - --- a remote single-shard UPDATE followed by a local single shard SELECT --- does not require to use actual 2PC. This is because a single node --- is involved in modification -BEGIN; - UPDATE event_responses_no_pkey SET user_id = 1 WHERE event_id = 1; - SELECT count(*) FROM event_responses_no_pkey WHERE event_id = 2; - count ---------------------------------------------------------------------- - 10 -(1 row) - -COMMIT; -SELECT count(*) FROM pg_dist_transaction; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 0 -(1 row) - -\c - - - :master_port -SET search_path TO local_shard_execution; --- verify the local_hostname guc is used for local executions that should connect to the --- local host -ALTER SYSTEM SET citus.local_hostname TO 'foobar'; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(0.1); -- wait to make sure the config has changed before running the GUC - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -SET citus.enable_local_execution TO false; -- force a connection to the dummy placements --- run queries that use dummy placements for local execution -SELECT * FROM event_responses WHERE FALSE; -ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: -WITH cte_1 AS (SELECT * FROM event_responses LIMIT 1) SELECT count(*) FROM cte_1; -ERROR: connection to the remote node postgres@foobar:57636 failed with the following error: could not translate host name "foobar" to address: -ALTER SYSTEM RESET citus.local_hostname; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(.1); -- wait to make sure the config has changed before running the GUC - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO ERROR; -SET search_path TO public; -DROP SCHEMA local_shard_execution CASCADE; diff --git a/src/test/regress/expected/local_shard_execution_replicated.out b/src/test/regress/expected/local_shard_execution_replicated.out index d0593db4a..b086d7a84 100644 --- a/src/test/regress/expected/local_shard_execution_replicated.out +++ b/src/test/regress/expected/local_shard_execution_replicated.out @@ -1,17 +1,6 @@ -- -- LOCAL_SHARD_EXECUTION_REPLICATED -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA local_shard_execution_replicated; SET search_path TO local_shard_execution_replicated; SET citus.shard_count TO 4; diff --git a/src/test/regress/expected/local_shard_execution_replicated_0.out b/src/test/regress/expected/local_shard_execution_replicated_0.out deleted file mode 100644 index 992ff6b81..000000000 --- a/src/test/regress/expected/local_shard_execution_replicated_0.out +++ /dev/null @@ -1,2462 +0,0 @@ --- --- LOCAL_SHARD_EXECUTION_REPLICATED --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA local_shard_execution_replicated; -SET search_path TO local_shard_execution_replicated; -SET citus.shard_count TO 4; -SET citus.shard_replication_factor TO 2; -SET citus.next_shard_id TO 1500000; -CREATE TABLE reference_table (key int PRIMARY KEY); -SELECT create_reference_table('reference_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE distributed_table (key int PRIMARY KEY , value text, age bigint CHECK (age > 10)); -SELECT create_distributed_table('distributed_table','key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE second_distributed_table (key int PRIMARY KEY , value text); -SELECT create_distributed_table('second_distributed_table','key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- ingest some data to enable some tests with data -INSERT INTO distributed_table VALUES (1, '1', 20); --- This GUC prevents to acquire the remote lock for replicated --- tables -BEGIN; - SET LOCAL citus.allow_modifications_from_workers_to_replicated_tables TO false; - INSERT INTO second_distributed_table VALUES (1, '1'); - INSERT INTO reference_table VALUES (1); -COMMIT; --- a simple test for -CREATE TABLE collections_list ( - key bigserial, - ser bigserial, - ts timestamptz, - collection_id integer, - value numeric, - PRIMARY KEY(key, collection_id) -) PARTITION BY LIST (collection_id ); -SELECT create_distributed_table('collections_list', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE collections_list_0 - PARTITION OF collections_list (key, ser, ts, collection_id, value) - FOR VALUES IN ( 0 ); --- create a volatile function that returns the local node id -CREATE OR REPLACE FUNCTION get_local_node_id_volatile() -RETURNS INT AS $$ -DECLARE localGroupId int; -BEGIN - SELECT groupid INTO localGroupId FROM pg_dist_local_group; - RETURN localGroupId; -END; $$ language plpgsql VOLATILE; -SELECT create_distributed_function('get_local_node_id_volatile()'); -NOTICE: procedure local_shard_execution_replicated.get_local_node_id_volatile is already distributed -DETAIL: Citus distributes procedures with CREATE [PROCEDURE|FUNCTION|AGGREGATE] commands - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - --- test case for issue #3556 -CREATE TABLE accounts (id text PRIMARY KEY); -CREATE TABLE stats (account_id text PRIMARY KEY, spent int); -SELECT create_distributed_table('accounts', 'id', colocate_with => 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('stats', 'account_id', colocate_with => 'accounts'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO accounts (id) VALUES ('foo'); -INSERT INTO stats (account_id, spent) VALUES ('foo', 100); -CREATE TABLE abcd(a int, b int, c int, d int); -SELECT create_distributed_table('abcd', 'b'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO abcd VALUES (1,2,3,4); -INSERT INTO abcd VALUES (2,3,4,5); -INSERT INTO abcd VALUES (3,4,5,6); -ALTER TABLE abcd DROP COLUMN a; --- connection worker and get ready for the tests -\c - - - :worker_1_port -SET search_path TO local_shard_execution_replicated; --- test case for issue #3556 -SET citus.log_intermediate_results TO TRUE; -SET client_min_messages TO DEBUG1; -SELECT * -FROM -( - WITH accounts_cte AS ( - SELECT id AS account_id - FROM accounts - ), - joined_stats_cte_1 AS ( - SELECT spent, account_id - FROM stats - INNER JOIN accounts_cte USING (account_id) - ), - joined_stats_cte_2 AS ( - SELECT spent, account_id - FROM joined_stats_cte_1 - INNER JOIN accounts_cte USING (account_id) - ) - SELECT SUM(spent) OVER (PARTITION BY coalesce(account_id, NULL)) - FROM accounts_cte - INNER JOIN joined_stats_cte_2 USING (account_id) -) inner_query; -DEBUG: CTE joined_stats_cte_1 is going to be inlined via distributed planning -DEBUG: CTE joined_stats_cte_2 is going to be inlined via distributed planning -DEBUG: generating subplan XXX_1 for CTE accounts_cte: SELECT id AS account_id FROM local_shard_execution_replicated.accounts -DEBUG: generating subplan XXX_2 for subquery SELECT sum(joined_stats_cte_2.spent) OVER (PARTITION BY COALESCE(accounts_cte.account_id, NULL::text)) AS sum FROM ((SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte JOIN (SELECT joined_stats_cte_1.spent, joined_stats_cte_1.account_id FROM ((SELECT stats.spent, stats.account_id FROM (local_shard_execution_replicated.stats JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte_2 USING (account_id))) joined_stats_cte_1 JOIN (SELECT intermediate_result.account_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(account_id text)) accounts_cte_1 USING (account_id))) joined_stats_cte_2 USING (account_id)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT sum FROM (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint)) inner_query -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file - sum ---------------------------------------------------------------------- - 100 -(1 row) - -SET citus.log_intermediate_results TO DEFAULT; -SET client_min_messages TO DEFAULT; ---- enable logging to see which tasks are executed locally -SET citus.log_local_commands TO ON; --- first, make sure that local execution works fine --- with simple queries that are not in transcation blocks -SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - --- multiple tasks both of which are local should NOT use local execution --- because local execution means executing the tasks locally, so the executor --- favors parallel execution even if everyting is local to node -SELECT count(*) FROM distributed_table WHERE key IN (1,6); - count ---------------------------------------------------------------------- - 1 -(1 row) - --- queries that hit any remote shards should NOT use local execution -SELECT count(*) FROM distributed_table WHERE key IN (1,11); - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM distributed_table; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- modifications also follow the same rules -INSERT INTO reference_table VALUES (1) ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 AS citus_table_alias (key) VALUES (1) ON CONFLICT DO NOTHING -INSERT INTO distributed_table VALUES (1, '1', 21) ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '1'::text, 21) ON CONFLICT DO NOTHING --- local query -DELETE FROM distributed_table WHERE key = 1 AND age = 21; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((key OPERATOR(pg_catalog.=) 1) AND (age OPERATOR(pg_catalog.=) 21)) --- hitting multiple shards, so should be a distributed execution -DELETE FROM distributed_table WHERE age = 21; --- although both shards are local, the executor choose the parallel execution --- over the wire because as noted above local execution is sequential -DELETE FROM second_distributed_table WHERE key IN (1,6); --- similarly, any multi-shard query just follows distributed execution -DELETE FROM second_distributed_table; --- load some more data for the following tests -INSERT INTO second_distributed_table VALUES (1, '1'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.second_distributed_table_1500005 (key, value) VALUES (1, '1'::text) --- INSERT .. SELECT hitting a single single (co-located) shard(s) should --- be executed locally -INSERT INTO distributed_table -SELECT - distributed_table.* -FROM - distributed_table, second_distributed_table -WHERE - distributed_table.key = 1 and distributed_table.key=second_distributed_table.key -ON CONFLICT(key) DO UPDATE SET value = '22' -RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table, local_shard_execution_replicated.second_distributed_table_1500005 second_distributed_table WHERE (((distributed_table.key OPERATOR(pg_catalog.=) 1) AND (distributed_table.key OPERATOR(pg_catalog.=) second_distributed_table.key)) AND (distributed_table.key IS NOT NULL)) ON CONFLICT(key) DO UPDATE SET value = '22'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - --- INSERT .. SELECT hitting multi-shards should go thourgh distributed execution -INSERT INTO distributed_table -SELECT - distributed_table.* -FROM - distributed_table, second_distributed_table -WHERE - distributed_table.key != 1 and distributed_table.key=second_distributed_table.key -ON CONFLICT(key) DO UPDATE SET value = '22' -RETURNING *; - key | value | age ---------------------------------------------------------------------- -(0 rows) - --- INSERT..SELECT via coordinator consists of two steps, select + COPY --- that's why it is disallowed to use local execution even if the SELECT --- can be executed locally -INSERT INTO distributed_table SELECT sum(key), value, max(age) FROM distributed_table WHERE key = 1 GROUP BY value ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: SELECT int4(sum(key)) AS key, value, max(age) AS age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) GROUP BY value -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) SELECT key, value, age FROM read_intermediate_result('insert_select_XXX_1500001'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint) ON CONFLICT DO NOTHING -INSERT INTO distributed_table SELECT 1, '1',15 FROM distributed_table WHERE key = 2 LIMIT 1 ON CONFLICT DO NOTHING; -NOTICE: executing the command locally: SELECT 1 AS key, '1'::text AS value, int8(15) AS age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (key OPERATOR(pg_catalog.=) 2) LIMIT 1 --- sanity check: multi-shard INSERT..SELECT pushdown goes through distributed execution -INSERT INTO distributed_table SELECT * FROM distributed_table ON CONFLICT DO NOTHING; --- Ensure tuple data in explain analyze output is the same on all PG versions -SET citus.enable_binary_protocol = TRUE; --- EXPLAIN for local execution just works fine --- though going through distributed execution -EXPLAIN (COSTS OFF) SELECT * FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Index Scan using distributed_table_pkey_1500001 on distributed_table_1500001 distributed_table - Index Cond: (key = 1) - Filter: (age = 20) -(8 rows) - -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - Task Count: 1 - Tuple data received from nodes: 14 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 14 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Index Scan using distributed_table_pkey_1500001 on distributed_table_1500001 distributed_table (actual rows=1 loops=1) - Index Cond: (key = 1) - Filter: (age = 20) -(10 rows) - -EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF) -WITH r AS ( SELECT GREATEST(random(), 2) z,* FROM distributed_table) -SELECT 1 FROM r WHERE z < 3; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - -> Distributed Subplan XXX_1 - Intermediate Data Size: 40 bytes - Result destination: Write locally - -> Custom Scan (Citus Adaptive) (actual rows=1 loops=1) - Task Count: 4 - Tuple data received from nodes: 22 bytes - Tasks Shown: One of 4 - -> Task - Tuple data received from node: 22 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on distributed_table_1500001 distributed_table (actual rows=1 loops=1) - Task Count: 1 - Tuple data received from nodes: 4 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 4 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) - Filter: (z < '3'::double precision) -(20 rows) - -EXPLAIN (COSTS OFF) DELETE FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Delete on distributed_table_1500001 distributed_table - -> Index Scan using distributed_table_pkey_1500001 on distributed_table_1500001 distributed_table - Index Cond: (key = 1) - Filter: (age = 20) -(9 rows) - -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) DELETE FROM distributed_table WHERE key = 1 AND age = 20; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=0 loops=1) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Delete on distributed_table_1500001 distributed_table (actual rows=0 loops=1) - -> Index Scan using distributed_table_pkey_1500001 on distributed_table_1500001 distributed_table (actual rows=1 loops=1) - Index Cond: (key = 1) - Filter: (age = 20) -(9 rows) - --- show that EXPLAIN ANALYZE deleted the row -SELECT * FROM distributed_table WHERE key = 1 AND age = 20 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((key OPERATOR(pg_catalog.=) 1) AND (age OPERATOR(pg_catalog.=) 20)) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM second_distributed_table WHERE key = 1 ORDER BY 1,2; -NOTICE: executing the command locally: SELECT key, value FROM local_shard_execution_replicated.second_distributed_table_1500005 second_distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value - key | value ---------------------------------------------------------------------- - 1 | 1 -(1 row) - --- Put row back for other tests -INSERT INTO distributed_table VALUES (1, '22', 20); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 (key, value, age) VALUES (1, '22'::text, 20) -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW abcd_view AS SELECT * FROM abcd; -RESET citus.enable_ddl_propagation; -SELECT * FROM abcd first join abcd second on first.b = second.b ORDER BY 1,2,3,4; - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -BEGIN; -SELECT * FROM abcd first join abcd second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500025 first JOIN local_shard_execution_replicated.abcd_1500025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500026 first JOIN local_shard_execution_replicated.abcd_1500026 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500027 first JOIN local_shard_execution_replicated.abcd_1500027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.b, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500028 first JOIN local_shard_execution_replicated.abcd_1500028 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd_view first join abcd_view second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution_replicated.abcd_1500025 abcd JOIN local_shard_execution_replicated.abcd_1500025 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution_replicated.abcd_1500026 abcd JOIN local_shard_execution_replicated.abcd_1500026 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution_replicated.abcd_1500027 abcd JOIN local_shard_execution_replicated.abcd_1500027 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true -NOTICE: executing the command locally: SELECT abcd.b, abcd.c, abcd.d, abcd_1.b, abcd_1.c, abcd_1.d FROM (local_shard_execution_replicated.abcd_1500028 abcd JOIN local_shard_execution_replicated.abcd_1500028 abcd_1 ON ((abcd.b OPERATOR(pg_catalog.=) abcd_1.b))) WHERE true - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first full join abcd second on first.b = second.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution_replicated.abcd_1500025 first FULL JOIN local_shard_execution_replicated.abcd_1500025 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution_replicated.abcd_1500026 first FULL JOIN local_shard_execution_replicated.abcd_1500026 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution_replicated.abcd_1500027 first FULL JOIN local_shard_execution_replicated.abcd_1500027 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS b, worker_column_2 AS c, worker_column_3 AS d, worker_column_4 AS b, worker_column_5 AS c, worker_column_6 AS d FROM (SELECT first.b AS worker_column_1, first.c AS worker_column_2, first.d AS worker_column_3, second.b AS worker_column_4, second.c AS worker_column_5, second.d AS worker_column_6 FROM (local_shard_execution_replicated.abcd_1500028 first FULL JOIN local_shard_execution_replicated.abcd_1500028 second ON ((first.b OPERATOR(pg_catalog.=) second.b)))) worker_subquery - b | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first join abcd second USING(b) ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500025 first JOIN local_shard_execution_replicated.abcd_1500025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500026 first JOIN local_shard_execution_replicated.abcd_1500026 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500027 first JOIN local_shard_execution_replicated.abcd_1500027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d FROM (local_shard_execution_replicated.abcd_1500028 first JOIN local_shard_execution_replicated.abcd_1500028 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) WHERE true - b | c | d | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 3 | 4 - 3 | 4 | 5 | 4 | 5 - 4 | 5 | 6 | 5 | 6 -(3 rows) - -END; -BEGIN; -SELECT * FROM abcd first join abcd second USING(b) join abcd third on first.b=third.b ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution_replicated.abcd_1500025 first JOIN local_shard_execution_replicated.abcd_1500025 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution_replicated.abcd_1500025 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution_replicated.abcd_1500026 first JOIN local_shard_execution_replicated.abcd_1500026 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution_replicated.abcd_1500026 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution_replicated.abcd_1500027 first JOIN local_shard_execution_replicated.abcd_1500027 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution_replicated.abcd_1500027 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true -NOTICE: executing the command locally: SELECT first.b, first.c, first.d, second.c, second.d, third.b, third.c, third.d FROM ((local_shard_execution_replicated.abcd_1500028 first JOIN local_shard_execution_replicated.abcd_1500028 second ON ((first.b OPERATOR(pg_catalog.=) second.b))) JOIN local_shard_execution_replicated.abcd_1500028 third ON ((first.b OPERATOR(pg_catalog.=) third.b))) WHERE true - b | c | d | c | d | b | c | d ---------------------------------------------------------------------- - 2 | 3 | 4 | 3 | 4 | 2 | 3 | 4 - 3 | 4 | 5 | 4 | 5 | 3 | 4 | 5 - 4 | 5 | 6 | 5 | 6 | 4 | 5 | 6 -(3 rows) - -END; --- copy always happens via distributed execution irrespective of the --- shards that are accessed -COPY reference_table FROM STDIN; -COPY distributed_table FROM STDIN WITH CSV; -COPY second_distributed_table FROM STDIN WITH CSV; --- the behaviour in transaction blocks is the following: - -- (a) Unless the first query is a local query, always use distributed execution. - -- (b) If the executor has used local execution, it has to use local execution - -- for the remaining of the transaction block. If that's not possible, the - -- executor has to error out --- rollback should be able to rollback local execution -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - -ROLLBACK; --- make sure that the value is rollbacked -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - --- rollback should be able to rollback both the local and distributed executions -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 29 | 20 -(1 row) - - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table - SELECT count(*) FROM second_distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500005 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500006 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500007 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500008 second_distributed_table WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -ROLLBACK; --- make sure that everything is rollbacked -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 22 | 20 -(1 row) - -SELECT count(*) FROM second_distributed_table; - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT * FROM second_distributed_table ORDER BY 1; - key | value ---------------------------------------------------------------------- - 1 | 1 - 6 | '6' -(2 rows) - --- very simple examples, an SELECTs should see the modifications --- that has done before -BEGIN; - -- INSERT is executed locally - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '23' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '23'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- since the INSERT is executed locally, the SELECT should also be - -- executed locally and see the changes - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- multi-shard SELECTs are now forced to use local execution on - -- the shards that reside on this node - SELECT * FROM distributed_table WHERE value = '23' ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - key | value | age ---------------------------------------------------------------------- - 1 | 23 | 20 -(1 row) - - -- similarly, multi-shard modifications should use local exection - -- on the shards that reside on this node - DELETE FROM distributed_table WHERE value = '23'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - -- make sure that the value is deleted - SELECT * FROM distributed_table WHERE value = '23' ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (value OPERATOR(pg_catalog.=) '23'::text) - key | value | age ---------------------------------------------------------------------- -(0 rows) - -COMMIT; --- make sure that we've committed everything -SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - --- if we start with a distributed execution, we should keep --- using that and never switch back to local execution -BEGIN; - DELETE FROM distributed_table WHERE value = '11'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (value OPERATOR(pg_catalog.=) '11'::text) - -- although this command could have been executed - -- locally, it is not going to be executed locally - SELECT * FROM distributed_table WHERE key = 1 ORDER BY 1,2,3; -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) ORDER BY key, value, age - key | value | age ---------------------------------------------------------------------- -(0 rows) - - -- but we can still execute parallel queries, even if - -- they are utility commands - TRUNCATE distributed_table CASCADE; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE - -- TRUNCATE didn't cascade into second_distributed_table - SELECT count(*) FROM second_distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500005 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500006 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500007 second_distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500008 second_distributed_table WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -ROLLBACK; --- load some data -INSERT INTO reference_table SELECT i FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx --- show that complex tx blocks work fine -BEGIN; - INSERT INTO reference_table VALUES (701); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (701) - INSERT INTO distributed_table VALUES (701, '701', 701); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 (key, value, age) VALUES (701, '701'::text, 701) - INSERT INTO second_distributed_table VALUES (701, '701'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.second_distributed_table_1500005 (key, value) VALUES (701, '701'::text) - DELETE FROM reference_table WHERE key = 701; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.reference_table_1500000 reference_table WHERE (key OPERATOR(pg_catalog.=) 701) - SELECT count(*) FROM distributed_table WHERE key = 701; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 701) - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT count(*) FROM second_distributed_table WHERE key = 701; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.second_distributed_table_1500005 second_distributed_table WHERE (key OPERATOR(pg_catalog.=) 701) - count ---------------------------------------------------------------------- - 1 -(1 row) - - -- multi-shard commands should also see the changes - SELECT count(*) FROM distributed_table WHERE key > 700; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (key OPERATOR(pg_catalog.>) 700) - count ---------------------------------------------------------------------- - 1 -(1 row) - - -- we can still do multi-shard commands - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table -ROLLBACK; --- multiple queries hitting different shards can be executed locally -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - SELECT count(*) FROM distributed_table WHERE key = 6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT count(*) FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - count ---------------------------------------------------------------------- - 0 -(1 row) - -ROLLBACK; --- a local query followed by TRUNCATE command can be executed locally -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - TRUNCATE distributed_table CASCADE; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -ROLLBACK; --- a local query is followed by an INSERT..SELECT via the coordinator -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - - INSERT INTO distributed_table (key) SELECT i FROM generate_series(1,1) i; -NOTICE: executing the copy locally for shard xxxxx -ROLLBACK; -BEGIN; -SET citus.enable_repartition_joins TO ON; -SET citus.enable_unique_job_ids TO off; -SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT count(*) FROM distributed_table d1 join distributed_table d2 using(age); -NOTICE: executing the command locally: SELECT partition_index, 'repartition_69_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_69_1','SELECT age AS column1 FROM local_shard_execution_replicated.distributed_table_1500001 d1 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_69_3' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_69_3','SELECT age AS column1 FROM local_shard_execution_replicated.distributed_table_1500003 d1 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_70_1' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_70_1','SELECT age AS column1 FROM local_shard_execution_replicated.distributed_table_1500001 d2 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartition_70_3' || '_' || partition_index::text , rows_written FROM pg_catalog.worker_partition_query_result('repartition_70_3','SELECT age AS column1 FROM local_shard_execution_replicated.distributed_table_1500003 d2 WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true,true,true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_1_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_2_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_3_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_4_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_0']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_1_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_2_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_3_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_4_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_1']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_1']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_1_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_2_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_3_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_4_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_2']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_2']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_1_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_2_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_3_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_69_4_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_1_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_2_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_3_3']::text[],'localhost',57637) bytes -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartition_70_4_3']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_69_1_0,repartition_69_2_0,repartition_69_3_0,repartition_69_4_0}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_70_1_0,repartition_70_2_0,repartition_70_3_0,repartition_70_4_0}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_69_1_1,repartition_69_2_1,repartition_69_3_1,repartition_69_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_70_1_1,repartition_70_2_1,repartition_70_3_1,repartition_70_4_1}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_69_1_2,repartition_69_2_2,repartition_69_3_2,repartition_69_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_70_1_2,repartition_70_2_2,repartition_70_3_2,repartition_70_4_2}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM (read_intermediate_results('{repartition_69_1_3,repartition_69_2_3,repartition_69_3_3,repartition_69_4_3}'::text[], 'binary'::citus_copy_format) intermediate_result(column1 bigint) JOIN read_intermediate_results('{repartition_70_1_3,repartition_70_2_3,repartition_70_3_3,repartition_70_4_3}'::text[], 'binary'::citus_copy_format) intermediate_result_1(column1 bigint) ON ((intermediate_result.column1 OPERATOR(pg_catalog.=) intermediate_result_1.column1))) WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - -ROLLBACK; --- a local query is followed by an INSERT..SELECT with re-partitioning -BEGIN; - SELECT count(*) FROM distributed_table WHERE key = 6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 1 -(1 row) - - INSERT INTO reference_table (key) SELECT -key FROM distributed_table; -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true -NOTICE: executing the copy locally for shard xxxxx - INSERT INTO distributed_table (key) SELECT -key FROM distributed_table; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1500001_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1500001_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1500002_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1500002_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1500003_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1500003_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_1500004_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_1500004_to','SELECT (OPERATOR(pg_catalog.-) key) AS key FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key) SELECT key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1500003_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 AS citus_table_alias (key) SELECT key FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1500004_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer) - SELECT count(*) FROM distributed_table WHERE key = -6; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) '-6'::integer) - count ---------------------------------------------------------------------- - 1 -(1 row) - -ROLLBACK; -INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 -(1 row) - -BEGIN; - DELETE FROM distributed_table WHERE key = 1; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - EXPLAIN ANALYZE DELETE FROM distributed_table WHERE key = 1; -ERROR: cannot execute command because a local execution has accessed a placement in the transaction -DETAIL: Some parallel commands cannot be executed if a previous command has already been executed locally -HINT: Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;" -ROLLBACK; -BEGIN; - INSERT INTO distributed_table VALUES (11, '111',29) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 AS citus_table_alias (key, value, age) VALUES (11, '111'::text, 29) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 11 | 29 | 121 -(1 row) - - -- this is already disallowed on the nodes, adding it in case we - -- support DDLs from the worker nodes in the future - ALTER TABLE distributed_table ADD COLUMN x INT; -ERROR: operation is not allowed on this node -HINT: Connect to the coordinator and run it again. -ROLLBACK; -BEGIN; - INSERT INTO distributed_table VALUES (11, '111',29) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 AS citus_table_alias (key, value, age) VALUES (11, '111'::text, 29) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 11 | 29 | 121 -(1 row) - - -- this is already disallowed because VACUUM cannot be executed in tx block - -- adding in case this is supported some day - VACUUM second_distributed_table; -ERROR: VACUUM cannot run inside a transaction block -ROLLBACK; --- make sure that functions can use local execution -SET citus.enable_metadata_sync TO OFF; -CREATE OR REPLACE PROCEDURE only_local_execution() AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = 1; - DELETE FROM distributed_table WHERE key = 1; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution(); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function only_local_execution() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "DELETE FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution() line XX at SQL statement --- insert a row that we need in the next tests -INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text --- make sure that functions can use local execution -CREATE OR REPLACE PROCEDURE only_local_execution_with_function_evaluation() AS $$ - DECLARE nodeId INT; - BEGIN - -- fast path router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table WHERE key = 1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - - -- regular router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = 1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_function_evaluation(); -NOTICE: executing the command locally: SELECT local_shard_execution_replicated.get_local_node_id_volatile() AS get_local_node_id_volatile FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table WHERE key = 1" -PL/pgSQL function only_local_execution_with_function_evaluation() line XX at SQL statement -NOTICE: executing the command locally: SELECT local_shard_execution_replicated.get_local_node_id_volatile() AS get_local_node_id_volatile FROM (local_shard_execution_replicated.distributed_table_1500001 d1(key, value, age) JOIN local_shard_execution_replicated.distributed_table_1500001 d2(key, value, age) USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = 1" -PL/pgSQL function only_local_execution_with_function_evaluation() line XX at SQL statement -CREATE OR REPLACE PROCEDURE only_local_execution_with_params(int) AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES ($1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = $1; - DELETE FROM distributed_table WHERE key = $1; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_params(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES ($1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "DELETE FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_params(integer) line XX at SQL statement -CREATE OR REPLACE PROCEDURE only_local_execution_with_function_evaluation_param(int) AS $$ - DECLARE nodeId INT; - BEGIN - -- fast path router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table WHERE key = $1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - - -- regular router - SELECT get_local_node_id_volatile() INTO nodeId FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = $1; - IF nodeId <= 0 THEN - RAISE NOTICE 'unexpected node id'; - END IF; - END; -$$ LANGUAGE plpgsql; -CALL only_local_execution_with_function_evaluation_param(1); -NOTICE: executing the command locally: SELECT local_shard_execution_replicated.get_local_node_id_volatile() AS get_local_node_id_volatile FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table WHERE key = $1" -PL/pgSQL function only_local_execution_with_function_evaluation_param(integer) line XX at SQL statement -NOTICE: executing the command locally: SELECT local_shard_execution_replicated.get_local_node_id_volatile() AS get_local_node_id_volatile FROM (local_shard_execution_replicated.distributed_table_1500001 d1(key, value, age) JOIN local_shard_execution_replicated.distributed_table_1500001 d2(key, value, age) USING (key)) WHERE (d1.key OPERATOR(pg_catalog.=) $1) -CONTEXT: SQL statement "SELECT get_local_node_id_volatile() FROM distributed_table d1 JOIN distributed_table d2 USING (key) WHERE d1.key = $1" -PL/pgSQL function only_local_execution_with_function_evaluation_param(integer) line XX at SQL statement -CREATE OR REPLACE PROCEDURE local_execution_followed_by_dist() AS $$ - DECLARE cnt INT; - BEGIN - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'; - SELECT count(*) INTO cnt FROM distributed_table WHERE key = 1; - DELETE FROM distributed_table; - SELECT count(*) INTO cnt FROM distributed_table; - END; -$$ LANGUAGE plpgsql; -RESET citus.enable_metadata_sync; -CALL local_execution_followed_by_dist(); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text -CONTEXT: SQL statement "INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29'" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table WHERE key = 1" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table -CONTEXT: SQL statement "DELETE FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true -CONTEXT: SQL statement "SELECT count(*) FROM distributed_table" -PL/pgSQL function local_execution_followed_by_dist() line XX at SQL statement --- test CTEs, including modifying CTEs -WITH local_insert AS (INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *), -distributed_local_mixed AS (SELECT * FROM reference_table WHERE key IN (SELECT key FROM local_insert)) -SELECT * FROM local_insert, distributed_local_mixed; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: SELECT key FROM local_shard_execution_replicated.reference_table_1500000 reference_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT local_insert.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert)) -NOTICE: executing the command locally: SELECT local_insert.key, local_insert.value, local_insert.age, distributed_local_mixed.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT intermediate_result.key FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) distributed_local_mixed - key | value | age | key ---------------------------------------------------------------------- - 1 | 11 | 21 | 1 -(1 row) - --- since we start with parallel execution, we do not switch back to local execution in the --- latter CTEs -WITH distributed_local_mixed AS (SELECT * FROM distributed_table), -local_insert AS (INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '29' RETURNING *) -SELECT * FROM local_insert, distributed_local_mixed ORDER BY 1,2,3,4,5; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '29'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table) distributed_local_mixed) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table) distributed_local_mixed) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table) distributed_local_mixed) worker_subquery -NOTICE: executing the command locally: SELECT worker_column_1 AS key, worker_column_2 AS value, worker_column_3 AS age, worker_column_4 AS key, worker_column_5 AS value, worker_column_6 AS age FROM (SELECT local_insert.key AS worker_column_1, local_insert.value AS worker_column_2, local_insert.age AS worker_column_3, distributed_local_mixed.key AS worker_column_4, distributed_local_mixed.value AS worker_column_5, distributed_local_mixed.age AS worker_column_6 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) local_insert, (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table) distributed_local_mixed) worker_subquery - key | value | age | key | value | age ---------------------------------------------------------------------- - 1 | 29 | 21 | 1 | 11 | 21 -(1 row) - --- router CTE pushdown -WITH all_data AS (SELECT * FROM distributed_table WHERE key = 1) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.key AND distributed_table.key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table, (SELECT distributed_table_1.key, distributed_table_1.value, distributed_table_1.age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) - count ---------------------------------------------------------------------- - 1 -(1 row) - -INSERT INTO reference_table VALUES (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (2) -INSERT INTO distributed_table VALUES (2, '29', 29); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 (key, value, age) VALUES (2, '29'::text, 29) -INSERT INTO second_distributed_table VALUES (2, '29'); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.second_distributed_table_1500008 (key, value) VALUES (2, '29'::text) --- single shard that is not a local query followed by a local query -WITH all_data AS (SELECT * FROM second_distributed_table WHERE key = 2) -SELECT - distributed_table.key -FROM - distributed_table, all_data -WHERE - distributed_table.value = all_data.value AND distributed_table.key = 1 -ORDER BY - 1 DESC; -NOTICE: executing the command locally: SELECT key, value FROM local_shard_execution_replicated.second_distributed_table_1500008 second_distributed_table WHERE (key OPERATOR(pg_catalog.=) 2) -NOTICE: executing the command locally: SELECT distributed_table.key FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table, (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) all_data WHERE ((distributed_table.value OPERATOR(pg_catalog.=) all_data.value) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) ORDER BY distributed_table.key DESC - key ---------------------------------------------------------------------- - 1 -(1 row) - --- multi-shard CTE is followed by a query which could be executed locally, but --- since the query started with a parallel query, it doesn't use local execution --- note that if we allow Postgres to inline the CTE (e.g., not have the EXISTS --- subquery), then it'd pushdown the filters and the query becomes single-shard, --- locally executable query -WITH all_data AS (SELECT * FROM distributed_table) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.key AND distributed_table.key = 1 - AND EXISTS (SELECT * FROM all_data); -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT key, value, age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.key) AND (distributed_table.key OPERATOR(pg_catalog.=) 1) AND (EXISTS (SELECT all_data_1.key, all_data_1.value, all_data_1.age FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, age bigint)) all_data_1))) - count ---------------------------------------------------------------------- - 1 -(1 row) - --- in pg12, the following CTE can be inlined, still the query becomes --- a subquery that needs to be recursively planned and a parallel --- query, so do not use local execution -WITH all_data AS (SELECT age FROM distributed_table) -SELECT - count(*) -FROM - distributed_table, all_data -WHERE - distributed_table.key = all_data.age AND distributed_table.key = 1; -NOTICE: executing the command locally: SELECT age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT age FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT age FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table, (SELECT intermediate_result.age FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(age bigint)) all_data WHERE ((distributed_table.key OPERATOR(pg_catalog.=) all_data.age) AND (distributed_table.key OPERATOR(pg_catalog.=) 1)) - count ---------------------------------------------------------------------- - 0 -(1 row) - --- get ready for the next commands -TRUNCATE reference_table, distributed_table, second_distributed_table; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.reference_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.second_distributed_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.second_distributed_table_xxxxx CASCADE --- local execution of returning of reference tables -INSERT INTO reference_table VALUES (1),(2),(3),(4),(5),(6) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 AS citus_table_alias (key) VALUES (1), (2), (3), (4), (5), (6) RETURNING citus_table_alias.key - key ---------------------------------------------------------------------- - 1 - 2 - 3 - 4 - 5 - 6 -(6 rows) - --- local execution of multi-row INSERTs -INSERT INTO distributed_table VALUES (1, '11',21), (5,'55',22) ON CONFLICT(key) DO UPDATE SET value = (EXCLUDED.value::int + 1)::text RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'11'::text,'21'::bigint), (5,'55'::text,'22'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 - 5 | 55 | 22 -(2 rows) - --- distributed execution of multi-rows INSERTs, where executor --- is smart enough to execute local tasks via local execution -INSERT INTO distributed_table VALUES (1, '11',21), (2,'22',22), (3,'33',33), (4,'44',44),(5,'55',55) ON CONFLICT(key) DO UPDATE SET value = (EXCLUDED.value::int + 1)::text RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'11'::text,'21'::bigint), (5,'55'::text,'55'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500002 AS citus_table_alias (key, value, age) VALUES (3,'33'::text,'33'::bigint), (4,'44'::text,'44'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 AS citus_table_alias (key, value, age) VALUES (2,'22'::text,'22'::bigint) ON CONFLICT(key) DO UPDATE SET value = (((excluded.value)::integer OPERATOR(pg_catalog.+) 1))::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 12 | 21 - 2 | 22 | 22 - 3 | 33 | 33 - 4 | 44 | 44 - 5 | 56 | 22 -(5 rows) - -PREPARE local_prepare_no_param AS SELECT count(*) FROM distributed_table WHERE key = 1; -PREPARE local_prepare_no_param_subquery AS -SELECT DISTINCT trim(value) FROM ( - SELECT value FROM distributed_table - WHERE - key IN (1, 6, 500, 701) - AND (select 2) > random() - order by 1 - limit 2 - ) t; -PREPARE local_prepare_param (int) AS SELECT count(*) FROM distributed_table WHERE key = $1; -PREPARE remote_prepare_param (int) AS SELECT count(*) FROM distributed_table WHERE key != $1; -BEGIN; - -- 8 local execution without params - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_no_param; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - -- 8 local execution without params and some subqueries - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - EXECUTE local_prepare_no_param_subquery; -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT worker_column_1 AS value FROM (SELECT distributed_table.value AS worker_column_1 FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE ((distributed_table.key OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6, 500, 701])) AND (((SELECT 2))::double precision OPERATOR(pg_catalog.>) random()))) worker_subquery ORDER BY worker_column_1 LIMIT '2'::bigint -NOTICE: executing the command locally: SELECT DISTINCT TRIM(BOTH FROM value) AS btrim FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) t - btrim ---------------------------------------------------------------------- - 12 -(1 row) - - -- 8 local executions with params - EXECUTE local_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(5); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 5) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(5); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.=) 5) - count ---------------------------------------------------------------------- - 1 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - EXECUTE local_prepare_param(6); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 6) - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- followed by a non-local execution - EXECUTE remote_prepare_param(1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 1) - count ---------------------------------------------------------------------- - 4 -(1 row) - -COMMIT; -PREPARE local_insert_prepare_no_param AS INSERT INTO distributed_table VALUES (1+0*random(), '11',21::int) ON CONFLICT(key) DO UPDATE SET value = '29' || '28' RETURNING *, key + 1, value || '30', age * 15; -PREPARE local_insert_prepare_param (int) AS INSERT INTO distributed_table VALUES ($1+0*random(), '11',21::int) ON CONFLICT(key) DO UPDATE SET value = '29' || '28' RETURNING *, key + 1, value || '30', age * 15; -BEGIN; - -- 8 local execution without params - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - -- 8 local executions with params - EXECUTE local_insert_prepare_param(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 5 | 2928 | 22 | 6 | 292830 | 330 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 11 | 21 | 7 | 1130 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 1 | 2928 | 21 | 2 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 5 | 2928 | 22 | 6 | 292830 | 330 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - EXECUTE local_insert_prepare_param(6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6, '11'::text, '21'::bigint) ON CONFLICT(key) DO UPDATE SET value = '2928'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age, (citus_table_alias.key OPERATOR(pg_catalog.+) 1), (citus_table_alias.value OPERATOR(pg_catalog.||) '30'::text), (citus_table_alias.age OPERATOR(pg_catalog.*) 15) - key | value | age | ?column? | ?column? | ?column? ---------------------------------------------------------------------- - 6 | 2928 | 21 | 7 | 292830 | 315 -(1 row) - - -- followed by a non-local execution - EXECUTE remote_prepare_param(2); -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (key OPERATOR(pg_catalog.<>) 2) - count ---------------------------------------------------------------------- - 5 -(1 row) - -COMMIT; -PREPARE local_multi_row_insert_prepare_no_param AS - INSERT INTO distributed_table VALUES (1,'55', 21), (5,'15',33) ON CONFLICT (key) WHERE key > 3 and key < 4 DO UPDATE SET value = '88' || EXCLUDED.value; -PREPARE local_multi_row_insert_prepare_no_param_multi_shard AS - INSERT INTO distributed_table VALUES (6,'55', 21), (5,'15',33) ON CONFLICT (key) WHERE key > 3 AND key < 4 DO UPDATE SET value = '88' || EXCLUDED.value;; -PREPARE local_multi_row_insert_prepare_params(int,int) AS - INSERT INTO distributed_table VALUES ($1,'55', 21), ($2,'15',33) ON CONFLICT (key) WHERE key > 3 and key < 4 DO UPDATE SET value = '88' || EXCLUDED.value;; -INSERT INTO reference_table VALUES (11); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (11) -BEGIN; - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_no_param_multi_shard; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(6,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint), (1,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(5,1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint), (1,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,6); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500003 AS citus_table_alias (key, value, age) VALUES (6,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - EXECUTE local_multi_row_insert_prepare_params(1,5); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1,'55'::text,'21'::bigint), (5,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) - -- one task is remote - EXECUTE local_multi_row_insert_prepare_params(5,11); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (5,'55'::text,'21'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 AS citus_table_alias (key, value, age) VALUES (11,'15'::text,'33'::bigint) ON CONFLICT(key) WHERE ((key OPERATOR(pg_catalog.>) 3) AND (key OPERATOR(pg_catalog.<) 4)) DO UPDATE SET value = ('88'::text OPERATOR(pg_catalog.||) excluded.value) -ROLLBACK; --- failures of local execution should rollback both the --- local execution and remote executions --- fail on a local execution -BEGIN; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 100 | 21 -(1 row) - - UPDATE distributed_table SET value = '200'; -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.distributed_table_1500001 distributed_table SET value = '200'::text -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.distributed_table_1500002 distributed_table SET value = '200'::text -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.distributed_table_1500003 distributed_table SET value = '200'::text -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.distributed_table_1500004 distributed_table SET value = '200'::text - INSERT INTO distributed_table VALUES (1, '100',21) ON CONFLICT(key) DO UPDATE SET value = (1 / (100.0 - EXCLUDED.value::int))::text RETURNING *; -ERROR: division by zero -CONTEXT: while executing command on localhost:xxxxx -ROLLBACK; --- we've rollbacked everything -SELECT count(*) FROM distributed_table WHERE value = '200'; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- RETURNING should just work fine for reference tables -INSERT INTO reference_table VALUES (500) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - -DELETE FROM reference_table WHERE key = 500 RETURNING *; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.reference_table_1500000 reference_table WHERE (key OPERATOR(pg_catalog.=) 500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - --- should be able to skip local execution even if in a sequential mode of execution -BEGIN; - SET LOCAL citus.multi_shard_modify_mode TO sequential ; - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 11 | 21 -(1 row) - -ROLLBACK; --- sequential execution should just work fine after a local execution -BEGIN; - SET citus.multi_shard_modify_mode TO sequential ; - INSERT INTO distributed_table VALUES (1, '11',21) ON CONFLICT(key) DO UPDATE SET value = '100' RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 AS citus_table_alias (key, value, age) VALUES (1, '11'::text, 21) ON CONFLICT(key) DO UPDATE SET value = '100'::text RETURNING citus_table_alias.key, citus_table_alias.value, citus_table_alias.age - key | value | age ---------------------------------------------------------------------- - 1 | 100 | 21 -(1 row) - - DELETE FROM distributed_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table -ROLLBACK; --- load some data so that foreign keys won't complain with the next tests -TRUNCATE reference_table CASCADE; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.reference_table_xxxxx CASCADE -INSERT INTO reference_table SELECT i FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx -INSERT INTO distributed_table SELECT i, i::text, i % 10 + 25 FROM generate_series(500, 600) i; -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx --- show that both local, and mixed local-distributed executions --- calculate rows processed correctly -BEGIN; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - DELETE FROM distributed_table WHERE value != '123123213123213'; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE (value OPERATOR(pg_catalog.<>) '123123213123213'::text) -ROLLBACK; -BEGIN; - DELETE FROM reference_table WHERE key = 500 RETURNING *; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.reference_table_1500000 reference_table WHERE (key OPERATOR(pg_catalog.=) 500) RETURNING key - key ---------------------------------------------------------------------- - 500 -(1 row) - - DELETE FROM reference_table; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.reference_table_1500000 reference_table -ROLLBACK; -BEGIN; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true - count ---------------------------------------------------------------------- - 106 -(1 row) - -ROLLBACK; -BEGIN; - SET LOCAL client_min_messages TO INFO; - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true - count ---------------------------------------------------------------------- - 107 -(1 row) - - SET LOCAL client_min_messages TO LOG; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -ROLLBACK; --- probably not a realistic case since views are not very --- well supported with MX -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW v_local_query_execution AS -SELECT * FROM distributed_table WHERE key = 500; -RESET citus.enable_ddl_propagation; -SELECT * FROM v_local_query_execution; -NOTICE: executing the command locally: SELECT key, value, age FROM (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (distributed_table.key OPERATOR(pg_catalog.=) 500)) v_local_query_execution - key | value | age ---------------------------------------------------------------------- - 500 | 500 | 25 -(1 row) - --- similar test, but this time the view itself is a non-local --- query, but the query on the view is local -SET citus.enable_ddl_propagation TO OFF; -CREATE VIEW v_local_query_execution_2 AS -SELECT * FROM distributed_table; -RESET citus.enable_ddl_propagation; -SELECT * FROM v_local_query_execution_2 WHERE key = 500; -NOTICE: executing the command locally: SELECT key, value, age FROM (SELECT distributed_table.key, distributed_table.value, distributed_table.age FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table) v_local_query_execution_2 WHERE (key OPERATOR(pg_catalog.=) 500) - key | value | age ---------------------------------------------------------------------- - 500 | 500 | 25 -(1 row) - --- even if we switch from remote execution -> local execution, --- we are able to use remote execution after rollback -BEGIN; - SAVEPOINT my_savepoint; - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true - count ---------------------------------------------------------------------- - 107 -(1 row) - - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - ROLLBACK TO SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -COMMIT; --- even if we switch from local execution -> remote execution, --- we are able to use local execution after rollback -BEGIN; - SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) - SELECT count(*) FROM distributed_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table WHERE true - count ---------------------------------------------------------------------- - 106 -(1 row) - - ROLLBACK TO SAVEPOINT my_savepoint; - DELETE FROM distributed_table WHERE key = 500; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) -COMMIT; --- sanity check: local execution on partitions -INSERT INTO collections_list (collection_id) VALUES (0) RETURNING *; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500011 (key, ser, collection_id) VALUES ('3940649673949185'::bigint, '3940649673949185'::bigint, 0) RETURNING key, ser, ts, collection_id, value - key | ser | ts | collection_id | value ---------------------------------------------------------------------- - 3940649673949185 | 3940649673949185 | | 0 | -(1 row) - -BEGIN; - INSERT INTO collections_list (key, collection_id) VALUES (1,0); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500009 (key, ser, collection_id) VALUES ('1'::bigint, '3940649673949186'::bigint, 0) - SELECT count(*) FROM collections_list_0 WHERE key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.collections_list_0_1500013 collections_list_0 WHERE (key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT count(*) FROM collections_list; -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.collections_list_1500009 collections_list WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.collections_list_1500010 collections_list WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.collections_list_1500011 collections_list WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.collections_list_1500012 collections_list WHERE true - count ---------------------------------------------------------------------- - 2 -(1 row) - - SELECT * FROM collections_list ORDER BY 1,2,3,4; -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution_replicated.collections_list_1500009 collections_list WHERE true -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution_replicated.collections_list_1500010 collections_list WHERE true -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution_replicated.collections_list_1500011 collections_list WHERE true -NOTICE: executing the command locally: SELECT key, ser, ts, collection_id, value FROM local_shard_execution_replicated.collections_list_1500012 collections_list WHERE true - key | ser | ts | collection_id | value ---------------------------------------------------------------------- - 1 | 3940649673949186 | | 0 | - 3940649673949185 | 3940649673949185 | | 0 | -(2 rows) - -COMMIT; -TRUNCATE collections_list; --- make sure that even if local execution is used, the sequence values --- are generated locally -SET citus.enable_ddl_propagation TO OFF; -ALTER SEQUENCE collections_list_key_seq NO MINVALUE NO MAXVALUE; -RESET citus.enable_ddl_propagation; -PREPARE serial_prepared_local AS INSERT INTO collections_list (collection_id) VALUES (0) RETURNING key, ser; -SELECT setval('collections_list_key_seq', 4); - setval ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500009 (key, ser, collection_id) VALUES ('5'::bigint, '3940649673949187'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 5 | 3940649673949187 -(1 row) - -SELECT setval('collections_list_key_seq', 5); - setval ---------------------------------------------------------------------- - 5 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500011 (key, ser, collection_id) VALUES ('6'::bigint, '3940649673949188'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 6 | 3940649673949188 -(1 row) - -SELECT setval('collections_list_key_seq', 499); - setval ---------------------------------------------------------------------- - 499 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500011 (key, ser, collection_id) VALUES ('500'::bigint, '3940649673949189'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 500 | 3940649673949189 -(1 row) - -SELECT setval('collections_list_key_seq', 700); - setval ---------------------------------------------------------------------- - 700 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500009 (key, ser, collection_id) VALUES ('701'::bigint, '3940649673949190'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 701 | 3940649673949190 -(1 row) - -SELECT setval('collections_list_key_seq', 708); - setval ---------------------------------------------------------------------- - 708 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500011 (key, ser, collection_id) VALUES ('709'::bigint, '3940649673949191'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 709 | 3940649673949191 -(1 row) - -SELECT setval('collections_list_key_seq', 709); - setval ---------------------------------------------------------------------- - 709 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500009 (key, ser, collection_id) VALUES ('710'::bigint, '3940649673949192'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 710 | 3940649673949192 -(1 row) - --- get ready for the next executions -DELETE FROM collections_list WHERE key IN (5,6); -SELECT setval('collections_list_key_seq', 4); - setval ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500009 (key, ser, collection_id) VALUES ('5'::bigint, '3940649673949193'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 5 | 3940649673949193 -(1 row) - -SELECT setval('collections_list_key_seq', 5); - setval ---------------------------------------------------------------------- - 5 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500011 (key, ser, collection_id) VALUES ('6'::bigint, '3940649673949194'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 6 | 3940649673949194 -(1 row) - --- and, one remote test -SELECT setval('collections_list_key_seq', 10); - setval ---------------------------------------------------------------------- - 10 -(1 row) - -EXECUTE serial_prepared_local; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.collections_list_1500012 (key, ser, collection_id) VALUES ('11'::bigint, '3940649673949195'::bigint, 0) RETURNING key, ser - key | ser ---------------------------------------------------------------------- - 11 | 3940649673949195 -(1 row) - --- the final queries for the following CTEs are going to happen on the intermediate results only --- one of them will be executed remotely, and the other is locally --- Citus currently doesn't allow using task_assignment_policy for intermediate results -WITH distributed_local_mixed AS (INSERT INTO reference_table VALUES (1000) RETURNING *) SELECT * FROM distributed_local_mixed; -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (1000) RETURNING key -NOTICE: executing the command locally: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) distributed_local_mixed - key ---------------------------------------------------------------------- - 1000 -(1 row) - --- clean the table for the next tests -SET search_path TO local_shard_execution_replicated; -TRUNCATE distributed_table CASCADE; --- load some data on a remote shard -INSERT INTO reference_table (key) VALUES (1), (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 AS citus_table_alias (key) VALUES (1), (2) -INSERT INTO distributed_table (key) VALUES (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500004 (key) VALUES (2) -BEGIN; - -- local execution followed by a distributed query - INSERT INTO distributed_table (key) VALUES (1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.distributed_table_1500001 (key) VALUES (1) - DELETE FROM distributed_table RETURNING key; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table RETURNING key -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500002 distributed_table RETURNING key -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500003 distributed_table RETURNING key -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.distributed_table_1500004 distributed_table RETURNING key - key ---------------------------------------------------------------------- - 1 - 2 -(2 rows) - -COMMIT; --- a similar test with a reference table -TRUNCATE reference_table CASCADE; -NOTICE: executing the command locally: TRUNCATE TABLE local_shard_execution_replicated.reference_table_xxxxx CASCADE --- load some data on a remote shard -INSERT INTO reference_table (key) VALUES (2); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (2) -BEGIN; - -- local execution followed by a distributed query - INSERT INTO reference_table (key) VALUES (1); -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.reference_table_1500000 (key) VALUES (1) - DELETE FROM reference_table RETURNING key; -NOTICE: executing the command locally: DELETE FROM local_shard_execution_replicated.reference_table_1500000 reference_table RETURNING key - key ---------------------------------------------------------------------- - 1 - 2 -(2 rows) - -COMMIT; --- however complex the query, local execution can handle -SET client_min_messages TO LOG; -SET citus.log_local_commands TO ON; -WITH cte_1 AS - (SELECT * - FROM - (WITH cte_1 AS - (SELECT * - FROM distributed_table - WHERE key = 1) SELECT * - FROM cte_1) AS foo) -SELECT count(*) -FROM cte_1 -JOIN distributed_table USING (key) -WHERE distributed_table.key = 1 - AND distributed_table.key IN - (SELECT key - FROM distributed_table - WHERE key = 1); -NOTICE: executing the command locally: SELECT count(*) AS count FROM ((SELECT foo.key, foo.value, foo.age FROM (SELECT cte_1_1.key, cte_1_1.value, cte_1_1.age FROM (SELECT distributed_table_1.key, distributed_table_1.value, distributed_table_1.age FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)) cte_1_1) foo) cte_1 JOIN local_shard_execution_replicated.distributed_table_1500001 distributed_table(key, value, age) USING (key)) WHERE ((distributed_table.key OPERATOR(pg_catalog.=) 1) AND (distributed_table.key OPERATOR(pg_catalog.=) ANY (SELECT distributed_table_1.key FROM local_shard_execution_replicated.distributed_table_1500001 distributed_table_1 WHERE (distributed_table_1.key OPERATOR(pg_catalog.=) 1)))) - count ---------------------------------------------------------------------- - 0 -(1 row) - -RESET client_min_messages; -RESET citus.log_local_commands; -\c - - - :master_port -SET citus.next_shard_id TO 1501000; --- test both local and remote execution with custom type -SET citus.shard_replication_factor TO 2; -SET search_path TO local_shard_execution_replicated; -CREATE TYPE invite_resp AS ENUM ('yes', 'no', 'maybe'); -CREATE TABLE event_responses ( - event_id int, - user_id int, - response invite_resp, - primary key (event_id, user_id) -); -SELECT create_distributed_table('event_responses', 'event_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO event_responses VALUES (1, 1, 'yes'), (2, 2, 'yes'), (3, 3, 'no'), (4, 4, 'no'); -CREATE OR REPLACE FUNCTION regular_func(p invite_resp) -RETURNS int AS $$ -DECLARE - q1Result INT; - q2Result INT; - q3Result INT; -BEGIN -SELECT count(*) INTO q1Result FROM event_responses WHERE response = $1; -SELECT count(*) INTO q2Result FROM event_responses e1 LEFT JOIN event_responses e2 USING (event_id) WHERE e2.response = $1; -SELECT count(*) INTO q3Result FROM (SELECT * FROM event_responses WHERE response = $1 LIMIT 5) as foo; -RETURN q3Result+q2Result+q1Result; -END; -$$ LANGUAGE plpgsql; -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -SELECT regular_func('yes'); - regular_func ---------------------------------------------------------------------- - 6 -(1 row) - -CREATE OR REPLACE PROCEDURE regular_procedure(p invite_resp) -AS $$ -BEGIN -PERFORM * FROM event_responses WHERE response = $1 ORDER BY 1 DESC, 2 DESC, 3 DESC; -PERFORM * FROM event_responses e1 LEFT JOIN event_responses e2 USING (event_id) WHERE e2.response = $1 ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; -PERFORM * FROM (SELECT * FROM event_responses WHERE response = $1 LIMIT 5) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC; -END; -$$ LANGUAGE plpgsql; -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -CALL regular_procedure('no'); -PREPARE multi_shard_no_dist_key(invite_resp) AS select * from event_responses where response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_no_dist_key('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -PREPARE multi_shard_with_dist_key(int, invite_resp) AS select * from event_responses where event_id > $1 AND response = $2::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -EXECUTE multi_shard_with_dist_key(1, 'yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes -(1 row) - -PREPARE query_pushdown_no_dist_key(invite_resp) AS select * from event_responses e1 LEFT JOIN event_responses e2 USING(event_id) where e1.response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC LIMIT 1; -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -EXECUTE query_pushdown_no_dist_key('yes'); - event_id | user_id | response | user_id | response ---------------------------------------------------------------------- - 2 | 2 | yes | 2 | yes -(1 row) - -PREPARE insert_select_via_coord(invite_resp) AS INSERT INTO event_responses SELECT * FROM event_responses where response = $1::invite_resp LIMIT 1 ON CONFLICT (event_id, user_id) DO NOTHING ; -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -EXECUTE insert_select_via_coord('yes'); -PREPARE insert_select_pushdown(invite_resp) AS INSERT INTO event_responses SELECT * FROM event_responses where response = $1::invite_resp ON CONFLICT (event_id, user_id) DO NOTHING; -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -EXECUTE insert_select_pushdown('yes'); -PREPARE router_select_with_no_dist_key_filter(invite_resp) AS select * from event_responses where event_id = 1 AND response = $1::invite_resp ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 1; -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - -EXECUTE router_select_with_no_dist_key_filter('yes'); - event_id | user_id | response ---------------------------------------------------------------------- - 1 | 1 | yes -(1 row) - --- rest of the tests assume the table is empty -TRUNCATE event_responses; -CREATE OR REPLACE PROCEDURE register_for_event(p_event_id int, p_user_id int, p_choice invite_resp) -LANGUAGE plpgsql AS $fn$ -BEGIN - INSERT INTO local_shard_execution_replicated.event_responses VALUES (p_event_id, p_user_id, p_choice) - ON CONFLICT (event_id, user_id) - DO UPDATE SET response = EXCLUDED.response; - - PERFORM count(*) FROM local_shard_execution_replicated.event_responses WHERE event_id = p_event_id; - - PERFORM count(*) FROM local_shard_execution_replicated.event_responses WHERE event_id = p_event_id AND false; - - UPDATE local_shard_execution_replicated.event_responses SET response = p_choice WHERE event_id = p_event_id; - -END; -$fn$; -SELECT create_distributed_function('register_for_event(int,int,invite_resp)'); -NOTICE: procedure local_shard_execution_replicated.register_for_event is already distributed -DETAIL: Citus distributes procedures with CREATE [PROCEDURE|FUNCTION|AGGREGATE] commands - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - --- call 8 times to make sure it works after the 5th time(postgres binds values after the 5th time and Citus 2nd time) --- after 6th, the local execution caches the local plans and uses it --- execute it both locally and remotely -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -\c - - - :worker_2_port -SET search_path TO local_shard_execution_replicated; -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); -CALL register_for_event(16, 1, 'yes'); --- values 16, 17 and 19 hits the same --- shard, so we're re-using the same cached --- plans per statement across different distribution --- key values -CALL register_for_event(17, 1, 'yes'); -CALL register_for_event(19, 1, 'yes'); -CALL register_for_event(17, 1, 'yes'); -CALL register_for_event(19, 1, 'yes'); --- should work fine if the logs are enabled -\set VERBOSITY terse -SET citus.log_local_commands TO ON; -SET client_min_messages TO DEBUG2; -CALL register_for_event(19, 1, 'yes'); -DEBUG: stored procedure does not have co-located tables -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.event_responses_1501001 AS citus_table_alias (event_id, user_id, response) VALUES (19, 1, 'yes'::local_shard_execution_replicated.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.event_responses_1501001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 19) -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT NULL::integer AS event_id, NULL::integer AS user_id, NULL::local_shard_execution_replicated.invite_resp AS response WHERE false) event_responses(event_id, user_id, response) WHERE ((event_id OPERATOR(pg_catalog.=) 19) AND false) -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.event_responses_1501001 event_responses SET response = 'yes'::local_shard_execution_replicated.invite_resp WHERE (event_id OPERATOR(pg_catalog.=) 19) --- should be fine even if no parameters exists in the query -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.event_responses_1501001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 16) - count ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution_replicated.event_responses_1501001 event_responses WHERE (event_id OPERATOR(pg_catalog.=) 16) - count ---------------------------------------------------------------------- - 1 -(1 row) - -UPDATE event_responses SET response = 'no' WHERE event_id = 16; -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: UPDATE local_shard_execution_replicated.event_responses_1501001 event_responses SET response = 'no'::local_shard_execution_replicated.invite_resp WHERE (event_id OPERATOR(pg_catalog.=) 16) -INSERT INTO event_responses VALUES (16, 666, 'maybe') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.event_responses_1501001 AS citus_table_alias (event_id, user_id, response) VALUES (16, 666, 'maybe'::local_shard_execution_replicated.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response RETURNING citus_table_alias.event_id, citus_table_alias.user_id, citus_table_alias.response - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe -(1 row) - --- multi row INSERTs hitting the same shard -INSERT INTO event_responses VALUES (16, 666, 'maybe'), (17, 777, 'no') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -NOTICE: executing the command locally: INSERT INTO local_shard_execution_replicated.event_responses_1501001 AS citus_table_alias (event_id, user_id, response) VALUES (16,666,'maybe'::local_shard_execution_replicated.invite_resp), (17,777,'no'::local_shard_execution_replicated.invite_resp) ON CONFLICT(event_id, user_id) DO UPDATE SET response = excluded.response RETURNING citus_table_alias.event_id, citus_table_alias.user_id, citus_table_alias.response - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe - 17 | 777 | no -(2 rows) - --- now, similar tests with some settings changed -SET citus.enable_local_execution TO false; -SET citus.enable_fast_path_router_planner TO false; -CALL register_for_event(19, 1, 'yes'); -DEBUG: stored procedure does not have co-located tables --- should be fine even if no parameters exists in the query -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - count ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT count(*) FROM event_responses WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - count ---------------------------------------------------------------------- - 2 -(1 row) - -UPDATE event_responses SET response = 'no' WHERE event_id = 16; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 -INSERT INTO event_responses VALUES (16, 666, 'maybe') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 16 - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe -(1 row) - --- multi row INSERTs hitting the same shard -INSERT INTO event_responses VALUES (16, 666, 'maybe'), (17, 777, 'no') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -DEBUG: Creating router plan - event_id | user_id | response ---------------------------------------------------------------------- - 16 | 666 | maybe - 17 | 777 | no -(2 rows) - --- not allow commands over the workers when user disables -SET citus.allow_modifications_from_workers_to_replicated_tables TO false; -INSERT INTO event_responses VALUES (16, 666, 'maybe'), (17, 777, 'no') -ON CONFLICT (event_id, user_id) -DO UPDATE SET response = EXCLUDED.response RETURNING *; -ERROR: modifications via the worker nodes are not allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1. -\c - - - :master_port -SET client_min_messages TO ERROR; -SET search_path TO public; -DROP SCHEMA local_shard_execution_replicated CASCADE; diff --git a/src/test/regress/expected/merge.out b/src/test/regress/expected/merge.out index 5056ba543..1e5e85242 100644 --- a/src/test/regress/expected/merge.out +++ b/src/test/regress/expected/merge.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS merge_schema CASCADE; NOTICE: schema "merge_schema" does not exist, skipping diff --git a/src/test/regress/expected/merge_0.out b/src/test/regress/expected/merge_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_arbitrary.out b/src/test/regress/expected/merge_arbitrary.out index b55306b44..052a9d066 100644 --- a/src/test/regress/expected/merge_arbitrary.out +++ b/src/test/regress/expected/merge_arbitrary.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif SET search_path TO merge_arbitrary_schema; INSERT INTO target_cj VALUES (1, 'target', 0); INSERT INTO target_cj VALUES (2, 'target', 0); diff --git a/src/test/regress/expected/merge_arbitrary_0.out b/src/test/regress/expected/merge_arbitrary_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_arbitrary_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_arbitrary_create.out b/src/test/regress/expected/merge_arbitrary_create.out index aff9ecd97..1d0a25f6a 100644 --- a/src/test/regress/expected/merge_arbitrary_create.out +++ b/src/test/regress/expected/merge_arbitrary_create.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif DROP SCHEMA IF EXISTS merge_arbitrary_schema CASCADE; CREATE SCHEMA merge_arbitrary_schema; SET search_path TO merge_arbitrary_schema; diff --git a/src/test/regress/expected/merge_arbitrary_create_0.out b/src/test/regress/expected/merge_arbitrary_create_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_arbitrary_create_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_partition_tables.out b/src/test/regress/expected/merge_partition_tables.out index 5ac375817..6ca7d6398 100644 --- a/src/test/regress/expected/merge_partition_tables.out +++ b/src/test/regress/expected/merge_partition_tables.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/expected/merge_partition_tables_0.out b/src/test/regress/expected/merge_partition_tables_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_partition_tables_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_repartition1.out b/src/test/regress/expected/merge_repartition1.out index 279358e30..ac718f73c 100644 --- a/src/test/regress/expected/merge_repartition1.out +++ b/src/test/regress/expected/merge_repartition1.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/expected/merge_repartition1_0.out b/src/test/regress/expected/merge_repartition1_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_repartition1_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_repartition2.out b/src/test/regress/expected/merge_repartition2.out index 898b7c77a..524ae84f7 100644 --- a/src/test/regress/expected/merge_repartition2.out +++ b/src/test/regress/expected/merge_repartition2.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/expected/merge_repartition2_0.out b/src/test/regress/expected/merge_repartition2_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_repartition2_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_schema_sharding.out b/src/test/regress/expected/merge_schema_sharding.out index 17f6f6adb..a6fb11998 100644 --- a/src/test/regress/expected/merge_schema_sharding.out +++ b/src/test/regress/expected/merge_schema_sharding.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS schema_shard_table1 CASCADE; NOTICE: schema "schema_shard_table1" does not exist, skipping diff --git a/src/test/regress/expected/merge_schema_sharding_0.out b/src/test/regress/expected/merge_schema_sharding_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_schema_sharding_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/merge_vcore.out b/src/test/regress/expected/merge_vcore.out index 0eccb811b..b3b6eb1ff 100644 --- a/src/test/regress/expected/merge_vcore.out +++ b/src/test/regress/expected/merge_vcore.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; NOTICE: schema "merge_vcore_schema" does not exist, skipping diff --git a/src/test/regress/expected/merge_vcore_0.out b/src/test/regress/expected/merge_vcore_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/merge_vcore_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out b/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out index 0c268264d..6a6251f9e 100644 --- a/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out +++ b/src/test/regress/expected/multi_alter_table_add_constraints_without_name.out @@ -214,13 +214,8 @@ SELECT con.conname \c - - :master_host :master_port ALTER TABLE AT_AddConstNoName.products DROP CONSTRAINT products_product_no_key; -- Check "ADD UNIQUE NULLS NOT DISTICT" -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 ALTER TABLE AT_AddConstNoName.products ADD UNIQUE NULLS NOT DISTINCT (product_no, price); ALTER TABLE AT_AddConstNoName.products DROP CONSTRAINT products_product_no_price_key; -\endif -- Check "ADD UNIQUE ... DEFERRABLE" ALTER TABLE AT_AddConstNoName.products ADD UNIQUE(product_no) INCLUDE(price) DEFERRABLE; \c - - :public_worker_1_host :worker_1_port diff --git a/src/test/regress/expected/multi_deparse_shard_query.out b/src/test/regress/expected/multi_deparse_shard_query.out index 4657db10d..407f89b8c 100644 --- a/src/test/regress/expected/multi_deparse_shard_query.out +++ b/src/test/regress/expected/multi_deparse_shard_query.out @@ -1,17 +1,6 @@ -- -- MULTI_DEPARSE_SHARD_QUERY -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA multi_deparse_shard_query; SET search_path TO multi_deparse_shard_query; SET citus.next_shard_id TO 13100000; diff --git a/src/test/regress/expected/multi_deparse_shard_query_0.out b/src/test/regress/expected/multi_deparse_shard_query_0.out deleted file mode 100644 index 4f2ca98b8..000000000 --- a/src/test/regress/expected/multi_deparse_shard_query_0.out +++ /dev/null @@ -1,423 +0,0 @@ --- --- MULTI_DEPARSE_SHARD_QUERY --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA multi_deparse_shard_query; -SET search_path TO multi_deparse_shard_query; -SET citus.next_shard_id TO 13100000; -SET citus.shard_replication_factor TO 1; -CREATE FUNCTION deparse_shard_query_test(text) - RETURNS VOID - AS 'citus' - LANGUAGE C STRICT; --- create the first table -CREATE TABLE raw_events_1 - (tenant_id bigint, - value_1 int, - value_2 int, - value_3 float, - value_4 bigint, - value_5 text, - value_6 int DEfAULT 10, - value_7 int, - event_at date DEfAULT now() - ); -SELECT create_distributed_table('raw_events_1', 'tenant_id', 'hash'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- create the first table -CREATE TABLE raw_events_2 - (tenant_id bigint, - value_1 int, - value_2 int, - value_3 float, - value_4 bigint, - value_5 text, - value_6 float DEfAULT (random()*100)::float, - value_7 int, - event_at date DEfAULT now() - ); -SELECT create_distributed_table('raw_events_2', 'tenant_id', 'hash'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE aggregated_events - (tenant_id bigint, - sum_value_1 bigint, - average_value_2 float, - average_value_3 float, - sum_value_4 bigint, - sum_value_5 float, - average_value_6 int, - rollup_hour date); -SELECT create_distributed_table('aggregated_events', 'tenant_id', 'hash'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- start with very simple examples on a single table -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1 -SELECT * FROM raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_1, value_2, value_3, value_4, value_5, value_6, value_7, event_at) SELECT tenant_id, value_1, value_2, value_3, value_4, value_5, value_6, value_7, event_at FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(tenant_id, value_4) -SELECT - tenant_id, value_4 -FROM - raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_4, value_6, event_at) SELECT tenant_id, value_4, 10 AS value_6, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- now that shuffle columns a bit on a single table -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(value_5, value_2, tenant_id, value_4) -SELECT - value_2::text, value_5::int, tenant_id, value_4 -FROM - raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_2, value_4, value_5, value_6, event_at) SELECT tenant_id, (value_5)::integer AS value_5, value_4, (value_2)::text AS value_2, 10 AS value_6, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- same test on two different tables -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(value_5, value_2, tenant_id, value_4) -SELECT - value_2::text, value_5::int, tenant_id, value_4 -FROM - raw_events_2; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_2, value_4, value_5, value_6, event_at) SELECT tenant_id, (value_5)::integer AS value_5, value_4, (value_2)::text AS value_2, 10 AS value_6, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_2 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- lets do some simple aggregations -SELECT deparse_shard_query_test(E' -INSERT INTO aggregated_events (tenant_id, rollup_hour, sum_value_1, average_value_3, average_value_6, sum_value_4) -SELECT - tenant_id, date_trunc(\'hour\', event_at) , sum(value_1), avg(value_3), avg(value_6), sum(value_4) -FROM - raw_events_1 -GROUP BY - tenant_id, date_trunc(\'hour\', event_at) -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_1, average_value_3, sum_value_4, average_value_6, rollup_hour) SELECT tenant_id, sum(value_1) AS sum, avg(value_3) AS avg, sum(value_4) AS sum, avg(value_6) AS avg, date_trunc('hour'::text, (event_at)::timestamp with time zone) AS date_trunc FROM multi_deparse_shard_query.raw_events_1 GROUP BY tenant_id, (date_trunc('hour'::text, (event_at)::timestamp with time zone)) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- also some subqueries, JOINS with a complicated target lists --- a simple JOIN -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1 (value_3, tenant_id) -SELECT - raw_events_2.value_3, raw_events_1.tenant_id -FROM - raw_events_1, raw_events_2 -WHERE - raw_events_1.tenant_id = raw_events_2.tenant_id; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_3, value_6, event_at) SELECT raw_events_1.tenant_id, raw_events_2.value_3, 10 AS value_6, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1, multi_deparse_shard_query.raw_events_2 WHERE (raw_events_1.tenant_id OPERATOR(pg_catalog.=) raw_events_2.tenant_id) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- join with group by -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1 (value_3, tenant_id) -SELECT - max(raw_events_2.value_3), avg(raw_events_1.value_3) -FROM - raw_events_1, raw_events_2 -WHERE - raw_events_1.tenant_id = raw_events_2.tenant_id GROUP BY raw_events_1.event_at -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_3, value_6, event_at) SELECT avg(raw_events_1.value_3) AS avg, max(raw_events_2.value_3) AS max, 10 AS value_6, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1, multi_deparse_shard_query.raw_events_2 WHERE (raw_events_1.tenant_id OPERATOR(pg_catalog.=) raw_events_2.tenant_id) GROUP BY raw_events_1.event_at - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- a more complicated JOIN -SELECT deparse_shard_query_test(' -INSERT INTO aggregated_events (sum_value_4, tenant_id) -SELECT - max(r1.value_4), r3.tenant_id -FROM - raw_events_1 r1, raw_events_2 r2, raw_events_1 r3 -WHERE - r1.tenant_id = r2.tenant_id AND r2.tenant_id = r3.tenant_id -GROUP BY - r1.value_1, r3.tenant_id, r2.event_at -ORDER BY - r2.event_at DESC; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_4) SELECT r3.tenant_id, max(r1.value_4) AS max FROM multi_deparse_shard_query.raw_events_1 r1, multi_deparse_shard_query.raw_events_2 r2, multi_deparse_shard_query.raw_events_1 r3 WHERE ((r1.tenant_id OPERATOR(pg_catalog.=) r2.tenant_id) AND (r2.tenant_id OPERATOR(pg_catalog.=) r3.tenant_id)) GROUP BY r1.value_1, r3.tenant_id, r2.event_at ORDER BY r2.event_at DESC - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- queries with CTEs are supported -SELECT deparse_shard_query_test(' -WITH first_tenant AS (SELECT event_at, value_5, tenant_id FROM raw_events_1) -INSERT INTO aggregated_events (rollup_hour, sum_value_5, tenant_id) -SELECT - event_at, sum(value_5::int), tenant_id -FROM - raw_events_1 -GROUP BY - event_at, tenant_id; -'); -INFO: query: WITH first_tenant AS (SELECT raw_events_1.event_at, raw_events_1.value_5, raw_events_1.tenant_id FROM multi_deparse_shard_query.raw_events_1) INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_5, rollup_hour) SELECT tenant_id, sum((value_5)::integer) AS sum, event_at FROM multi_deparse_shard_query.raw_events_1 GROUP BY event_at, tenant_id - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -WITH first_tenant AS (SELECT event_at, value_5, tenant_id FROM raw_events_1) -INSERT INTO aggregated_events (sum_value_5, tenant_id) -SELECT - sum(value_5::int), tenant_id -FROM - raw_events_1 -GROUP BY - event_at, tenant_id; -'); -INFO: query: WITH first_tenant AS (SELECT raw_events_1.event_at, raw_events_1.value_5, raw_events_1.tenant_id FROM multi_deparse_shard_query.raw_events_1) INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_5) SELECT tenant_id, sum((value_5)::integer) AS sum FROM multi_deparse_shard_query.raw_events_1 GROUP BY event_at, tenant_id - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -INSERT INTO aggregated_events (sum_value_1, sum_value_5, tenant_id) -WITH RECURSIVE hierarchy as ( - SELECT value_1, 1 AS LEVEL, tenant_id - FROM raw_events_1 - WHERE tenant_id = 1 - UNION - SELECT re.value_2, (h.level+1), re.tenant_id - FROM hierarchy h JOIN raw_events_1 re - ON (h.tenant_id = re.tenant_id AND - h.value_1 = re.value_6)) -SELECT * FROM hierarchy WHERE LEVEL <= 2; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_1, sum_value_5) WITH RECURSIVE hierarchy AS (SELECT raw_events_1.value_1, 1 AS level, raw_events_1.tenant_id FROM multi_deparse_shard_query.raw_events_1 WHERE (raw_events_1.tenant_id OPERATOR(pg_catalog.=) 1) UNION SELECT re.value_2, (h.level OPERATOR(pg_catalog.+) 1), re.tenant_id FROM (hierarchy h JOIN multi_deparse_shard_query.raw_events_1 re ON (((h.tenant_id OPERATOR(pg_catalog.=) re.tenant_id) AND (h.value_1 OPERATOR(pg_catalog.=) re.value_6))))) SELECT tenant_id, value_1, level FROM hierarchy WHERE (level OPERATOR(pg_catalog.<=) 2) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -INSERT INTO aggregated_events (sum_value_1) -SELECT - DISTINCT value_1 -FROM - raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (sum_value_1) SELECT DISTINCT value_1 FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- many filters suffled -SELECT deparse_shard_query_test(E' -INSERT INTO aggregated_events (sum_value_5, sum_value_1, tenant_id) -SELECT value_3, value_2, tenant_id - FROM raw_events_1 - WHERE (value_5 like \'%s\' or value_5 like \'%a\') and (tenant_id = 1) and (value_6 < 3000 or value_3 > 8000); -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_1, sum_value_5) SELECT tenant_id, value_2, value_3 FROM multi_deparse_shard_query.raw_events_1 WHERE (((value_5 OPERATOR(pg_catalog.~~) '%s'::text) OR (value_5 OPERATOR(pg_catalog.~~) '%a'::text)) AND (tenant_id OPERATOR(pg_catalog.=) 1) AND ((value_6 OPERATOR(pg_catalog.<) 3000) OR (value_3 OPERATOR(pg_catalog.>) (8000)::double precision))) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(E' -INSERT INTO aggregated_events (sum_value_5, tenant_id) -SELECT rank() OVER (PARTITION BY tenant_id ORDER BY value_6), tenant_id - FROM raw_events_1 - WHERE event_at = now(); -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_5) SELECT tenant_id, rank() OVER (PARTITION BY tenant_id ORDER BY value_6) AS rank FROM multi_deparse_shard_query.raw_events_1 WHERE (event_at OPERATOR(pg_catalog.=) now()) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(E' -INSERT INTO aggregated_events (sum_value_5, tenant_id, sum_value_4) -SELECT random(), int4eq(1, max(value_1))::int, value_6 - FROM raw_events_1 - WHERE event_at = now() - GROUP BY event_at, value_7, value_6; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_4, sum_value_5) SELECT (int4eq(1, max(value_1)))::integer AS int4eq, value_6, random() AS random FROM multi_deparse_shard_query.raw_events_1 WHERE (event_at OPERATOR(pg_catalog.=) now()) GROUP BY event_at, value_7, value_6 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -INSERT INTO aggregated_events (sum_value_1, tenant_id) -SELECT - count(DISTINCT CASE - WHEN - value_1 > 100 - THEN - tenant_id - ELSE - value_6 - END) as c, - max(tenant_id) - FROM - raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_1) SELECT max(tenant_id) AS max, count(DISTINCT CASE WHEN (value_1 OPERATOR(pg_catalog.>) 100) THEN tenant_id ELSE (value_6)::bigint END) AS c FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(value_7, value_1, tenant_id) -SELECT - value_7, value_1, tenant_id -FROM - (SELECT - tenant_id, value_2 as value_7, value_1 - FROM - raw_events_2 - ) as foo -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_1, value_6, value_7, event_at) SELECT tenant_id, value_1, 10 AS value_6, value_7, (now())::date AS event_at FROM (SELECT raw_events_2.tenant_id, raw_events_2.value_2 AS value_7, raw_events_2.value_1 FROM multi_deparse_shard_query.raw_events_2) foo - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(E' -INSERT INTO aggregated_events(sum_value_1, tenant_id, sum_value_5) -SELECT - sum(value_1), tenant_id, sum(value_5::bigint) -FROM - (SELECT - raw_events_1.event_at, raw_events_2.tenant_id, raw_events_2.value_5, raw_events_1.value_1 - FROM - raw_events_2, raw_events_1 - WHERE - raw_events_1.tenant_id = raw_events_2.tenant_id - ) as foo -GROUP BY - tenant_id, date_trunc(\'hour\', event_at) -'); -INFO: query: INSERT INTO multi_deparse_shard_query.aggregated_events (tenant_id, sum_value_1, sum_value_5) SELECT tenant_id, sum(value_1) AS sum, sum((value_5)::bigint) AS sum FROM (SELECT raw_events_1.event_at, raw_events_2.tenant_id, raw_events_2.value_5, raw_events_1.value_1 FROM multi_deparse_shard_query.raw_events_2, multi_deparse_shard_query.raw_events_1 WHERE (raw_events_1.tenant_id OPERATOR(pg_catalog.=) raw_events_2.tenant_id)) foo GROUP BY tenant_id, (date_trunc('hour'::text, (event_at)::timestamp with time zone)) - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(E' -INSERT INTO raw_events_2(tenant_id, value_1, value_2, value_3, value_4) -SELECT - tenant_id, value_1, value_2, value_3, value_4 -FROM - (SELECT - value_2, value_4, tenant_id, value_1, value_3 - FROM - raw_events_1 - ) as foo -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_2 (tenant_id, value_1, value_2, value_3, value_4, value_6, event_at) SELECT tenant_id, value_1, value_2, value_3, value_4, (random() OPERATOR(pg_catalog.*) (100)::double precision) AS value_6, (now())::date AS event_at FROM (SELECT raw_events_1.value_2, raw_events_1.value_4, raw_events_1.tenant_id, raw_events_1.value_1, raw_events_1.value_3 FROM multi_deparse_shard_query.raw_events_1) foo - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SELECT deparse_shard_query_test(E' -INSERT INTO raw_events_2(tenant_id, value_1, value_4, value_2, value_3) -SELECT - * -FROM - (SELECT - value_2, value_4, tenant_id, value_1, value_3 - FROM - raw_events_1 - ) as foo -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_2 (tenant_id, value_1, value_2, value_3, value_4, value_6, event_at) SELECT value_2, value_4, value_1, value_3, tenant_id, (random() OPERATOR(pg_catalog.*) (100)::double precision) AS value_6, (now())::date AS event_at FROM (SELECT raw_events_1.value_2, raw_events_1.value_4, raw_events_1.tenant_id, raw_events_1.value_1, raw_events_1.value_3 FROM multi_deparse_shard_query.raw_events_1) foo - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- use a column multiple times -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(tenant_id, value_7, value_4) -SELECT - tenant_id, value_7, value_7 -FROM - raw_events_1 -ORDER BY - value_2, value_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_4, value_6, value_7, event_at) SELECT tenant_id, value_7, 10 AS value_6, value_7, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1 ORDER BY value_2, value_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - --- test dropped table as well -ALTER TABLE raw_events_1 DROP COLUMN value_5; -SELECT deparse_shard_query_test(' -INSERT INTO raw_events_1(tenant_id, value_7, value_4) -SELECT - tenant_id, value_7, value_4 -FROM - raw_events_1; -'); -INFO: query: INSERT INTO multi_deparse_shard_query.raw_events_1 (tenant_id, value_4, value_6, value_7, event_at) SELECT tenant_id, value_4, 10 AS value_6, value_7, (now())::date AS event_at FROM multi_deparse_shard_query.raw_events_1 - deparse_shard_query_test ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO ERROR; -DROP SCHEMA multi_deparse_shard_query CASCADE; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 2f27722f7..128872df5 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -747,14 +747,8 @@ SELECT * FROM multi_extension.print_extension_changes(); -- recreate public schema, and recreate citus_tables in the public schema by default CREATE SCHEMA public; --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 +-- public schema is owned by pg_database_owner role ALTER SCHEMA public OWNER TO pg_database_owner; -\endif GRANT ALL ON SCHEMA public TO public; ALTER EXTENSION citus UPDATE TO '9.5-1'; ALTER EXTENSION citus UPDATE TO '10.0-4'; diff --git a/src/test/regress/expected/multi_insert_select.out b/src/test/regress/expected/multi_insert_select.out index 26a7dfcf5..58d22583e 100644 --- a/src/test/regress/expected/multi_insert_select.out +++ b/src/test/regress/expected/multi_insert_select.out @@ -1,19 +1,8 @@ -- -- MULTI_INSERT_SELECT -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- CREATE SCHEMA multi_insert_select; SET search_path = multi_insert_select,public; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - SET citus.next_shard_id TO 13300000; SET citus.next_placement_id TO 13300000; -- create co-located tables diff --git a/src/test/regress/expected/multi_insert_select_0.out b/src/test/regress/expected/multi_insert_select_0.out deleted file mode 100644 index 193c869b1..000000000 --- a/src/test/regress/expected/multi_insert_select_0.out +++ /dev/null @@ -1,3507 +0,0 @@ --- --- MULTI_INSERT_SELECT --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -CREATE SCHEMA multi_insert_select; -SET search_path = multi_insert_select,public; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -SET citus.next_shard_id TO 13300000; -SET citus.next_placement_id TO 13300000; --- create co-located tables -SET citus.shard_count = 4; -SET citus.shard_replication_factor = 2; --- order of execution might change in parallel executions --- and the error details might contain the worker node --- so be less verbose with \set VERBOSITY TERSE when necessary -CREATE TABLE raw_events_first (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint, UNIQUE(user_id, value_1)); -SELECT create_distributed_table('raw_events_first', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE raw_events_second (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint, UNIQUE(user_id, value_1)); -SELECT create_distributed_table('raw_events_second', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE agg_events (user_id int, value_1_agg int, value_2_agg int, value_3_agg float, value_4_agg bigint, agg_time timestamp, UNIQUE(user_id, value_1_agg)); -SELECT create_distributed_table('agg_events', 'user_id');; - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- create the reference table as well -CREATE TABLE reference_table (user_id int); -SELECT create_reference_table('reference_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE insert_select_varchar_test (key varchar, value int); -SELECT create_distributed_table('insert_select_varchar_test', 'key', 'hash'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- set back to the defaults -SET citus.shard_count = DEFAULT; -SET citus.shard_replication_factor = DEFAULT; -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (1, now(), 10, 100, 1000.1, 10000); -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (2, now(), 20, 200, 2000.1, 20000); -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (3, now(), 30, 300, 3000.1, 30000); -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (4, now(), 40, 400, 4000.1, 40000); -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (5, now(), 50, 500, 5000.1, 50000); -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (6, now(), 60, 600, 6000.1, 60000); -SET client_min_messages TO DEBUG2; --- raw table to raw table -INSERT INTO raw_events_second SELECT * FROM raw_events_first; -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) --- see that our first multi shard INSERT...SELECT works expected -SET client_min_messages TO INFO; -SELECT - raw_events_first.user_id -FROM - raw_events_first, raw_events_second -WHERE - raw_events_first.user_id = raw_events_second.user_id -ORDER BY - user_id DESC; - user_id ---------------------------------------------------------------------- - 6 - 5 - 4 - 3 - 2 - 1 -(6 rows) - --- see that we get unique vialitons -\set VERBOSITY TERSE -INSERT INTO raw_events_second SELECT * FROM raw_events_first; -ERROR: duplicate key value violates unique constraint "raw_events_second_user_id_value_1_key_13300004" -\set VERBOSITY DEFAULT --- stable functions should be allowed -INSERT INTO raw_events_second (user_id, time) -SELECT - user_id, now() -FROM - raw_events_first -WHERE - user_id < 0; -INSERT INTO raw_events_second (user_id) -SELECT - user_id -FROM - raw_events_first -WHERE - time > now() + interval '1 day'; --- hide version-dependent PL/pgSQL context messages -\set VERBOSITY terse --- make sure we evaluate stable functions on the master, once -CREATE OR REPLACE FUNCTION evaluate_on_master() -RETURNS int LANGUAGE plpgsql STABLE -AS $function$ -BEGIN - RAISE NOTICE 'evaluating on master'; - RETURN 0; -END; -$function$; -INSERT INTO raw_events_second (user_id, value_1) -SELECT - user_id, evaluate_on_master() -FROM - raw_events_first -WHERE - user_id < 0; -NOTICE: evaluating on master --- make sure we don't evaluate stable functions with column arguments -SET citus.enable_metadata_sync TO OFF; -CREATE OR REPLACE FUNCTION evaluate_on_master(x int) -RETURNS int LANGUAGE plpgsql STABLE -AS $function$ -BEGIN - RAISE NOTICE 'evaluating on master'; - RETURN x; -END; -$function$; -RESET citus.enable_metadata_sync; -INSERT INTO raw_events_second (user_id, value_1) -SELECT - user_id, evaluate_on_master(value_1) -FROM - raw_events_first -WHERE - user_id = 0; -ERROR: function multi_insert_select.evaluate_on_master(integer) does not exist --- add one more row -INSERT INTO raw_events_first (user_id, time) VALUES - (7, now()); --- try a single shard query -SET client_min_messages TO DEBUG2; -INSERT INTO raw_events_second (user_id, time) SELECT user_id, time FROM raw_events_first WHERE user_id = 7; -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id, "time") SELECT user_id, "time" FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) 7) AND (user_id IS NOT NULL)) -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away -SET client_min_messages TO INFO; --- add one more row -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (8, now(), 80, 800, 8000, 80000); --- reorder columns -SET client_min_messages TO DEBUG2; -INSERT INTO raw_events_second (value_2, value_1, value_3, value_4, user_id, time) -SELECT - value_2, value_1, value_3, value_4, user_id, time -FROM - raw_events_first -WHERE - user_id = 8; -DEBUG: Creating router plan -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) 8) AND (user_id IS NOT NULL)) -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away --- a zero shard select -INSERT INTO raw_events_second (value_2, value_1, value_3, value_4, user_id, time) -SELECT - value_2, value_1, value_3, value_4, user_id, time -FROM - raw_events_first -WHERE - false; -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away --- another zero shard select -INSERT INTO raw_events_second (value_2, value_1, value_3, value_4, user_id, time) -SELECT - value_2, value_1, value_3, value_4, user_id, time -FROM - raw_events_first -WHERE - 0 != 0; -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away --- add one more row -SET client_min_messages TO INFO; -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (9, now(), 90, 900, 9000, 90000); --- show that RETURNING also works -SET client_min_messages TO DEBUG2; -INSERT INTO raw_events_second (user_id, value_1, value_3) -SELECT - user_id, value_1, value_3 -FROM - raw_events_first -WHERE - value_3 = 9000 -RETURNING *; -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((value_3 OPERATOR(pg_catalog.=) (9000)::double precision) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((value_3 OPERATOR(pg_catalog.=) (9000)::double precision) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((value_3 OPERATOR(pg_catalog.=) (9000)::double precision) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((value_3 OPERATOR(pg_catalog.=) (9000)::double precision) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 - user_id | time | value_1 | value_2 | value_3 | value_4 ---------------------------------------------------------------------- - 9 | | 90 | | 9000 | -(1 row) - --- hits two shards -\set VERBOSITY TERSE -INSERT INTO raw_events_second (user_id, value_1, value_3) -SELECT - user_id, value_1, value_3 -FROM - raw_events_first -WHERE - user_id = 9 OR user_id = 16 -RETURNING *; -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (((user_id OPERATOR(pg_catalog.=) 9) OR (user_id OPERATOR(pg_catalog.=) 16)) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (((user_id OPERATOR(pg_catalog.=) 9) OR (user_id OPERATOR(pg_catalog.=) 16)) AND (user_id IS NOT NULL)) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -ERROR: duplicate key value violates unique constraint "raw_events_second_user_id_value_1_key_13300007" --- now do some aggregations -INSERT INTO agg_events -SELECT - user_id, sum(value_1), avg(value_2), sum(value_3), count(value_4) -FROM - raw_events_first -GROUP BY - user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg, value_2_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, avg(value_2) AS avg, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg, value_2_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, avg(value_2) AS avg, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg, value_2_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, avg(value_2) AS avg, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg, value_2_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, avg(value_2) AS avg, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id --- group by column not exists on the SELECT target list -INSERT INTO agg_events (value_3_agg, value_4_agg, value_1_agg, user_id) -SELECT - sum(value_3), count(value_4), sum(value_1), user_id -FROM - raw_events_first -GROUP BY - value_2, user_id -RETURNING *; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY value_2, user_id RETURNING citus_table_alias.user_id, citus_table_alias.value_1_agg, citus_table_alias.value_2_agg, citus_table_alias.value_3_agg, citus_table_alias.value_4_agg, citus_table_alias.agg_time -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY value_2, user_id RETURNING citus_table_alias.user_id, citus_table_alias.value_1_agg, citus_table_alias.value_2_agg, citus_table_alias.value_3_agg, citus_table_alias.value_4_agg, citus_table_alias.agg_time -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY value_2, user_id RETURNING citus_table_alias.user_id, citus_table_alias.value_1_agg, citus_table_alias.value_2_agg, citus_table_alias.value_3_agg, citus_table_alias.value_4_agg, citus_table_alias.agg_time -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg, value_3_agg, value_4_agg) SELECT user_id, sum(value_1) AS sum, sum(value_3) AS sum, count(value_4) AS count FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY value_2, user_id RETURNING citus_table_alias.user_id, citus_table_alias.value_1_agg, citus_table_alias.value_2_agg, citus_table_alias.value_3_agg, citus_table_alias.value_4_agg, citus_table_alias.agg_time -ERROR: duplicate key value violates unique constraint "agg_events_user_id_value_1_agg_key_13300008" --- some subquery tests -INSERT INTO agg_events - (value_1_agg, - user_id) -SELECT SUM(value_1), - id -FROM (SELECT raw_events_second.user_id AS id, - raw_events_second.value_1 - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id) AS foo -GROUP BY id -ORDER BY id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT id, sum(value_1) AS sum FROM (SELECT raw_events_second.user_id AS id, raw_events_second.value_1 FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id)) foo WHERE (id IS NOT NULL) GROUP BY id ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT id, sum(value_1) AS sum FROM (SELECT raw_events_second.user_id AS id, raw_events_second.value_1 FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id)) foo WHERE (id IS NOT NULL) GROUP BY id ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT id, sum(value_1) AS sum FROM (SELECT raw_events_second.user_id AS id, raw_events_second.value_1 FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id)) foo WHERE (id IS NOT NULL) GROUP BY id ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT id, sum(value_1) AS sum FROM (SELECT raw_events_second.user_id AS id, raw_events_second.value_1 FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id)) foo WHERE (id IS NOT NULL) GROUP BY id ORDER BY id -ERROR: duplicate key value violates unique constraint "agg_events_user_id_value_1_agg_key_13300008" --- subquery one more level depth -INSERT INTO agg_events - (value_4_agg, - value_1_agg, - user_id) -SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id) AS foo -ORDER BY id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg, value_4_agg) SELECT id, v1, v4 FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id) foo WHERE (id IS NOT NULL) ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg, value_4_agg) SELECT id, v1, v4 FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id) foo WHERE (id IS NOT NULL) ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg, value_4_agg) SELECT id, v1, v4 FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id) foo WHERE (id IS NOT NULL) ORDER BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg, value_4_agg) SELECT id, v1, v4 FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id) foo WHERE (id IS NOT NULL) ORDER BY id -ERROR: duplicate key value violates unique constraint "agg_events_user_id_value_1_agg_key_13300008" -\set VERBOSITY DEFAULT --- join between subqueries -INSERT INTO agg_events - (user_id) -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id); -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f2.id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f2.id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f2.id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f2.id IS NOT NULL) --- add one more level subqueris on top of subquery JOINs -INSERT INTO agg_events - (user_id, value_4_agg) -SELECT - outer_most.id, max(outer_most.value) -FROM -( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id)) as outer_most -GROUP BY - outer_most.id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id --- subqueries in WHERE clause -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT user_id - FROM raw_events_second - WHERE user_id = 2); -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) 2))) AND (user_id IS NOT NULL)) -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT user_id - FROM raw_events_second - WHERE user_id != 2 AND value_1 = 2000) -ON conflict (user_id, value_1) DO NOTHING; -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.<>) 2) AND (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000)))) AND (user_id IS NOT NULL)) ON CONFLICT(user_id, value_1) DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300005 raw_events_second WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.<>) 2) AND (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000)))) AND (user_id IS NOT NULL)) ON CONFLICT(user_id, value_1) DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300006 raw_events_second WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.<>) 2) AND (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000)))) AND (user_id IS NOT NULL)) ON CONFLICT(user_id, value_1) DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.<>) 2) AND (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000)))) AND (user_id IS NOT NULL)) ON CONFLICT(user_id, value_1) DO NOTHING -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT user_id - FROM raw_events_second WHERE false); -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT user_id - FROM raw_events_second - WHERE value_1 = 1000 OR value_1 = 2000 OR value_1 = 3000); -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE ((raw_events_second.value_1 OPERATOR(pg_catalog.=) 1000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 3000)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300005 raw_events_second WHERE ((raw_events_second.value_1 OPERATOR(pg_catalog.=) 1000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 3000)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300006 raw_events_second WHERE ((raw_events_second.value_1 OPERATOR(pg_catalog.=) 1000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 3000)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE ((raw_events_second.value_1 OPERATOR(pg_catalog.=) 1000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 2000) OR (raw_events_second.value_1 OPERATOR(pg_catalog.=) 3000)))) AND (user_id IS NOT NULL)) --- lets mix subqueries in FROM clause and subqueries in WHERE -INSERT INTO agg_events - (user_id) -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 1000) AS foo2 ) as f2 -ON (f.id = f2.id) -WHERE f.id IN (SELECT user_id - FROM raw_events_second); -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (1000)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE ((f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second)) AND (f2.id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (1000)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE ((f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300005 raw_events_second)) AND (f2.id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (1000)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE ((f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300006 raw_events_second)) AND (f2.id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (1000)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE ((f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second)) AND (f2.id IS NOT NULL)) --- some UPSERTS -INSERT INTO agg_events AS ae - ( - user_id, - value_1_agg, - agg_time - ) -SELECT user_id, - value_1, - time -FROM raw_events_first -ON conflict (user_id, value_1_agg) -DO UPDATE - SET agg_time = EXCLUDED.agg_time - WHERE ae.agg_time < EXCLUDED.agg_time; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) --- upserts with returning -INSERT INTO agg_events AS ae - ( - user_id, - value_1_agg, - agg_time - ) -SELECT user_id, - value_1, - time -FROM raw_events_first -ON conflict (user_id, value_1_agg) -DO UPDATE - SET agg_time = EXCLUDED.agg_time - WHERE ae.agg_time < EXCLUDED.agg_time -RETURNING user_id, value_1_agg; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) RETURNING ae.user_id, ae.value_1_agg -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) RETURNING ae.user_id, ae.value_1_agg -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) RETURNING ae.user_id, ae.value_1_agg -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS ae (user_id, value_1_agg, agg_time) SELECT user_id, value_1, "time" FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) ON CONFLICT(user_id, value_1_agg) DO UPDATE SET agg_time = excluded.agg_time WHERE (ae.agg_time OPERATOR(pg_catalog.<) excluded.agg_time) RETURNING ae.user_id, ae.value_1_agg - user_id | value_1_agg ---------------------------------------------------------------------- - 7 | -(1 row) - -INSERT INTO agg_events (user_id, value_1_agg) -SELECT - user_id, sum(value_1 + value_2) -FROM - raw_events_first GROUP BY user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) AS sum FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) AS sum FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) AS sum FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) AS sum FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id --- FILTER CLAUSE -INSERT INTO agg_events (user_id, value_1_agg) -SELECT - user_id, sum(value_1 + value_2) FILTER (where value_3 = 15) -FROM - raw_events_first GROUP BY user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) FILTER (WHERE (value_3 OPERATOR(pg_catalog.=) (15)::double precision)) AS sum FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) FILTER (WHERE (value_3 OPERATOR(pg_catalog.=) (15)::double precision)) AS sum FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) FILTER (WHERE (value_3 OPERATOR(pg_catalog.=) (15)::double precision)) AS sum FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, sum((value_1 OPERATOR(pg_catalog.+) value_2)) FILTER (WHERE (value_3 OPERATOR(pg_catalog.=) (15)::double precision)) AS sum FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) GROUP BY user_id --- a test with reference table JOINs -INSERT INTO - agg_events (user_id, value_1_agg) -SELECT - raw_events_first.user_id, sum(value_1) -FROM - reference_table, raw_events_first -WHERE - raw_events_first.user_id = reference_table.user_id -GROUP BY - raw_events_first.user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT raw_events_first.user_id, sum(raw_events_first.value_1) AS sum FROM multi_insert_select.reference_table_13300012 reference_table, multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id) AND (raw_events_first.user_id IS NOT NULL)) GROUP BY raw_events_first.user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT raw_events_first.user_id, sum(raw_events_first.value_1) AS sum FROM multi_insert_select.reference_table_13300012 reference_table, multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id) AND (raw_events_first.user_id IS NOT NULL)) GROUP BY raw_events_first.user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT raw_events_first.user_id, sum(raw_events_first.value_1) AS sum FROM multi_insert_select.reference_table_13300012 reference_table, multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id) AND (raw_events_first.user_id IS NOT NULL)) GROUP BY raw_events_first.user_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT raw_events_first.user_id, sum(raw_events_first.value_1) AS sum FROM multi_insert_select.reference_table_13300012 reference_table, multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id) AND (raw_events_first.user_id IS NOT NULL)) GROUP BY raw_events_first.user_id --- a note on the outer joins is that --- we filter out outer join results --- where partition column returns --- NULL. Thus, we could INSERT less rows --- than we expect from subquery result. --- see the following tests -SET client_min_messages TO INFO; --- we don't want to see constraint violations, so truncate first -TRUNCATE agg_events; --- add a row to first table to make table contents different -INSERT INTO raw_events_second (user_id, time, value_1, value_2, value_3, value_4) VALUES - (10, now(), 100, 10000, 10000, 100000); -DELETE FROM raw_events_second WHERE user_id = 2; --- we select 11 rows -SELECT t1.user_id AS col1, - t2.user_id AS col2 - FROM raw_events_first t1 - FULL JOIN raw_events_second t2 - ON t1.user_id = t2.user_id - ORDER BY t1.user_id, - t2.user_id; - col1 | col2 ---------------------------------------------------------------------- - 1 | 1 - 2 | - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - | 10 -(10 rows) - -SET client_min_messages TO DEBUG2; --- we insert 10 rows since we filtered out --- NULL partition column values -INSERT INTO agg_events (user_id, value_1_agg) -SELECT t1.user_id AS col1, - t2.user_id AS col2 -FROM raw_events_first t1 - FULL JOIN raw_events_second t2 - ON t1.user_id = t2.user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT t1.user_id AS col1, t2.user_id AS col2 FROM (multi_insert_select.raw_events_first_13300000 t1 FULL JOIN multi_insert_select.raw_events_second_13300004 t2 ON ((t1.user_id OPERATOR(pg_catalog.=) t2.user_id))) WHERE (t1.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT t1.user_id AS col1, t2.user_id AS col2 FROM (multi_insert_select.raw_events_first_13300001 t1 FULL JOIN multi_insert_select.raw_events_second_13300005 t2 ON ((t1.user_id OPERATOR(pg_catalog.=) t2.user_id))) WHERE (t1.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT t1.user_id AS col1, t2.user_id AS col2 FROM (multi_insert_select.raw_events_first_13300002 t1 FULL JOIN multi_insert_select.raw_events_second_13300006 t2 ON ((t1.user_id OPERATOR(pg_catalog.=) t2.user_id))) WHERE (t1.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT t1.user_id AS col1, t2.user_id AS col2 FROM (multi_insert_select.raw_events_first_13300003 t1 FULL JOIN multi_insert_select.raw_events_second_13300007 t2 ON ((t1.user_id OPERATOR(pg_catalog.=) t2.user_id))) WHERE (t1.user_id IS NOT NULL) -SET client_min_messages TO INFO; --- see that the results are different from the SELECT query -SELECT - user_id, value_1_agg -FROM - agg_events -ORDER BY - user_id, value_1_agg; - user_id | value_1_agg ---------------------------------------------------------------------- - 1 | 1 - 2 | - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 -(9 rows) - --- we don't want to see constraint violations, so truncate first -SET client_min_messages TO INFO; -TRUNCATE agg_events; -SET client_min_messages TO DEBUG2; --- DISTINCT clause -INSERT INTO agg_events (value_1_agg, user_id) - SELECT - DISTINCT value_1, user_id - FROM - raw_events_first; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT user_id, value_1 FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT user_id, value_1 FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT user_id, value_1 FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT user_id, value_1 FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) --- we don't want to see constraint violations, so truncate first -SET client_min_messages TO INFO; -truncate agg_events; -SET client_min_messages TO DEBUG2; --- DISTINCT ON clauses are supported --- distinct on(non-partition column) --- values are pulled to master -INSERT INTO agg_events (value_1_agg, user_id) - SELECT - DISTINCT ON (value_1) value_1, user_id - FROM - raw_events_first; -DEBUG: cannot push down this subquery -DETAIL: Distinct on columns without partition column is currently unsupported -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT user_id, value_1_agg FROM agg_events ORDER BY 1,2; -DEBUG: Router planner cannot handle multi-shard select queries - user_id | value_1_agg ---------------------------------------------------------------------- - 1 | 10 - 2 | 20 - 3 | 30 - 4 | 40 - 5 | 50 - 6 | 60 - 7 | - 8 | 80 - 9 | 90 -(9 rows) - --- we don't want to see constraint violations, so truncate first -SET client_min_messages TO INFO; -truncate agg_events; -SET client_min_messages TO DEBUG2; --- distinct on(partition column) --- queries are forwared to workers -INSERT INTO agg_events (value_1_agg, user_id) - SELECT - DISTINCT ON (user_id) value_1, user_id - FROM - raw_events_first; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT ON (user_id) user_id, value_1 FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT ON (user_id) user_id, value_1 FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT ON (user_id) user_id, value_1 FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT DISTINCT ON (user_id) user_id, value_1 FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) -SELECT user_id, value_1_agg FROM agg_events ORDER BY 1,2; -DEBUG: Router planner cannot handle multi-shard select queries - user_id | value_1_agg ---------------------------------------------------------------------- - 1 | 10 - 2 | 20 - 3 | 30 - 4 | 40 - 5 | 50 - 6 | 60 - 7 | - 8 | 80 - 9 | 90 -(9 rows) - --- We support CTEs -BEGIN; -WITH fist_table_agg AS MATERIALIZED - (SELECT max(value_1)+1 as v1_agg, user_id FROM raw_events_first GROUP BY user_id) -INSERT INTO agg_events - (value_1_agg, user_id) - SELECT - v1_agg, user_id - FROM - fist_table_agg; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE fist_table_agg: SELECT (max(value_1) OPERATOR(pg_catalog.+) 1) AS v1_agg, user_id FROM multi_insert_select.raw_events_first GROUP BY user_id -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id, v1_agg AS value_1_agg FROM (SELECT fist_table_agg.user_id, fist_table_agg.v1_agg FROM (SELECT intermediate_result.v1_agg, intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v1_agg integer, user_id integer)) fist_table_agg) citus_insert_select_subquery -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator -ROLLBACK; --- We do support CTEs that are referenced in the target list -INSERT INTO agg_events - WITH sub_cte AS (SELECT 1) - SELECT - raw_events_first.user_id, (SELECT * FROM sub_cte) - FROM - raw_events_first; -DEBUG: CTE sub_cte is going to be inlined via distributed planning -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, (SELECT sub_cte."?column?" FROM (SELECT 1) sub_cte("?column?")) FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, (SELECT sub_cte."?column?" FROM (SELECT 1) sub_cte("?column?")) FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, (SELECT sub_cte."?column?" FROM (SELECT 1) sub_cte("?column?")) FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_1_agg) SELECT user_id, (SELECT sub_cte."?column?" FROM (SELECT 1) sub_cte("?column?")) FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE (user_id IS NOT NULL) --- We support set operations -BEGIN; -INSERT INTO - raw_events_first(user_id) -SELECT - user_id -FROM - ((SELECT user_id FROM raw_events_first) UNION - (SELECT user_id FROM raw_events_second)) as foo; -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300000 AS citus_table_alias (user_id) SELECT user_id FROM (SELECT raw_events_first.user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first UNION SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second) foo WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300001 AS citus_table_alias (user_id) SELECT user_id FROM (SELECT raw_events_first.user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first UNION SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300005 raw_events_second) foo WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300002 AS citus_table_alias (user_id) SELECT user_id FROM (SELECT raw_events_first.user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first UNION SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300006 raw_events_second) foo WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300003 AS citus_table_alias (user_id) SELECT user_id FROM (SELECT raw_events_first.user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first UNION SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second) foo WHERE (user_id IS NOT NULL) -ROLLBACK; --- We do support set operations through recursive planning -BEGIN; -SET LOCAL client_min_messages TO DEBUG; -INSERT INTO - raw_events_first(user_id) - (SELECT user_id FROM raw_events_first) INTERSECT - (SELECT user_id FROM raw_events_first); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM multi_insert_select.raw_events_first -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT user_id FROM multi_insert_select.raw_events_first -DEBUG: Creating router plan -DEBUG: generating subplan XXX_3 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) INTERSECT SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) citus_insert_select_subquery -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator -ROLLBACK; --- If the query is router plannable then it is executed via the coordinator -INSERT INTO - raw_events_first(user_id) -SELECT - user_id -FROM - ((SELECT user_id FROM raw_events_first WHERE user_id = 15) EXCEPT - (SELECT user_id FROM raw_events_second where user_id = 17)) as foo; -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: router planner does not support queries that reference non-colocated distributed tables -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 15 -DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM multi_insert_select.raw_events_first WHERE (user_id OPERATOR(pg_catalog.=) 15) -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: query has a single distribution column value: 17 -DEBUG: generating subplan XXX_2 for subquery SELECT user_id FROM multi_insert_select.raw_events_second WHERE (user_id OPERATOR(pg_catalog.=) 17) -DEBUG: Creating router plan -DEBUG: generating subplan XXX_3 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) EXCEPT SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) foo -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator --- some supported LEFT joins - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (raw_events_first.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300001 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (raw_events_first.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300002 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (raw_events_first.user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300003 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (raw_events_first.user_id IS NOT NULL) - INSERT INTO agg_events (user_id) - SELECT - raw_events_second.user_id - FROM - reference_table LEFT JOIN raw_events_second ON reference_table.user_id = raw_events_second.user_id; -DEBUG: cannot perform a lateral outer join when a distributed subquery references a reference table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: recursively planning right side of the left join since the outer side is a recurring rel -DEBUG: recursively planning distributed relation "raw_events_second" since it is part of a distributed join node that is outer joined with a recurring rel -DEBUG: Wrapping relation "raw_events_second" to a subquery -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM multi_insert_select.raw_events_second WHERE true -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT raw_events_second.user_id FROM (multi_insert_select.reference_table LEFT JOIN (SELECT raw_events_second_1.user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) raw_events_second_1) raw_events_second ON ((reference_table.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_first.user_id = 10; -DEBUG: Creating router plan -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) 10) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_second.user_id = 10 OR raw_events_second.user_id = 11; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (((raw_events_second.user_id OPERATOR(pg_catalog.=) 10) OR (raw_events_second.user_id OPERATOR(pg_catalog.=) 11)) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300001 raw_events_first LEFT JOIN (SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_second(user_id, "time", value_1, value_2, value_3, value_4) ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (((raw_events_second.user_id OPERATOR(pg_catalog.=) 10) OR (raw_events_second.user_id OPERATOR(pg_catalog.=) 11)) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300002 raw_events_first LEFT JOIN (SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_second(user_id, "time", value_1, value_2, value_3, value_4) ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (((raw_events_second.user_id OPERATOR(pg_catalog.=) 10) OR (raw_events_second.user_id OPERATOR(pg_catalog.=) 11)) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300003 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE (((raw_events_second.user_id OPERATOR(pg_catalog.=) 10) OR (raw_events_second.user_id OPERATOR(pg_catalog.=) 11)) AND (raw_events_first.user_id IS NOT NULL)) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_first.user_id = 10 AND raw_events_first.user_id = 20; -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300008 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_first.user_id = 10 AND raw_events_second.user_id = 20; -DEBUG: Creating router plan -DEBUG: Skipping target shard interval 13300008 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_first.user_id IN (19, 20, 21); -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300001 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300002 raw_events_first LEFT JOIN multi_insert_select.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) LEFT JOIN multi_insert_select.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_first.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id - WHERE raw_events_second.user_id IN (19, 20, 21); -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first JOIN multi_insert_select.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300001 raw_events_first JOIN multi_insert_select.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300002 raw_events_first JOIN multi_insert_select.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (multi_insert_select.raw_events_first_13300003 raw_events_first JOIN (SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_second(user_id, "time", value_1, value_2, value_3, value_4) ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id))) WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.=) ANY (ARRAY[19, 20, 21])) AND (raw_events_first.user_id IS NOT NULL)) -SET client_min_messages TO WARNING; - -- following query should use repartitioned joins and results should - -- be routed via coordinator - SET citus.enable_repartition_joins TO true; - INSERT INTO agg_events - (user_id) - SELECT raw_events_first.user_id - FROM raw_events_first, - raw_events_second - WHERE raw_events_second.user_id = raw_events_first.value_1 - AND raw_events_first.value_1 = 12; - -- some unsupported LEFT/INNER JOINs - -- JOIN on one table with partition column other is not - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; -ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns - -- same as the above with INNER JOIN - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; - -- a not meaningful query - INSERT INTO agg_events - (user_id) - SELECT raw_events_second.user_id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_first.value_1; -ERROR: cannot perform distributed planning on this query -DETAIL: Cartesian products are currently unsupported - -- both tables joined on non-partition columns - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.value_1 = raw_events_second.value_1; -ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns - -- same as the above with INNER JOIN - -- we support this with route to coordinator - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.value_1 = raw_events_second.value_1; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - --- EXPLAIN ANALYZE is not supported for INSERT ... SELECT via coordinator -EXPLAIN (costs off, analyze on) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.value_1 = raw_events_second.value_1; -ERROR: EXPLAIN ANALYZE is currently not supported for INSERT ... SELECT commands via coordinator --- even if there is a filter on the partition key, since the join is not on the partition key we reject --- this query -INSERT INTO agg_events (user_id) -SELECT - raw_events_first.user_id -FROM - raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 -WHERE - raw_events_first.user_id = 10; -ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns - -- same as the above with INNER JOIN - -- we support this with route to coordinator - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 - WHERE raw_events_first.user_id = 10; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - - -- make things a bit more complicate with IN clauses - -- we support this with route to coordinator - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events (user_id) - SELECT - raw_events_first.user_id - FROM - raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 - WHERE raw_events_first.value_1 IN (10, 11,12) OR raw_events_second.user_id IN (1,2,3,4); -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - - -- implicit join on non partition column should also not be pushed down, - -- so we fall back to route via coordinator - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events - (user_id) - SELECT raw_events_first.user_id - FROM raw_events_first, - raw_events_second - WHERE raw_events_second.user_id = raw_events_first.value_1; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - -RESET client_min_messages; - -- The following is again a tricky query for Citus. If the given filter was - -- on value_1 as shown in the above, Citus could push it down and use - -- distributed INSERT/SELECT. But we instead fall back to route via coordinator. - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events - (user_id) - SELECT raw_events_first.user_id - FROM raw_events_first, - raw_events_second - WHERE raw_events_second.user_id = raw_events_first.value_1 - AND raw_events_first.value_2 = 12; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - - -- foo is not joined on the partition key so the query is not - -- pushed down. So instead we route via coordinator. - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events - (user_id, value_4_agg) - SELECT - outer_most.id, max(outer_most.value) - FROM - ( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first LEFT JOIN - reference_table - ON (raw_events_first.value_1 = reference_table.user_id)) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 - ON (f.id = f2.id)) as outer_most - GROUP BY - outer_most.id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(8 rows) - - -- if the given filter was on value_1 as shown in the above, Citus could - -- push it down. But here the query falls back to route via coordinator. - SELECT coordinator_plan($Q$ - EXPLAIN (costs off) - INSERT INTO agg_events - (user_id) - SELECT raw_events_first.user_id - FROM raw_events_first, - raw_events_second - WHERE raw_events_second.user_id = raw_events_first.value_1 - AND raw_events_first.value_2 = 12; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - - -- foo is not joined on the partition key so the query is not - -- pushed down, and it falls back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) - INSERT INTO agg_events - (user_id, value_4_agg) - SELECT - outer_most.id, max(outer_most.value) - FROM - ( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first LEFT JOIN - reference_table - ON (raw_events_first.value_1 = reference_table.user_id)) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 - ON (f.id = f2.id)) as outer_most - GROUP BY - outer_most.id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(8 rows) - -INSERT INTO agg_events - (value_4_agg, - value_1_agg, - user_id) -SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id != raw_events_second.user_id - GROUP BY raw_events_second.user_id) AS foo; -ERROR: complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator -SET client_min_messages TO DEBUG2; --- INSERT returns NULL partition key value via coordinator -INSERT INTO agg_events - (value_4_agg, - value_1_agg, - user_id) -SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.value_3 AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.value_3) AS foo; -DEBUG: cannot push down this subquery -DETAIL: Group by list without partition column is currently unsupported when a subquery references a column from another query -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] -DEBUG: generating subplan XXX_1 for subquery SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.value_3 AS id FROM multi_insert_select.raw_events_first, multi_insert_select.raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.value_3 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT int4(id) AS user_id, int4(v1) AS value_1_agg, int8(v4) AS value_4_agg FROM (SELECT intermediate_result.v4, intermediate_result.v1, intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v4 numeric, v1 bigint, id double precision)) foo -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator -ERROR: the partition column of table multi_insert_select.agg_events cannot be NULL --- error cases --- no part column at all -INSERT INTO raw_events_second - (value_1) -SELECT value_1 -FROM raw_events_first; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: the query doesn't include the target table's partition column -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -ERROR: the partition column of table multi_insert_select.raw_events_second should have a value -INSERT INTO raw_events_second - (value_1) -SELECT user_id -FROM raw_events_first; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: the query doesn't include the target table's partition column -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -ERROR: the partition column of table multi_insert_select.raw_events_second should have a value -INSERT INTO raw_events_second - (user_id) -SELECT value_1 -FROM raw_events_first; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -ERROR: the partition column value cannot be NULL -CONTEXT: while executing command on localhost:xxxxx -INSERT INTO raw_events_second - (user_id) -SELECT user_id * 2 -FROM raw_events_first; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an operator in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300000_to_0,repartitioned_results_xxxxx_from_13300001_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300000_to_1,repartitioned_results_xxxxx_from_13300001_to_1,repartitioned_results_xxxxx_from_13300003_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300001_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300000_to_3,repartitioned_results_xxxxx_from_13300002_to_3,repartitioned_results_xxxxx_from_13300003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -INSERT INTO raw_events_second - (user_id) -SELECT user_id :: bigint -FROM raw_events_first; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an explicit cast in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300001_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300002_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM read_intermediate_results('{repartitioned_results_xxxxx_from_13300003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(user_id integer) -INSERT INTO agg_events - (value_3_agg, - value_4_agg, - value_1_agg, - value_2_agg, - user_id) -SELECT SUM(value_3), - Count(value_4), - user_id, - SUM(value_1), - Avg(value_2) -FROM raw_events_first -GROUP BY user_id; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an aggregation in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -ERROR: the partition column value cannot be NULL -CONTEXT: while executing command on localhost:xxxxx -INSERT INTO agg_events - (value_3_agg, - value_4_agg, - value_1_agg, - value_2_agg, - user_id) -SELECT SUM(value_3), - Count(value_4), - user_id, - SUM(value_1), - value_2 -FROM raw_events_first -GROUP BY user_id, - value_2; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -ERROR: the partition column value cannot be NULL -CONTEXT: while executing command on localhost:xxxxx --- tables should be co-located -INSERT INTO agg_events (user_id) -SELECT - user_id -FROM - reference_table; -DEBUG: Creating router plan -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Distributed planning for a fast-path router query -DEBUG: Creating router plan -DEBUG: Collecting INSERT ... SELECT results on coordinator --- foo2 is recursively planned and INSERT...SELECT is done via coordinator -INSERT INTO agg_events - (user_id) -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - raw_events_second.value_1 AS v1, - SUM(raw_events_second.user_id) AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.value_1 - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] -DEBUG: generating subplan XXX_1 for subquery SELECT sum(raw_events_second.value_4) AS v4, raw_events_second.value_1 AS v1, sum(raw_events_second.user_id) AS id FROM multi_insert_select.raw_events_first, multi_insert_select.raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.value_1 HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT int4(f2.id) AS user_id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first, multi_insert_select.reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT intermediate_result.v4, intermediate_result.v1, intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v4 numeric, v1 integer, id bigint)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' --- the second part of the query is not routable since --- GROUP BY not on the partition column (i.e., value_1) and thus join --- on f.id = f2.id is not on the partition key (instead on the sum of partition key) --- but we still recursively plan foo2 and run the query -INSERT INTO agg_events - (user_id) -SELECT f.id FROM -(SELECT - id -FROM (SELECT raw_events_first.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - raw_events_second.value_1 AS v1, - SUM(raw_events_second.user_id) AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.value_1 - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] -DEBUG: generating subplan XXX_1 for subquery SELECT sum(raw_events_second.value_4) AS v4, raw_events_second.value_1 AS v1, sum(raw_events_second.user_id) AS id FROM multi_insert_select.raw_events_first, multi_insert_select.raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.value_1 HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT f.id AS user_id FROM ((SELECT foo.id FROM (SELECT raw_events_first.user_id AS id FROM multi_insert_select.raw_events_first, multi_insert_select.reference_table WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT intermediate_result.v4, intermediate_result.v1, intermediate_result.id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(v4 numeric, v1 integer, id bigint)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'user_id' -SET client_min_messages TO WARNING; --- cannot pushdown the query since the JOIN is not equi JOIN --- falls back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events - (user_id, value_4_agg) -SELECT -outer_most.id, max(outer_most.value) - FROM -( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id != f2.id)) as outer_most -GROUP BY outer_most.id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(8 rows) - --- cannot pushdown since foo2 is not join on partition key --- falls back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events - (user_id, value_4_agg) -SELECT - outer_most.id, max(outer_most.value) -FROM -( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.value_1 - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id)) as outer_most -GROUP BY - outer_most.id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> HashAggregate - Group Key: remote_scan.id - Filter: (pg_catalog.sum(remote_scan.worker_column_4) > '10'::numeric) - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(11 rows) - --- cannot push down since foo doesn't have en equi join --- falls back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events - (user_id, value_4_agg) -SELECT - outer_most.id, max(outer_most.value) -FROM -( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id != reference_table.user_id ) AS foo) as f - INNER JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id)) as outer_most -GROUP BY - outer_most.id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(8 rows) - --- some unsupported LATERAL JOINs --- join on averages is not on the partition key --- should fall back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events (user_id, value_4_agg) -SELECT - averages.user_id, avg(averages.value_4) -FROM - (SELECT - raw_events_second.user_id - FROM - reference_table JOIN raw_events_second on (reference_table.user_id = raw_events_second.user_id) - ) reference_ids - JOIN LATERAL - (SELECT - user_id, value_4 - FROM - raw_events_first WHERE - value_4 = reference_ids.user_id) as averages ON true - GROUP BY averages.user_id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(6 rows) - --- join among reference_ids and averages is not on the partition key --- should fall back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events (user_id, value_4_agg) -SELECT - averages.user_id, avg(averages.value_4) -FROM - (SELECT - raw_events_second.user_id - FROM - reference_table JOIN raw_events_second on (reference_table.user_id = raw_events_second.user_id) - ) reference_ids - JOIN LATERAL - (SELECT - user_id, value_4 - FROM - raw_events_first) as averages ON averages.value_4 = reference_ids.user_id - GROUP BY averages.user_id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> HashAggregate - Group Key: remote_scan.user_id - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(8 rows) - --- join among the agg_ids and averages is not on the partition key --- should fall back to route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events (user_id, value_4_agg) -SELECT - averages.user_id, avg(averages.value_4) -FROM - (SELECT - raw_events_second.user_id - FROM - reference_table JOIN raw_events_second on (reference_table.user_id = raw_events_second.user_id) - ) reference_ids - JOIN LATERAL - (SELECT - user_id, value_4 - FROM - raw_events_first) as averages ON averages.user_id = reference_ids.user_id -JOIN LATERAL - (SELECT user_id, value_4 FROM agg_events) as agg_ids ON (agg_ids.value_4 = averages.user_id) - GROUP BY averages.user_id; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - --- Selected value in the WHERE is not partition key, so we cannot use distributed --- INSERT/SELECT and falls back route via coordinator -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT value_1 - FROM raw_events_second); -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(6 rows) - --- same as above but slightly more complex --- since it also includes subquery in FROM as well -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO agg_events - (user_id) -SELECT f2.id FROM - -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id) -WHERE f.id IN (SELECT value_1 - FROM raw_events_second); -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(6 rows) - --- some more semi-anti join tests -SET client_min_messages TO DEBUG2; --- join in where -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN (SELECT raw_events_second.user_id - FROM raw_events_second, raw_events_first - WHERE raw_events_second.user_id = raw_events_first.user_id AND raw_events_first.user_id = 200); -DEBUG: Creating router plan -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second, multi_insert_select.raw_events_first_13300000 raw_events_first_1 WHERE ((raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first_1.user_id) AND (raw_events_first_1.user_id OPERATOR(pg_catalog.=) 200)))) AND (user_id IS NOT NULL)) -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away -RESET client_min_messages; --- we cannot push this down since it is NOT IN --- we use repartition insert/select instead -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id NOT IN (SELECT raw_events_second.user_id - FROM raw_events_second, raw_events_first - WHERE raw_events_second.user_id = raw_events_first.user_id AND raw_events_first.user_id = 200); -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 1 -(6 rows) - -SET client_min_messages TO DEBUG2; --- safe to push down -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE EXISTS (SELECT 1 - FROM raw_events_second - WHERE raw_events_second.user_id =raw_events_first.user_id); -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id))) AND (user_id IS NOT NULL)) --- we cannot push down -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE NOT EXISTS (SELECT 1 - FROM raw_events_second - WHERE raw_events_second.user_id =raw_events_first.user_id); -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((NOT (EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((NOT (EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((NOT (EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((NOT (EXISTS (SELECT 1 FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id OPERATOR(pg_catalog.=) raw_events_first.user_id)))) AND (user_id IS NOT NULL)) --- more complex LEFT JOINs - INSERT INTO agg_events - (user_id, value_4_agg) - SELECT - outer_most.id, max(outer_most.value) - FROM - ( - SELECT f2.id as id, f2.v4 as value FROM - (SELECT - id - FROM (SELECT raw_events_first.user_id AS id - FROM raw_events_first LEFT JOIN - reference_table - ON (raw_events_first.user_id = reference_table.user_id)) AS foo) as f - LEFT JOIN - (SELECT v4, - v1, - id - FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 - ON (f.id = f2.id)) as outer_most - GROUP BY - outer_most.id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300008 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT raw_events_first.user_id AS id FROM (multi_insert_select.raw_events_first_13300000 raw_events_first LEFT JOIN multi_insert_select.reference_table_13300012 reference_table ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)))) foo) f LEFT JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300009 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT raw_events_first.user_id AS id FROM (multi_insert_select.raw_events_first_13300001 raw_events_first LEFT JOIN multi_insert_select.reference_table_13300012 reference_table ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)))) foo) f LEFT JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300010 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT raw_events_first.user_id AS id FROM (multi_insert_select.raw_events_first_13300002 raw_events_first LEFT JOIN multi_insert_select.reference_table_13300012 reference_table ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)))) foo) f LEFT JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -DEBUG: distributed statement: INSERT INTO multi_insert_select.agg_events_13300011 AS citus_table_alias (user_id, value_4_agg) SELECT id, max(value) AS max FROM (SELECT f2.id, f2.v4 AS value FROM ((SELECT foo.id FROM (SELECT raw_events_first.user_id AS id FROM (multi_insert_select.raw_events_first_13300003 raw_events_first LEFT JOIN multi_insert_select.reference_table_13300012 reference_table ON ((raw_events_first.user_id OPERATOR(pg_catalog.=) reference_table.user_id)))) foo) f LEFT JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id)))) outer_most WHERE (id IS NOT NULL) GROUP BY id -RESET client_min_messages; --- cannot push down since the f.id IN is matched with value_1 --- we use repartition insert/select instead -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN ( -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id) -WHERE f.id IN (SELECT value_1 - FROM raw_events_second)); -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(6 rows) - -SET client_min_messages TO DEBUG2; --- same as above, but this time is it safe to push down since --- f.id IN is matched with user_id -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN ( -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id) -WHERE f.id IN (SELECT user_id - FROM raw_events_second)); -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300000 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first_1, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first_1.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300000 raw_events_first_1, multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300004 raw_events_second)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300001 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first_1, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first_1.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300001 raw_events_first_1, multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300005 raw_events_second)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300002 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first_1, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first_1.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300002 raw_events_first_1, multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300006 raw_events_second)))) AND (user_id IS NOT NULL)) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_second_13300007 AS citus_table_alias (user_id) SELECT user_id FROM multi_insert_select.raw_events_first_13300003 raw_events_first WHERE ((user_id OPERATOR(pg_catalog.=) ANY (SELECT f2.id FROM ((SELECT foo.id FROM (SELECT reference_table.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first_1, multi_insert_select.reference_table_13300012 reference_table WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) reference_table.user_id)) foo) f JOIN (SELECT foo2.v4, foo2.v1, foo2.id FROM (SELECT sum(raw_events_second.value_4) AS v4, sum(raw_events_first_1.value_1) AS v1, raw_events_second.user_id AS id FROM multi_insert_select.raw_events_first_13300003 raw_events_first_1, multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (raw_events_first_1.user_id OPERATOR(pg_catalog.=) raw_events_second.user_id) GROUP BY raw_events_second.user_id HAVING (sum(raw_events_second.value_4) OPERATOR(pg_catalog.>) (10)::numeric)) foo2) f2 ON ((f.id OPERATOR(pg_catalog.=) f2.id))) WHERE (f.id OPERATOR(pg_catalog.=) ANY (SELECT raw_events_second.user_id FROM multi_insert_select.raw_events_second_13300007 raw_events_second)))) AND (user_id IS NOT NULL)) -RESET client_min_messages; --- cannot push down since top level user_id is matched with NOT IN -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id NOT IN ( -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id = f2.id) -WHERE f.id IN (SELECT user_id - FROM raw_events_second)); --- cannot push down since join is not equi join (f.id > f2.id) -INSERT INTO raw_events_second - (user_id) -SELECT user_id -FROM raw_events_first -WHERE user_id IN ( -SELECT f2.id FROM -(SELECT - id -FROM (SELECT reference_table.user_id AS id - FROM raw_events_first, - reference_table - WHERE raw_events_first.user_id = reference_table.user_id ) AS foo) as f -INNER JOIN -(SELECT v4, - v1, - id -FROM (SELECT SUM(raw_events_second.value_4) AS v4, - SUM(raw_events_first.value_1) AS v1, - raw_events_second.user_id AS id - FROM raw_events_first, - raw_events_second - WHERE raw_events_first.user_id = raw_events_second.user_id - GROUP BY raw_events_second.user_id - HAVING SUM(raw_events_second.value_4) > 10) AS foo2 ) as f2 -ON (f.id > f2.id) -WHERE f.id IN (SELECT user_id - FROM raw_events_second)); --- we currently not support grouping sets -INSERT INTO agg_events - (user_id, - value_1_agg, - value_2_agg) -SELECT user_id, - Sum(value_1) AS sum_val1, - Sum(value_2) AS sum_val2 -FROM raw_events_second -GROUP BY grouping sets ( ( user_id ), ( value_1 ), ( user_id, value_1 ), ( ) ); -ERROR: could not run distributed query with GROUPING SETS, CUBE, or ROLLUP -HINT: Consider using an equality filter on the distributed table's partition column. --- set back to INFO -SET client_min_messages TO INFO; --- avoid constraint violations -TRUNCATE raw_events_first; --- we don't support LIMIT for subquery pushdown, but --- we recursively plan the query and run it via coordinator -INSERT INTO agg_events(user_id) -SELECT user_id -FROM users_table -WHERE user_id - IN (SELECT - user_id - FROM ( - ( - SELECT - user_id - FROM - ( - SELECT - e1.user_id - FROM - users_table u1, events_table e1 - WHERE - e1.user_id = u1.user_id LIMIT 3 - ) as f_inner - ) - ) AS f2); --- Altering a table and selecting from it using a multi-shard statement --- in the same transaction is allowed because we will use the same --- connections for all co-located placements. -BEGIN; -ALTER TABLE raw_events_second DROP COLUMN value_4; -INSERT INTO raw_events_first SELECT * FROM raw_events_second; -ROLLBACK; --- Alterating a table and selecting from it using a single-shard statement --- in the same transaction is disallowed because we will use a different --- connection. -BEGIN; -ALTER TABLE raw_events_second DROP COLUMN value_4; -INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 100; -ROLLBACK; --- Altering a reference table and then performing an INSERT ... SELECT which --- joins with the reference table is allowed, since the INSERT ... SELECT --- would read from the reference table over the same connections with the ones --- that performed the parallel DDL. -BEGIN; -ALTER TABLE reference_table ADD COLUMN z int; -INSERT INTO raw_events_first (user_id) -SELECT user_id FROM raw_events_second JOIN reference_table USING (user_id); -ROLLBACK; --- the same test with sequential DDL should work fine -BEGIN; -SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; -ALTER TABLE reference_table ADD COLUMN z int; -INSERT INTO raw_events_first (user_id) -SELECT user_id FROM raw_events_second JOIN reference_table USING (user_id); -ROLLBACK; --- Insert after copy is allowed -BEGIN; -COPY raw_events_second (user_id, value_1) FROM STDIN DELIMITER ','; -INSERT INTO raw_events_first SELECT * FROM raw_events_second; -ROLLBACK; --- Insert after copy is currently allowed for single-shard operation. --- Both insert and copy are rolled back successfully. -BEGIN; -COPY raw_events_second (user_id, value_1) FROM STDIN DELIMITER ','; -INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 101; -SELECT user_id FROM raw_events_first WHERE user_id = 101; - user_id ---------------------------------------------------------------------- - 101 -(1 row) - -ROLLBACK; -BEGIN; -INSERT INTO raw_events_first SELECT * FROM raw_events_second; -COPY raw_events_first (user_id, value_1) FROM STDIN DELIMITER ','; -ROLLBACK; -BEGIN; -INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 100; -COPY raw_events_first (user_id, value_1) FROM STDIN DELIMITER ','; -ROLLBACK; --- Similarly, multi-row INSERTs will take part in transactions and reuse connections... -BEGIN; -INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 100; -COPY raw_events_first (user_id, value_1) FROM STDIN DELIMITER ','; -INSERT INTO raw_events_first (user_id, value_1) VALUES (105, 105), (106, 106); -ROLLBACK; --- selecting from views works -CREATE VIEW test_view AS SELECT * FROM raw_events_first; -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (16, now(), 60, 600, 6000.1, 60000); -SELECT count(*) FROM raw_events_second; - count ---------------------------------------------------------------------- - 45 -(1 row) - -INSERT INTO raw_events_second SELECT * FROM test_view; -INSERT INTO raw_events_first (user_id, time, value_1, value_2, value_3, value_4) VALUES - (17, now(), 60, 600, 6000.1, 60000); -INSERT INTO raw_events_second SELECT * FROM test_view WHERE user_id = 17 GROUP BY 1,2,3,4,5,6; -SELECT count(*) FROM raw_events_second; - count ---------------------------------------------------------------------- - 47 -(1 row) - --- intermediate results (CTEs) should be allowed when doing INSERT...SELECT within a CTE -WITH series AS ( - SELECT s AS val FROM generate_series(60,70) s -), -inserts AS ( - INSERT INTO raw_events_second (user_id) - SELECT - user_id - FROM - raw_events_first JOIN series ON (value_1 = val) - RETURNING - NULL -) -SELECT count(*) FROM inserts; - count ---------------------------------------------------------------------- - 2 -(1 row) - --- we need this in our next test -truncate raw_events_first; -SET client_min_messages TO DEBUG2; --- first show that the query works now -INSERT INTO raw_events_first SELECT * FROM raw_events_second; -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300000 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300001 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_second_13300005 raw_events_second WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300002 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_second_13300006 raw_events_second WHERE (user_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300003 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_second_13300007 raw_events_second WHERE (user_id IS NOT NULL) -SET client_min_messages TO INFO; -truncate raw_events_first; -SET client_min_messages TO DEBUG2; --- now show that it works for a single shard query as well -INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 5; -DEBUG: Creating router plan -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300000 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM multi_insert_select.raw_events_second_13300004 raw_events_second WHERE ((user_id OPERATOR(pg_catalog.=) 5) AND (user_id IS NOT NULL)) -DEBUG: Skipping target shard interval 13300001 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300002 since SELECT query for it pruned away -DEBUG: Skipping target shard interval 13300003 since SELECT query for it pruned away -SET client_min_messages TO INFO; --- now do some tests with varchars -INSERT INTO insert_select_varchar_test VALUES ('test_1', 10); -INSERT INTO insert_select_varchar_test VALUES ('test_2', 30); -INSERT INTO insert_select_varchar_test (key, value) -SELECT *, 100 -FROM (SELECT f1.key - FROM (SELECT key - FROM insert_select_varchar_test - GROUP BY 1 - HAVING Count(key) < 3) AS f1, - (SELECT key - FROM insert_select_varchar_test - GROUP BY 1 - HAVING Sum(COALESCE(insert_select_varchar_test.value, 0)) > - 20.0) - AS f2 - WHERE f1.key = f2.key - GROUP BY 1) AS foo; -SELECT * FROM insert_select_varchar_test ORDER BY 1 DESC, 2 DESC; - key | value ---------------------------------------------------------------------- - test_2 | 100 - test_2 | 30 - test_1 | 10 -(3 rows) - --- some tests with DEFAULT columns and constant values --- this test is mostly importantly intended for deparsing the query correctly --- but still it is preferable to have this test here instead of multi_deparse_shard_query -CREATE TABLE table_with_defaults -( - store_id int, - first_name text, - default_1 int DEFAULT 1, - last_name text, - default_2 text DEFAULT '2' -); --- we don't need many shards -SET citus.shard_count = 2; -SELECT create_distributed_table('table_with_defaults', 'store_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- let's see the queries -SET client_min_messages TO DEBUG2; --- a very simple query -INSERT INTO table_with_defaults SELECT * FROM table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, first_name, default_1, last_name, default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, first_name, default_1, last_name, default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- see that defaults are filled -INSERT INTO table_with_defaults (store_id, first_name) -SELECT - store_id, first_name -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, 1 AS default_1, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, 1 AS default_1, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- shuffle one of the defaults and skip the other -INSERT INTO table_with_defaults (default_2, store_id, first_name) -SELECT - default_2, store_id, first_name -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, 1 AS default_1, default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, 1 AS default_1, default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- shuffle both defaults -INSERT INTO table_with_defaults (default_2, store_id, default_1, first_name) -SELECT - default_2, store_id, default_1, first_name -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, default_1, default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, first_name, default_1, default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- use constants instead of non-default column -INSERT INTO table_with_defaults (default_2, last_name, store_id, first_name) -SELECT - default_2, 'Freund', store_id, 'Andres' -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, 'Freund'::text AS last_name, default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, 'Freund'::text AS last_name, default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- use constants instead of non-default column and skip both defauls -INSERT INTO table_with_defaults (last_name, store_id, first_name) -SELECT - 'Freund', store_id, 'Andres' -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, 'Freund'::text AS last_name, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, 'Freund'::text AS last_name, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- use constants instead of default columns -INSERT INTO table_with_defaults (default_2, last_name, store_id, first_name, default_1) -SELECT - 20, last_name, store_id, first_name, 10 -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, first_name, 10, last_name, 20 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, first_name, 10, last_name, 20 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- use constants instead of both default columns and non-default columns -INSERT INTO table_with_defaults (default_2, last_name, store_id, first_name, default_1) -SELECT - 20, 'Freund', store_id, 'Andres', 10 -FROM - table_with_defaults; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 10, 'Freund'::text AS last_name, 20 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, last_name, default_2) SELECT store_id, 'Andres'::text AS first_name, 10, 'Freund'::text AS last_name, 20 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) --- some of the ultimate queries where we have constants, --- defaults and group by entry is not on the target entry -INSERT INTO table_with_defaults (default_2, store_id, first_name) -SELECT - '2000', store_id, 'Andres' -FROM - table_with_defaults -GROUP BY - last_name, store_id; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1 AS default_1, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id -INSERT INTO table_with_defaults (default_1, store_id, first_name, default_2) -SELECT - 1000, store_id, 'Andres', '2000' -FROM - table_with_defaults -GROUP BY - last_name, store_id, first_name; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name -INSERT INTO table_with_defaults (default_1, store_id, first_name, default_2) -SELECT - 1000, store_id, 'Andres', '2000' -FROM - table_with_defaults -GROUP BY - last_name, store_id, first_name, default_2; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name, default_2 -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2000'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name, default_2 -INSERT INTO table_with_defaults (default_1, store_id, first_name) -SELECT - 1000, store_id, 'Andres' -FROM - table_with_defaults -GROUP BY - last_name, store_id, first_name, default_2; -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300017 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300017 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name, default_2 -DEBUG: distributed statement: INSERT INTO multi_insert_select.table_with_defaults_13300018 AS citus_table_alias (store_id, first_name, default_1, default_2) SELECT store_id, 'Andres'::text AS first_name, 1000, '2'::text AS default_2 FROM multi_insert_select.table_with_defaults_13300018 table_with_defaults WHERE (store_id IS NOT NULL) GROUP BY last_name, store_id, first_name, default_2 -RESET client_min_messages; --- Stable function in default should be allowed -ALTER TABLE table_with_defaults ADD COLUMN t timestamptz DEFAULT now(); -INSERT INTO table_with_defaults (store_id, first_name, last_name) -SELECT - store_id, 'first '||store_id, 'last '||store_id -FROM - table_with_defaults -GROUP BY - store_id, first_name, last_name; --- Volatile function in default should be disallowed - SERIAL pseudo-types -CREATE TABLE table_with_serial ( - store_id int, - s bigserial -); -SELECT create_distributed_table('table_with_serial', 'store_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO table_with_serial (store_id) -SELECT - store_id -FROM - table_with_defaults -GROUP BY - store_id; --- Volatile function in default should be disallowed - user-defined sequence -CREATE SEQUENCE user_defined_sequence; -CREATE TABLE table_with_user_sequence ( - store_id int, - s bigint default nextval('user_defined_sequence') -); -SELECT create_distributed_table('table_with_user_sequence', 'store_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO table_with_user_sequence (store_id) -SELECT - store_id -FROM - table_with_defaults -GROUP BY - store_id; --- do some more error/error message checks -SET citus.shard_count TO 4; -SET citus.shard_replication_factor TO 1; -CREATE TABLE text_table (part_col text, val int); -CREATE TABLE char_table (part_col char[], val int); -create table table_with_starts_with_defaults (a int DEFAULT 5, b int, c int); -SELECT create_distributed_table('text_table', 'part_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('char_table','part_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('table_with_starts_with_defaults', 'c'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEBUG; -INSERT INTO text_table (part_col) - SELECT - CASE WHEN part_col = 'onder' THEN 'marco' - END -FROM text_table ; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains a case expression in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT COALESCE(part_col, 'onder') FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains a coalesce expression in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT GREATEST(part_col, 'jason') FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains a min/max expression in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT LEAST(part_col, 'andres') FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains a min/max expression in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT NULLIF(part_col, 'metin') FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an expression that is not a simple column reference in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT part_col isnull FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an expression that is not a simple column reference in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT part_col::text from char_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an explicit coercion in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT (part_col = 'burak') is true FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an expression that is not a simple column reference in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT val FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The data type of the target table's partition column should exactly match the data type of the corresponding simple column reference in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -INSERT INTO text_table (part_col) SELECT val::text FROM text_table; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: Subquery contains an explicit coercion in the same position as the target table's partition column. -HINT: Ensure the target table's partition column has a corresponding simple column reference to a distributed table's partition column in the subquery. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'part_col' -RESET client_min_messages; -insert into table_with_starts_with_defaults (b,c) select b,c FROM table_with_starts_with_defaults; --- Test on partition column without native hash function -CREATE TABLE raw_table -( - id BIGINT, - time DATE -); -CREATE TABLE summary_table -( - time DATE, - count BIGINT -); -SELECT create_distributed_table('raw_table', 'time'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('summary_table', 'time'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO raw_table VALUES(1, '11-11-1980'); -INSERT INTO summary_table SELECT time, COUNT(*) FROM raw_table GROUP BY time; -SELECT * FROM summary_table; - time | count ---------------------------------------------------------------------- - 11-11-1980 | 1 -(1 row) - --- Test INSERT ... SELECT via coordinator --- Select from constants -TRUNCATE raw_events_first; -INSERT INTO raw_events_first (user_id, value_1) -SELECT * FROM (VALUES (1,2), (3,4), (5,6)) AS v(int,int); -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 2 - 3 | 4 - 5 | 6 -(3 rows) - --- Select from local functions -TRUNCATE raw_events_first; -CREATE SEQUENCE insert_select_test_seq; -SET client_min_messages TO DEBUG; -INSERT INTO raw_events_first (user_id, value_1, value_2) -SELECT - s, nextval('insert_select_test_seq'), (random()*10)::int -FROM - generate_series(1, 5) s; -DEBUG: Creating router plan -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; -DEBUG: Router planner cannot handle multi-shard select queries - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- ON CONFLICT is supported -INSERT INTO raw_events_first (user_id, value_1) -SELECT s, nextval('insert_select_test_seq') FROM generate_series(1, 5) s -ON CONFLICT DO NOTHING; -DEBUG: Creating router plan -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300000 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300000'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) ON CONFLICT DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300001 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300001'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) ON CONFLICT DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300002 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300002'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) ON CONFLICT DO NOTHING -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300003 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300003'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) ON CONFLICT DO NOTHING --- RETURNING is supported -INSERT INTO raw_events_first (user_id, value_1) -SELECT s, nextval('insert_select_test_seq') FROM generate_series(1, 5) s -RETURNING *; -DEBUG: Creating router plan -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300000 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300000'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300001 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300001'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300002 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300002'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: distributed statement: INSERT INTO multi_insert_select.raw_events_first_13300003 AS citus_table_alias (user_id, value_1) SELECT user_id, value_1 FROM read_intermediate_result('insert_select_XXX_13300003'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 - user_id | time | value_1 | value_2 | value_3 | value_4 ---------------------------------------------------------------------- - 1 | | 11 | | | - 2 | | 12 | | | - 3 | | 13 | | | - 4 | | 14 | | | - 5 | | 15 | | | -(5 rows) - -RESET client_min_messages; --- INSERT ... SELECT and multi-shard SELECT in the same transaction is supported -TRUNCATE raw_events_first; -BEGIN; -INSERT INTO raw_events_first (user_id, value_1) -SELECT s, s FROM generate_series(1, 5) s; -SELECT user_id, value_1 FROM raw_events_first ORDER BY 1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - -ROLLBACK; --- INSERT ... SELECT and single-shard SELECT in the same transaction is supported -TRUNCATE raw_events_first; -BEGIN; -INSERT INTO raw_events_first (user_id, value_1) -SELECT s, s FROM generate_series(1, 5) s; -SELECT user_id, value_1 FROM raw_events_first WHERE user_id = 1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -COMMIT; --- Select from local table -TRUNCATE raw_events_first; -CREATE TEMPORARY TABLE raw_events_first_local AS -SELECT s AS u, 2*s AS v FROM generate_series(1, 5) s; -INSERT INTO raw_events_first (user_id, value_1) -SELECT u, v FROM raw_events_first_local; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 2 - 2 | 4 - 3 | 6 - 4 | 8 - 5 | 10 -(5 rows) - --- Use columns in opposite order -TRUNCATE raw_events_first; -INSERT INTO raw_events_first (value_1, user_id) -SELECT u, v FROM raw_events_first_local; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 2 | 1 - 4 | 2 - 6 | 3 - 8 | 4 - 10 | 5 -(5 rows) - --- Set operations can work with opposite column order -TRUNCATE raw_events_first; -INSERT INTO raw_events_first (value_3, user_id) -( SELECT v, u::bigint FROM raw_events_first_local ) -UNION ALL -( SELECT v, u FROM raw_events_first_local ); -SELECT user_id, value_3 FROM raw_events_first ORDER BY user_id, value_3; - user_id | value_3 ---------------------------------------------------------------------- - 1 | 2 - 1 | 2 - 2 | 4 - 2 | 4 - 3 | 6 - 3 | 6 - 4 | 8 - 4 | 8 - 5 | 10 - 5 | 10 -(10 rows) - --- Select from other distributed table with limit -TRUNCATE raw_events_first; -TRUNCATE raw_events_second; -INSERT INTO raw_events_second (user_id, value_4) -SELECT s, 3*s FROM generate_series (1,5) s; -INSERT INTO raw_events_first (user_id, value_1) -SELECT user_id, value_4 FROM raw_events_second LIMIT 5; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 3 - 2 | 6 - 3 | 9 - 4 | 12 - 5 | 15 -(5 rows) - --- CTEs are supported in local queries -TRUNCATE raw_events_first; -WITH removed_rows AS ( - DELETE FROM raw_events_first_local RETURNING u -) -INSERT INTO raw_events_first (user_id, value_1) -WITH value AS (SELECT 1) -SELECT * FROM removed_rows, value; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 1 - 3 | 1 - 4 | 1 - 5 | 1 -(5 rows) - --- nested CTEs are also supported -TRUNCATE raw_events_first; -INSERT INTO raw_events_first_local SELECT s, 2*s FROM generate_series(0, 10) s; -WITH rows_to_remove AS ( - SELECT u FROM raw_events_first_local WHERE u > 0 -), -removed_rows AS ( - DELETE FROM raw_events_first_local - WHERE u IN (SELECT * FROM rows_to_remove) - RETURNING u, v -) -INSERT INTO raw_events_first (user_id, value_1) -WITH ultra_rows AS ( - WITH numbers AS ( - SELECT s FROM generate_series(1,10) s - ), - super_rows AS ( - SELECT u, v FROM removed_rows JOIN numbers ON (u = s) - ) - SELECT * FROM super_rows LIMIT 5 -) -SELECT u, v FROM ultra_rows; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 2 - 2 | 4 - 3 | 6 - 4 | 8 - 5 | 10 -(5 rows) - --- CTEs with duplicate names are also supported -TRUNCATE raw_events_first; -WITH super_rows AS ( - SELECT u FROM raw_events_first_local -) -INSERT INTO raw_events_first (user_id, value_1) -WITH super_rows AS ( - SELECT * FROM super_rows GROUP BY u -) -SELECT u, 5 FROM super_rows; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 0 | 5 -(1 row) - --- CTEs are supported in router queries -TRUNCATE raw_events_first; -WITH user_two AS ( - SELECT user_id, value_4 FROM raw_events_second WHERE user_id = 2 -) -INSERT INTO raw_events_first (user_id, value_1) -SELECT * FROM user_two; -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 2 | 6 -(1 row) - --- CTEs are supported when there are name collisions -WITH numbers AS ( - SELECT s FROM generate_series(1,10) s -) -INSERT INTO raw_events_first(user_id, value_1) -WITH numbers AS ( - SELECT s, s FROM generate_series(1,5) s -) -SELECT * FROM numbers; --- Select into distributed table with a sequence -CREATE TABLE "CaseSensitiveTable" ("UserID" int, "Value1" int); -SELECT create_distributed_table('"CaseSensitiveTable"', 'UserID'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO "CaseSensitiveTable" -SELECT s, s FROM generate_series(1,10) s; -SELECT * FROM "CaseSensitiveTable" ORDER BY "UserID"; - UserID | Value1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 6 - 7 | 7 - 8 | 8 - 9 | 9 - 10 | 10 -(10 rows) - -DROP TABLE "CaseSensitiveTable"; --- Select into distributed table with a sequence -CREATE TABLE dist_table_with_sequence (user_id serial, value_1 serial); -SELECT create_distributed_table('dist_table_with_sequence', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- from local query -INSERT INTO dist_table_with_sequence (value_1) -SELECT s FROM generate_series(1,5) s; -SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- from a distributed query -INSERT INTO dist_table_with_sequence (value_1) -SELECT value_1 FROM dist_table_with_sequence ORDER BY value_1; -SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 1 - 7 | 2 - 8 | 3 - 9 | 4 - 10 | 5 -(10 rows) - -TRUNCATE dist_table_with_sequence; -INSERT INTO dist_table_with_sequence (user_id) -SELECT user_id FROM raw_events_second ORDER BY user_id; -SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - -WITH top10 AS ( - SELECT user_id FROM raw_events_second WHERE value_1 IS NOT NULL ORDER BY value_1 LIMIT 10 -) -INSERT INTO dist_table_with_sequence (value_1) -SELECT * FROM top10; -SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- router queries become logical planner queries when there is a nextval call -INSERT INTO dist_table_with_sequence (user_id) -SELECT user_id FROM dist_table_with_sequence WHERE user_id = 1; -SELECT * FROM dist_table_with_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 1 | 6 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(6 rows) - -DROP TABLE dist_table_with_sequence; --- Select into distributed table with a user-defined sequence -CREATE SEQUENCE seq1; -CREATE SEQUENCE seq2; -CREATE TABLE dist_table_with_user_sequence (user_id int default nextval('seq1'), value_1 bigint default nextval('seq2')); -SELECT create_distributed_table('dist_table_with_user_sequence', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- from local query -INSERT INTO dist_table_with_user_sequence (value_1) -SELECT s FROM generate_series(1,5) s; -SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- from a distributed query -INSERT INTO dist_table_with_user_sequence (value_1) -SELECT value_1 FROM dist_table_with_user_sequence ORDER BY value_1; -SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 - 6 | 1 - 7 | 2 - 8 | 3 - 9 | 4 - 10 | 5 -(10 rows) - -TRUNCATE dist_table_with_user_sequence; -INSERT INTO dist_table_with_user_sequence (user_id) -SELECT user_id FROM raw_events_second ORDER BY user_id; -SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - -WITH top10 AS ( - SELECT user_id FROM raw_events_second WHERE value_1 IS NOT NULL ORDER BY value_1 LIMIT 10 -) -INSERT INTO dist_table_with_user_sequence (value_1) -SELECT * FROM top10; -SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- router queries become logical planner queries when there is a nextval call -INSERT INTO dist_table_with_user_sequence (user_id) -SELECT user_id FROM dist_table_with_user_sequence WHERE user_id = 1; -SELECT * FROM dist_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 1 - 1 | 6 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(6 rows) - -DROP TABLE dist_table_with_user_sequence; -DROP SEQUENCE seq1, seq2; --- Select from distributed table into reference table -CREATE TABLE ref_table (user_id serial, value_1 int); -SELECT create_reference_table('ref_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO ref_table -SELECT user_id, value_1 FROM raw_events_second; -SELECT * FROM ref_table ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 2 | - 3 | - 4 | - 5 | -(5 rows) - -INSERT INTO ref_table (value_1) -SELECT value_1 FROM raw_events_second ORDER BY value_1; -SELECT * FROM ref_table ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 1 | - 2 | - 2 | - 3 | - 3 | - 4 | - 4 | - 5 | - 5 | -(10 rows) - -INSERT INTO ref_table SELECT * FROM ref_table; -SELECT * FROM ref_table ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 1 | - 1 | - 1 | - 2 | - 2 | - 2 | - 2 | - 3 | - 3 | - 3 | - 3 | - 4 | - 4 | - 4 | - 4 | - 5 | - 5 | - 5 | - 5 | -(20 rows) - -DROP TABLE ref_table; --- Select from distributed table into reference table with user-defined sequence -CREATE SEQUENCE seq1; -CREATE TABLE ref_table_with_user_sequence (user_id int default nextval('seq1'), value_1 int); -SELECT create_reference_table('ref_table_with_user_sequence'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO ref_table_with_user_sequence -SELECT user_id, value_1 FROM raw_events_second; -SELECT * FROM ref_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 2 | - 3 | - 4 | - 5 | -(5 rows) - -INSERT INTO ref_table_with_user_sequence (value_1) -SELECT value_1 FROM raw_events_second ORDER BY value_1; -SELECT * FROM ref_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 1 | - 2 | - 2 | - 3 | - 3 | - 4 | - 4 | - 5 | - 5 | -(10 rows) - -INSERT INTO ref_table_with_user_sequence SELECT * FROM ref_table_with_user_sequence; -SELECT * FROM ref_table_with_user_sequence ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | - 1 | - 1 | - 1 | - 2 | - 2 | - 2 | - 2 | - 3 | - 3 | - 3 | - 3 | - 4 | - 4 | - 4 | - 4 | - 5 | - 5 | - 5 | - 5 | -(20 rows) - -DROP TABLE ref_table_with_user_sequence; -DROP SEQUENCE seq1; --- Select from reference table into reference table -CREATE TABLE ref1 (d timestamptz); -SELECT create_reference_table('ref1'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE ref2 (d date); -SELECT create_reference_table('ref2'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO ref2 VALUES ('2017-10-31'); -INSERT INTO ref1 SELECT * FROM ref2; -SELECT count(*) from ref1; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- also test with now() -INSERT INTO ref1 SELECT now() FROM ref2; -SELECT count(*) from ref1; - count ---------------------------------------------------------------------- - 2 -(1 row) - -DROP TABLE ref1; -DROP TABLE ref2; --- Select into an append-partitioned table is not supported -CREATE TABLE insert_append_table (user_id int, value_4 bigint); -SELECT create_distributed_table('insert_append_table', 'user_id', 'append'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO insert_append_table (user_id, value_4) -SELECT user_id, 1 FROM raw_events_second LIMIT 5; -ERROR: INSERT ... SELECT into an append-distributed table is not supported -DROP TABLE insert_append_table; --- Insert from other distributed table as prepared statement -TRUNCATE raw_events_first; -PREPARE insert_prep(int) AS -INSERT INTO raw_events_first (user_id, value_1) -SELECT $1, value_4 FROM raw_events_second ORDER BY value_4 LIMIT 1; -EXECUTE insert_prep(1); -EXECUTE insert_prep(2); -EXECUTE insert_prep(3); -EXECUTE insert_prep(4); -EXECUTE insert_prep(5); -EXECUTE insert_prep(6); -SELECT user_id, value_1 FROM raw_events_first ORDER BY user_id, value_1; - user_id | value_1 ---------------------------------------------------------------------- - 1 | 3 - 2 | 3 - 3 | 3 - 4 | 3 - 5 | 3 - 6 | 3 -(6 rows) - --- Inserting into views is handled via coordinator -TRUNCATE raw_events_first; -INSERT INTO test_view -SELECT * FROM raw_events_second; -SELECT user_id, value_4 FROM test_view ORDER BY user_id, value_4; - user_id | value_4 ---------------------------------------------------------------------- - 1 | 3 - 2 | 6 - 3 | 9 - 4 | 12 - 5 | 15 -(5 rows) - --- Drop the view now, because the column we are about to drop depends on it -DROP VIEW test_view; --- Make sure we handle dropped columns correctly -CREATE TABLE drop_col_table (col1 text, col2 text, col3 text); -SELECT create_distributed_table('drop_col_table', 'col2'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -ALTER TABLE drop_col_table DROP COLUMN col1; -INSERT INTO drop_col_table (col3, col2) -SELECT value_4, user_id FROM raw_events_second LIMIT 5; -SELECT * FROM drop_col_table ORDER BY col2, col3; - col2 | col3 ---------------------------------------------------------------------- - 1 | 3 - 2 | 6 - 3 | 9 - 4 | 12 - 5 | 15 -(5 rows) - --- make sure the tuple went to the right shard -SELECT * FROM drop_col_table WHERE col2 = '1'; - col2 | col3 ---------------------------------------------------------------------- - 1 | 3 -(1 row) - -RESET client_min_messages; --- make sure casts are handled correctly -CREATE TABLE coerce_events(user_id int, time timestamp, value_1 numeric); -SELECT create_distributed_table('coerce_events', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE coerce_agg (user_id int, value_1_agg int); -SELECT create_distributed_table('coerce_agg', 'user_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO coerce_events(user_id, value_1) VALUES (1, 1), (2, 2), (10, 10); --- numeric -> int (straight function) -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -ORDER BY 2 DESC, 1 DESC -LIMIT 5; --- int -> text -ALTER TABLE coerce_agg ALTER COLUMN value_1_agg TYPE text; -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -LIMIT 5; -SELECT * FROM coerce_agg ORDER BY 1 DESC, 2 DESC; - user_id | value_1_agg ---------------------------------------------------------------------- - 10 | 10 - 10 | 10 - 2 | 2 - 2 | 2 - 1 | 1 - 1 | 1 -(6 rows) - -TRUNCATE coerce_agg; --- int -> char(1) -ALTER TABLE coerce_agg ALTER COLUMN value_1_agg TYPE char(1); -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -LIMIT 5; -ERROR: value too long for type character(1) -SELECT * FROM coerce_agg ORDER BY 1 DESC, 2 DESC; - user_id | value_1_agg ---------------------------------------------------------------------- -(0 rows) - -TRUNCATE coerce_agg; -TRUNCATE coerce_events; --- char(5) -> char(1) -ALTER TABLE coerce_events ALTER COLUMN value_1 TYPE char(5); -INSERT INTO coerce_events(user_id, value_1) VALUES (1, 'aaaaa'), (2, 'bbbbb'); -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -LIMIT 5; -ERROR: value too long for type character(1) --- char(1) -> char(5) -ALTER TABLE coerce_events ALTER COLUMN value_1 TYPE char(1) USING value_1::char(1); -ALTER TABLE coerce_agg ALTER COLUMN value_1_agg TYPE char(5); -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -LIMIT 5; -SELECT * FROM coerce_agg ORDER BY 1 DESC, 2 DESC; - user_id | value_1_agg ---------------------------------------------------------------------- - 2 | b - 1 | a -(2 rows) - -TRUNCATE coerce_agg; -TRUNCATE coerce_events; --- integer -> integer (check VALUE < 5) -ALTER TABLE coerce_events ALTER COLUMN value_1 TYPE integer USING NULL; -ALTER TABLE coerce_agg ALTER COLUMN value_1_agg TYPE integer USING NULL; -ALTER TABLE coerce_agg ADD CONSTRAINT small_number CHECK (value_1_agg < 5); -INSERT INTO coerce_events (user_id, value_1) VALUES (1, 1), (10, 10); -\set VERBOSITY TERSE -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop; -ERROR: new row for relation "coerce_agg_13300067" violates check constraint "small_number_13300067" -\set VERBOSITY DEFAULT -SELECT * FROM coerce_agg ORDER BY 1 DESC, 2 DESC; - user_id | value_1_agg ---------------------------------------------------------------------- -(0 rows) - --- integer[3] -> text[3] -TRUNCATE coerce_events; -ALTER TABLE coerce_events ALTER COLUMN value_1 TYPE integer[3] USING NULL; -INSERT INTO coerce_events(user_id, value_1) VALUES (1, '{1,1,1}'), (2, '{2,2,2}'); -ALTER TABLE coerce_agg DROP COLUMN value_1_agg; -ALTER TABLE coerce_agg ADD COLUMN value_1_agg text[3]; -INSERT INTO coerce_agg(user_id, value_1_agg) -SELECT * -FROM ( - SELECT user_id, value_1 - FROM coerce_events -) AS ftop -LIMIT 5; -SELECT * FROM coerce_agg ORDER BY 1 DESC, 2 DESC; - user_id | value_1_agg ---------------------------------------------------------------------- - 2 | {2,2,2} - 1 | {1,1,1} -(2 rows) - --- INSERT..SELECT + prepared statements + recursive planning -BEGIN; -PREPARE prepared_recursive_insert_select AS -INSERT INTO users_table -SELECT * FROM users_table -WHERE value_1 IN (SELECT value_2 FROM events_table OFFSET 0); -EXECUTE prepared_recursive_insert_select; -EXECUTE prepared_recursive_insert_select; -EXECUTE prepared_recursive_insert_select; -EXECUTE prepared_recursive_insert_select; -EXECUTE prepared_recursive_insert_select; -EXECUTE prepared_recursive_insert_select; -ROLLBACK; --- upsert with on conflict update distribution column is unsupported -INSERT INTO agg_events AS ae - ( - user_id, - value_1_agg, - agg_time - ) -SELECT user_id, - value_1, - time -FROM raw_events_first -ON conflict (user_id, value_1_agg) -DO UPDATE - SET user_id = 42 -RETURNING user_id, value_1_agg; -ERROR: modifying the partition value of rows is not allowed --- test a small citus.remote_copy_flush_threshold -BEGIN; -SET LOCAL citus.remote_copy_flush_threshold TO 1; -INSERT INTO raw_events_first -SELECT * FROM raw_events_first OFFSET 0 -ON CONFLICT DO NOTHING; -ABORT; --- test fix for issue https://github.com/citusdata/citus/issues/5891 -CREATE TABLE dist_table_1( -dist_col integer, -int_col integer, -text_col_1 text, -text_col_2 text -); -SELECT create_distributed_table('dist_table_1', 'dist_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO dist_table_1 VALUES (1, 1, 'string', 'string'); -CREATE TABLE dist_table_2( -dist_col integer, -int_col integer -); -SELECT create_distributed_table('dist_table_2', 'dist_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO dist_table_2 VALUES (1, 1); -with a as (select random()) INSERT INTO dist_table_1 -SELECT -t1.dist_col, -1, -'string', -'string' -FROM a, dist_table_1 t1 -join dist_table_2 t2 using (dist_col) -limit 1 -returning text_col_1; - text_col_1 ---------------------------------------------------------------------- - string -(1 row) - -CREATE TABLE dist_table_3( -dist_col bigint, -int_col integer -); -SELECT create_distributed_table('dist_table_3', 'dist_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- dist_table_2 and dist_table_3 are non-colocated source tables. Repartitioning is also not possible due to --- different types for distribution columns. Citus would not be able to handle this complex insert select. -INSERT INTO dist_table_1 SELECT dist_table_2.dist_col FROM dist_table_2 JOIN dist_table_3 USING(dist_col); -ERROR: complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator -CREATE TABLE dist_table_4( -dist_col integer, -int_col integer -); -SELECT create_distributed_table('dist_table_4', 'dist_col'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- Even if target table distribution column is colocated with dist_table_2's distributed column, source tables dist_table_2 and dist_table_4 --- are non-colocated. Hence, SELECT part of the query should be pulled to coordinator. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_1 SELECT dist_table_2.dist_col FROM dist_table_2 JOIN dist_table_4 ON dist_table_2.dist_col = dist_table_4.int_col; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 6 -(4 rows) - --- For INSERT SELECT, when a lateral query references an outer query, push-down is possible even if limit clause exists in the lateral query. --- It is because subquery with limit does not need to be merged at coordinator as it is a lateral query. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_1 SELECT d1.dist_col FROM dist_table_1 d1 LEFT JOIN LATERAL (SELECT * FROM dist_table_2 d2 WHERE d1.dist_col = d2.dist_col LIMIT 3) dummy USING(dist_col); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 -(2 rows) - --- For INSERT SELECT, when push-down is NOT possible when limit clause exists in a subquery at SELECT part of INSERT SELECT. --- It is because the subquery with limit needs to be merged at coordinator. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_1 SELECT d1.dist_col FROM dist_table_1 d1 LEFT JOIN (SELECT * FROM dist_table_2 LIMIT 3) dummy USING(dist_col); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Limit - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(7 rows) - -CREATE TABLE dist_table_5(id int, id2 int); -SELECT create_distributed_table('dist_table_5','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE dist_table_6(id int, id2 int); -SELECT create_distributed_table('dist_table_6','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- verify that insert select with union can be pushed down since UNION clause has FROM clause at top level query. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5(id) SELECT id FROM (SELECT id FROM dist_table_5 UNION SELECT id FROM dist_table_6) dummy; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 -(2 rows) - --- verify that insert select with sublink can be pushed down when tables are colocated. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT id, (SELECT id FROM dist_table_5 WHERE dist_table_5.id = dist_table_6.id) FROM dist_table_6; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 -(2 rows) - -CREATE TABLE ref_table_1(id int); -SELECT create_reference_table('ref_table_1'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - --- verify that insert select with sublink cannot be pushed down when from clause does not contain any distributed relation. -INSERT INTO dist_table_5 SELECT id, (SELECT id FROM dist_table_5 WHERE dist_table_5.id = ref_table_1.id) FROM ref_table_1; -ERROR: correlated subqueries are not supported when the FROM clause contains a reference table --- verify that insert select cannot be pushed down when we have recurring range table in from clause. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT id, (SELECT id FROM ref_table_1 WHERE id = 1) FROM ref_table_1; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - Task Count: 1 -(4 rows) - --- verify that insert select cannot be pushed down when we have reference table in outside of outer join. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT a.id FROM dist_table_5 a LEFT JOIN ref_table_1 b ON (true) RIGHT JOIN ref_table_1 c ON (true); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(6 rows) - --- verify that insert select cannot be pushed down when it has a recurring outer join in a subquery. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT id FROM ref_table_1 LEFT JOIN dist_table_5 USING(id); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(6 rows) - -CREATE TABLE loc_table_1(id int); --- verify that insert select cannot be pushed down when it contains join between local and distributed tables. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT id FROM dist_table_5 JOIN loc_table_1 USING(id); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Seq Scan on loc_table_1 - Task Count: 4 -(6 rows) - -CREATE VIEW view_1 AS - SELECT id FROM dist_table_6; -CREATE MATERIALIZED VIEW view_2 AS - SELECT id FROM dist_table_6; --- verify that insert select cannot be pushed down when it contains view. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT * FROM view_1; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 -(2 rows) - --- verify that insert select cannot be pushed down when it contains materialized view. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 SELECT * FROM view_2; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Seq Scan on view_2 -(3 rows) - -CREATE TABLE append_table(id integer, data text, int_data int); -SELECT create_distributed_table('append_table', 'id', 'append'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT master_create_empty_shard('append_table'); - master_create_empty_shard ---------------------------------------------------------------------- - 13300096 -(1 row) - --- verify that insert select push down for append tables are not supported. -INSERT INTO append_table SELECT * FROM append_table; -ERROR: INSERT ... SELECT into an append-distributed table is not supported --- verify that CTEs at top level of INSERT SELECT, that can normally be inlined, would not be inlined by INSERT SELECT pushdown planner --- and handled by pull to coordinator. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) WITH cte_1 AS (SELECT id FROM dist_table_5 WHERE id > 5) - INSERT INTO dist_table_5 - SELECT id FROM dist_table_5 JOIN cte_1 USING(id) OFFSET 5; -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Limit - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(7 rows) - --- verify that CTEs at top level of SELECT part, would be inlined by Postgres and pushed down by INSERT SELECT planner. -SELECT coordinator_plan($$ - EXPLAIN (COSTS FALSE) INSERT INTO dist_table_5 - WITH cte_1 AS (SELECT id FROM dist_table_5 WHERE id = 5) - SELECT id FROM dist_table_5 JOIN cte_1 USING(id); -$$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 -(2 rows) - -SET client_min_messages TO ERROR; -DROP SCHEMA multi_insert_select CASCADE; diff --git a/src/test/regress/expected/multi_insert_select_conflict.out b/src/test/regress/expected/multi_insert_select_conflict.out index 5c06719d3..429ff024e 100644 --- a/src/test/regress/expected/multi_insert_select_conflict.out +++ b/src/test/regress/expected/multi_insert_select_conflict.out @@ -1,17 +1,6 @@ -- -- MULTI_INSERT_SELECT_CONFLICT -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA on_conflict; SET search_path TO on_conflict, public; SET citus.next_shard_id TO 1900000; diff --git a/src/test/regress/expected/multi_insert_select_conflict_0.out b/src/test/regress/expected/multi_insert_select_conflict_0.out deleted file mode 100644 index 4c2add1d7..000000000 --- a/src/test/regress/expected/multi_insert_select_conflict_0.out +++ /dev/null @@ -1,600 +0,0 @@ --- --- MULTI_INSERT_SELECT_CONFLICT --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA on_conflict; -SET search_path TO on_conflict, public; -SET citus.next_shard_id TO 1900000; -SET citus.shard_replication_factor TO 1; -CREATE TABLE target_table(col_1 int primary key, col_2 int); -SELECT create_distributed_table('target_table','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO target_table VALUES(1,2),(2,3),(3,4),(4,5),(5,6); -CREATE TABLE source_table_1(col_1 int primary key, col_2 int, col_3 int); -SELECT create_distributed_table('source_table_1','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_1 VALUES(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5); -CREATE TABLE source_table_2(col_1 int, col_2 int, col_3 int); -SELECT create_distributed_table('source_table_2','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_2 VALUES(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); -SET client_min_messages to debug1; --- Generate series directly on the coordinator and on conflict do nothing -INSERT INTO target_table (col_1, col_2) -SELECT - s, s -FROM - generate_series(1,10) s -ON CONFLICT DO NOTHING; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator --- Generate series directly on the coordinator and on conflict update the target table -INSERT INTO target_table (col_1, col_2) -SELECT s, s -FROM - generate_series(1,10) s -ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator --- Since partition columns do not match, pull the data to the coordinator --- and do not change conflicted values -INSERT INTO target_table -SELECT - col_2, col_3 -FROM - source_table_1 -ON CONFLICT DO NOTHING; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: performing repartitioned INSERT ... SELECT --- Since partition columns do not match, pull the data to the coordinator --- and update the non-partition column. Query is wrapped by CTE to return --- ordered result. -WITH inserted_table AS ( - INSERT INTO target_table - SELECT - col_2, col_3 - FROM - source_table_1 - ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 RETURNING * -) SELECT * FROM inserted_table ORDER BY 1; -DEBUG: generating subplan XXX_1 for CTE inserted_table: INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_2, col_3 FROM on_conflict.source_table_1 ON CONFLICT(col_1) DO UPDATE SET col_2 = excluded.col_2 RETURNING target_table.col_1, target_table.col_2 -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The target table's partition column should correspond to a partition column in the subquery. -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) inserted_table ORDER BY col_1 -DEBUG: performing repartitioned INSERT ... SELECT - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- Subquery should be recursively planned due to the limit and do nothing on conflict -INSERT INTO target_table -SELECT - col_1, col_2 -FROM ( - SELECT - col_1, col_2, col_3 - FROM - source_table_1 - LIMIT 5 -) as foo -ON CONFLICT DO NOTHING; -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) foo -DEBUG: Collecting INSERT ... SELECT results on coordinator --- Subquery should be recursively planned due to the limit and update on conflict --- Query is wrapped by CTE to return ordered result. -WITH inserted_table AS ( - INSERT INTO target_table - SELECT - col_1, col_2 - FROM ( - SELECT - col_1, col_2, col_3 - FROM - source_table_1 - LIMIT 5 - ) as foo - ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 RETURNING * -) SELECT * FROM inserted_table ORDER BY 1; -DEBUG: generating subplan XXX_1 for CTE inserted_table: INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_1, col_2 FROM (SELECT source_table_1.col_1, source_table_1.col_2, source_table_1.col_3 FROM on_conflict.source_table_1 LIMIT 5) foo ON CONFLICT(col_1) DO UPDATE SET col_2 = excluded.col_2 RETURNING target_table.col_1, target_table.col_2 -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) foo -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) inserted_table ORDER BY col_1 -DEBUG: Collecting INSERT ... SELECT results on coordinator - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 1 - 2 | 2 - 3 | 3 - 4 | 4 - 5 | 5 -(5 rows) - --- Test with multiple subqueries. Query is wrapped by CTE to return ordered result. -WITH inserted_table AS ( - INSERT INTO target_table - SELECT - col_1, col_2 - FROM ( - (SELECT - col_1, col_2, col_3 - FROM - source_table_1 - LIMIT 5) - UNION - (SELECT - col_1, col_2, col_3 - FROM - source_table_2 - LIMIT 5) - ) as foo - ON CONFLICT(col_1) DO UPDATE SET col_2 = 0 RETURNING * -) SELECT * FROM inserted_table ORDER BY 1; -DEBUG: generating subplan XXX_1 for CTE inserted_table: INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_1, col_2 FROM ((SELECT source_table_1.col_1, source_table_1.col_2, source_table_1.col_3 FROM on_conflict.source_table_1 LIMIT 5) UNION (SELECT source_table_2.col_1, source_table_2.col_2, source_table_2.col_3 FROM on_conflict.source_table_2 LIMIT 5)) foo ON CONFLICT(col_1) DO UPDATE SET col_2 = 0 RETURNING target_table.col_1, target_table.col_2 -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 LIMIT 5 -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_2 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_2 LIMIT 5 -DEBUG: generating subplan XXX_3 for subquery SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer) UNION SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) foo -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) inserted_table ORDER BY col_1 -DEBUG: Collecting INSERT ... SELECT results on coordinator - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 0 - 2 | 0 - 3 | 0 - 4 | 0 - 5 | 0 - 6 | 0 - 7 | 0 - 8 | 0 - 9 | 0 - 10 | 0 -(10 rows) - --- Get the select part from cte and do nothing on conflict -WITH cte AS MATERIALIZED ( - SELECT col_1, col_2 FROM source_table_1 -) -INSERT INTO target_table SELECT * FROM cte ON CONFLICT DO NOTHING; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: generating subplan XXX_1 for CTE cte: SELECT col_1, col_2 FROM on_conflict.source_table_1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT cte.col_1, cte.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator --- Get the select part from cte and update on conflict -WITH cte AS MATERIALIZED ( - SELECT col_1, col_2 FROM source_table_1 -) -INSERT INTO target_table SELECT * FROM cte ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: generating subplan XXX_1 for CTE cte: SELECT col_1, col_2 FROM on_conflict.source_table_1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT cte.col_1, cte.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 2 - 2 | 3 - 3 | 4 - 4 | 5 - 5 | 6 - 6 | 0 - 7 | 0 - 8 | 0 - 9 | 0 - 10 | 0 -(10 rows) - --- Test with multiple CTEs -WITH cte AS( - SELECT col_1, col_2 FROM source_table_1 -), cte_2 AS( - SELECT col_1, col_2 FROM source_table_2 -) -INSERT INTO target_table ((SELECT * FROM cte) UNION (SELECT * FROM cte_2)) ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: CTE cte is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: performing repartitioned INSERT ... SELECT -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 2 - 2 | 3 - 3 | 4 - 4 | 5 - 5 | 6 - 6 | 7 - 7 | 8 - 8 | 9 - 9 | 10 - 10 | 11 -(10 rows) - -WITH inserted_table AS MATERIALIZED ( - WITH cte AS MATERIALIZED ( - SELECT col_1, col_2, col_3 FROM source_table_1 - ), cte_2 AS MATERIALIZED ( - SELECT col_1, col_2 FROM cte - ) - INSERT INTO target_table SELECT * FROM cte_2 ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1 RETURNING * -) SELECT * FROM inserted_table ORDER BY 1; -DEBUG: generating subplan XXX_1 for CTE inserted_table: WITH cte AS MATERIALIZED (SELECT source_table_1.col_1, source_table_1.col_2, source_table_1.col_3 FROM on_conflict.source_table_1), cte_2 AS MATERIALIZED (SELECT cte.col_1, cte.col_2 FROM cte) INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_1, col_2 FROM cte_2 ON CONFLICT(col_1) DO UPDATE SET col_2 = (excluded.col_2 OPERATOR(pg_catalog.+) 1) RETURNING target_table.col_1, target_table.col_2 -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: generating subplan XXX_1 for CTE cte: SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) cte -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT cte_2.col_1, cte_2.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte_2) citus_insert_select_subquery -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) inserted_table ORDER BY col_1 -DEBUG: Collecting INSERT ... SELECT results on coordinator - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 2 - 2 | 3 - 3 | 4 - 4 | 5 - 5 | 6 -(5 rows) - -WITH cte AS MATERIALIZED ( - WITH basic AS MATERIALIZED ( - SELECT col_1, col_2 FROM source_table_1 - ) - INSERT INTO target_table (SELECT * FROM basic) ON CONFLICT DO NOTHING RETURNING * -) -UPDATE target_table SET col_2 = 4 WHERE col_1 IN (SELECT col_1 FROM cte); -DEBUG: generating subplan XXX_1 for CTE cte: WITH basic AS MATERIALIZED (SELECT source_table_1.col_1, source_table_1.col_2 FROM on_conflict.source_table_1) INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_1, col_2 FROM basic ON CONFLICT DO NOTHING RETURNING target_table.col_1, target_table.col_2 -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: generating subplan XXX_1 for CTE basic: SELECT col_1, col_2 FROM on_conflict.source_table_1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT basic.col_1, basic.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) basic) citus_insert_select_subquery -DEBUG: Plan XXX query after replacing subqueries and CTEs: UPDATE on_conflict.target_table SET col_2 = 4 WHERE (col_1 OPERATOR(pg_catalog.=) ANY (SELECT cte.col_1 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte)) -DEBUG: Collecting INSERT ... SELECT results on coordinator -RESET client_min_messages; --- Following query is supported by using repartition join for the insert/select -SELECT coordinator_plan($Q$ -EXPLAIN (costs off) -WITH cte AS ( - SELECT - col_1, col_2 - FROM - source_table_1 -) -INSERT INTO target_table -SELECT - source_table_1.col_1, - source_table_1.col_2 -FROM cte, source_table_1 -WHERE cte.col_1 = source_table_1.col_1 ON CONFLICT DO NOTHING; -$Q$); - coordinator_plan ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 -(4 rows) - --- Tests with foreign key to reference table -CREATE TABLE test_ref_table (key int PRIMARY KEY); -SELECT create_reference_table('test_ref_table'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO test_ref_table VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); -ALTER TABLE target_table ADD CONSTRAINT fkey FOREIGN KEY (col_1) REFERENCES test_ref_table(key) ON DELETE CASCADE; -BEGIN; - TRUNCATE test_ref_table CASCADE; -NOTICE: truncate cascades to table "target_table" - INSERT INTO - target_table - SELECT - col_2, - col_1 - FROM source_table_1 ON CONFLICT (col_1) DO UPDATE SET col_2 = 55 RETURNING *; -ERROR: insert or update on table "target_table_xxxxxxx" violates foreign key constraint "fkey_xxxxxxx" -DETAIL: Key (col_1)=(X) is not present in table "test_ref_table_xxxxxxx". -CONTEXT: while executing command on localhost:xxxxx -ROLLBACK; -BEGIN; - DELETE FROM test_ref_table WHERE key > 10; - WITH r AS ( - INSERT INTO - target_table - SELECT - col_2, - col_1 - FROM source_table_1 ON CONFLICT (col_1) DO UPDATE SET col_2 = 1 RETURNING *) - SELECT * FROM r ORDER BY col_1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 1 - 2 | 1 - 3 | 1 - 4 | 1 - 5 | 1 -(5 rows) - -ROLLBACK; --- Following two queries are supported since we no not modify but only select from --- the target_table after modification on test_ref_table. -BEGIN; - TRUNCATE test_ref_table CASCADE; -NOTICE: truncate cascades to table "target_table" - INSERT INTO - source_table_1 - SELECT - col_2, - col_1 - FROM target_table ON CONFLICT (col_1) DO UPDATE SET col_2 = 55 RETURNING *; - col_1 | col_2 | col_3 ---------------------------------------------------------------------- -(0 rows) - -ROLLBACK; -BEGIN; - DELETE FROM test_ref_table; - INSERT INTO - source_table_1 - SELECT - col_2, - col_1 - FROM target_table ON CONFLICT (col_1) DO UPDATE SET col_2 = 55 RETURNING *; - col_1 | col_2 | col_3 ---------------------------------------------------------------------- -(0 rows) - -ROLLBACK; --- INSERT .. SELECT with different column types -CREATE TABLE source_table_3(col_1 numeric, col_2 numeric, col_3 numeric); -SELECT create_distributed_table('source_table_3','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_3 VALUES(1,11,1),(2,22,2),(3,33,3),(4,44,4),(5,55,5); -CREATE TABLE source_table_4(id int, arr_val text[]); -SELECT create_distributed_table('source_table_4','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_4 VALUES(1, '{"abc","cde","efg"}'), (2, '{"xyz","tvu"}'); -CREATE TABLE target_table_2(id int primary key, arr_val char(10)[]); -SELECT create_distributed_table('target_table_2','id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO target_table_2 VALUES(1, '{"abc","def","gyx"}'); -SET client_min_messages to debug1; -INSERT INTO target_table -SELECT - col_1, col_2 -FROM - source_table_3 -ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2; -DEBUG: cannot perform distributed INSERT INTO ... SELECT because the partition columns in the source table and subquery do not match -DETAIL: The data type of the target table's partition column should exactly match the data type of the corresponding simple column reference in the subquery. -DEBUG: performing repartitioned INSERT ... SELECT -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 11 - 2 | 22 - 3 | 33 - 4 | 44 - 5 | 55 - 6 | 7 - 7 | 8 - 8 | 9 - 9 | 10 - 10 | 11 -(10 rows) - -INSERT INTO target_table_2 -SELECT - * -FROM - source_table_4 -ON CONFLICT DO NOTHING; -SELECT * FROM target_table_2 ORDER BY 1; - id | arr_val ---------------------------------------------------------------------- - 1 | {"abc ","def ","gyx "} - 2 | {"xyz ","tvu "} -(2 rows) - -RESET client_min_messages; --- Test with shard_replication_factor = 2 -SET citus.shard_replication_factor to 2; -DROP TABLE target_table, source_table_1, source_table_2; -CREATE TABLE target_table(col_1 int primary key, col_2 int); -SELECT create_distributed_table('target_table','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO target_table VALUES(1,2),(2,3),(3,4),(4,5),(5,6); -CREATE TABLE source_table_1(col_1 int, col_2 int, col_3 int); -SELECT create_distributed_table('source_table_1','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_1 VALUES(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5); -CREATE TABLE source_table_2(col_1 int, col_2 int, col_3 int); -SELECT create_distributed_table('source_table_2','col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table_2 VALUES(6,6,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10); -SET client_min_messages to debug1; --- Generate series directly on the coordinator and on conflict do nothing -INSERT INTO target_table (col_1, col_2) -SELECT - s, s -FROM - generate_series(1,10) s -ON CONFLICT DO NOTHING; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator --- Test with multiple subqueries -INSERT INTO target_table -SELECT - col_1, col_2 -FROM ( - (SELECT - col_1, col_2, col_3 - FROM - source_table_1 - LIMIT 5) - UNION - (SELECT - col_1, col_2, col_3 - FROM - source_table_2 - LIMIT 5) -) as foo -ON CONFLICT(col_1) DO UPDATE SET col_2 = 0; -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 LIMIT 5 -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_2 for subquery SELECT col_1, col_2, col_3 FROM on_conflict.source_table_2 LIMIT 5 -DEBUG: generating subplan XXX_3 for subquery SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer) UNION SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) foo -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 0 - 2 | 0 - 3 | 0 - 4 | 0 - 5 | 0 - 6 | 0 - 7 | 0 - 8 | 0 - 9 | 0 - 10 | 0 -(10 rows) - -WITH cte AS MATERIALIZED( - SELECT col_1, col_2, col_3 FROM source_table_1 -), cte_2 AS MATERIALIZED( - SELECT col_1, col_2 FROM cte -) -INSERT INTO target_table SELECT * FROM cte_2 ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: generating subplan XXX_1 for CTE cte: SELECT col_1, col_2, col_3 FROM on_conflict.source_table_1 -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2, intermediate_result.col_3 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer, col_3 integer)) cte -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT cte_2.col_1, cte_2.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte_2) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT * FROM target_table ORDER BY 1; - col_1 | col_2 ---------------------------------------------------------------------- - 1 | 2 - 2 | 3 - 3 | 4 - 4 | 5 - 5 | 6 - 6 | 0 - 7 | 0 - 8 | 0 - 9 | 0 - 10 | 0 -(10 rows) - --- make sure that even if COPY switchover happens --- the results are correct -SET citus.copy_switchover_threshold TO 1; -TRUNCATE target_table; --- load some data to make sure copy commands switch over connections -INSERT INTO target_table SELECT i,0 FROM generate_series(0,500)i; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: Collecting INSERT ... SELECT results on coordinator --- make sure that SELECT only uses 1 connection 1 node --- yet still COPY commands use 1 connection per co-located --- intermediate result file -SET citus.max_adaptive_executor_pool_size TO 1; -INSERT INTO target_table SELECT * FROM target_table LIMIT 10000 ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 10000 -DEBUG: Collecting INSERT ... SELECT results on coordinator -SELECT DISTINCT col_2 FROM target_table; - col_2 ---------------------------------------------------------------------- - 1 -(1 row) - -WITH cte_1 AS (INSERT INTO target_table SELECT * FROM target_table LIMIT 10000 ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1 RETURNING *) -SELECT DISTINCT col_2 FROM cte_1; -DEBUG: generating subplan XXX_1 for CTE cte_1: INSERT INTO on_conflict.target_table (col_1, col_2) SELECT col_1, col_2 FROM on_conflict.target_table LIMIT 10000 ON CONFLICT(col_1) DO UPDATE SET col_2 = (excluded.col_2 OPERATOR(pg_catalog.+) 1) RETURNING target_table.col_1, target_table.col_2 -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 10000 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT DISTINCT col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte_1 -DEBUG: Collecting INSERT ... SELECT results on coordinator - col_2 ---------------------------------------------------------------------- - 2 -(1 row) - -RESET client_min_messages; -DROP SCHEMA on_conflict CASCADE; -NOTICE: drop cascades to 8 other objects -DETAIL: drop cascades to table test_ref_table -drop cascades to table test_ref_table_1900012 -drop cascades to table source_table_3 -drop cascades to table source_table_4 -drop cascades to table target_table_2 -drop cascades to table target_table -drop cascades to table source_table_1 -drop cascades to table source_table_2 diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out index cb8f0c0e1..260c2366c 100644 --- a/src/test/regress/expected/multi_metadata_sync.out +++ b/src/test/regress/expected/multi_metadata_sync.out @@ -1,16 +1,6 @@ -- -- MULTI_METADATA_SYNC -- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - -- Tests for metadata snapshot functions, metadata syncing functions and propagation of -- metadata changes to MX tables. -- Turn metadata sync off at first diff --git a/src/test/regress/expected/multi_metadata_sync_0.out b/src/test/regress/expected/multi_metadata_sync_0.out deleted file mode 100644 index c81462e6f..000000000 --- a/src/test/regress/expected/multi_metadata_sync_0.out +++ /dev/null @@ -1,2262 +0,0 @@ --- --- MULTI_METADATA_SYNC --- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - --- Tests for metadata snapshot functions, metadata syncing functions and propagation of --- metadata changes to MX tables. --- Turn metadata sync off at first -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -NOTICE: dropping metadata on the node (localhost,57637) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); -NOTICE: dropping metadata on the node (localhost,57638) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1310000; -ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 2; -SELECT nextval('pg_catalog.pg_dist_placement_placementid_seq') AS last_placement_id -\gset -ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100000; -SELECT nextval('pg_catalog.pg_dist_groupid_seq') AS last_group_id \gset -SELECT nextval('pg_catalog.pg_dist_node_nodeid_seq') AS last_node_id \gset --- Create the necessary test utility function -SET citus.enable_metadata_sync TO OFF; -CREATE FUNCTION activate_node_snapshot() - RETURNS text[] - LANGUAGE C STRICT - AS 'citus'; -RESET citus.enable_metadata_sync; -COMMENT ON FUNCTION activate_node_snapshot() - IS 'commands to activate node snapshot'; --- Show that none of the existing tables are qualified to be MX tables -SELECT * FROM pg_dist_partition WHERE partmethod='h' AND repmodel='s'; - logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted ---------------------------------------------------------------------- -(0 rows) - --- Since password_encryption default has been changed to sha from md5 with PG14 --- we are updating it manually just for consistent test results between PG versions. -ALTER SYSTEM SET password_encryption TO md5; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(0.1); - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -ALTER ROLE CURRENT_USER WITH PASSWORD 'dummypassword'; --- Show that, with no MX tables, activate node snapshot contains only the delete commands, --- pg_dist_node entries, pg_dist_object entries and roles. -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; -(33 rows) - --- Create a test table with constraints and SERIAL and default from user defined sequence -CREATE SEQUENCE user_defined_seq; -CREATE TABLE mx_test_table (col_1 int UNIQUE, col_2 text NOT NULL, col_3 BIGSERIAL, col_4 BIGINT DEFAULT nextval('user_defined_seq')); -set citus.shard_count to 8; -set citus.shard_replication_factor to 1; -SELECT create_distributed_table('mx_test_table', 'col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -reset citus.shard_count; --- Set the replication model of the test table to streaming replication so that it is --- considered as an MX table -UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='mx_test_table'::regclass; --- add a single shard table and verify the creation commands are included in the activate node snapshot -CREATE TABLE single_shard_tbl(a int); -SELECT create_distributed_table('single_shard_tbl', null); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO single_shard_tbl VALUES (1); -reset citus.shard_replication_factor; --- Show that the created MX table is and its sequences are included in the activate node snapshot -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE public.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE public.mx_test_table OWNER TO postgres - ALTER TABLE public.single_shard_tbl OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - CREATE TABLE public.single_shard_tbl (a integer) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS public.mx_test_table CASCADE - DROP TABLE IF EXISTS public.single_shard_tbl CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('public.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT citus_internal_add_partition_metadata ('public.single_shard_tbl'::regclass, 'n', NULL, 3, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('public.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency('public.single_shard_tbl'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('public.mx_test_table_col_3_seq'::regclass,'public.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('public.mx_test_table') - SELECT worker_create_truncate_trigger('public.single_shard_tbl') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (2, 8, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (3, 1, 1, 0, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'single_shard_tbl']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 2, 100001), (1310002, 0, 1, 100002), (1310003, 0, 2, 100003), (1310004, 0, 1, 100004), (1310005, 0, 2, 100005), (1310006, 0, 1, 100006), (1310007, 0, 2, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310008, 0, 2, 100008)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('public.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('public.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('public.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('public.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('public.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('public.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('public.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.single_shard_tbl'::regclass, 1310008, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(61 rows) - --- Drop single shard table -DROP TABLE single_shard_tbl; --- Show that CREATE INDEX commands are included in the activate node snapshot -CREATE INDEX mx_index ON mx_test_table(col_2); -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE public.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE public.mx_test_table OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS public.mx_test_table CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('public.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('public.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('public.mx_test_table_col_3_seq'::regclass,'public.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('public.mx_test_table') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (2, 8, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 2, 100001), (1310002, 0, 1, 100002), (1310003, 0, 2, 100003), (1310004, 0, 1, 100004), (1310005, 0, 2, 100005), (1310006, 0, 1, 100006), (1310007, 0, 2, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('public.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('public.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('public.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('public.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('public.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('public.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('public.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(52 rows) - --- Show that schema changes are included in the activate node snapshot -CREATE SCHEMA mx_testing_schema; -ALTER TABLE mx_test_table SET SCHEMA mx_testing_schema; -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) - CREATE SCHEMA IF NOT EXISTS mx_testing_schema AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS mx_testing_schema.mx_test_table CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('mx_testing_schema.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('mx_testing_schema.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('mx_testing_schema.mx_test_table_col_3_seq'::regclass,'mx_testing_schema.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (2, 8, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_testing_schema']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['mx_testing_schema', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_testing_schema', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 2, 100001), (1310002, 0, 1, 100002), (1310003, 0, 2, 100003), (1310004, 0, 1, 100004), (1310005, 0, 2, 100005), (1310006, 0, 1, 100006), (1310007, 0, 2, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('mx_testing_schema.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('mx_testing_schema.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('mx_testing_schema.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('mx_testing_schema.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('mx_testing_schema.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('mx_testing_schema.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('mx_testing_schema.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(54 rows) - --- Show that append distributed tables are not included in the activate node snapshot -CREATE TABLE non_mx_test_table (col_1 int, col_2 text); -SELECT create_distributed_table('non_mx_test_table', 'col_1', 'append'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) - CREATE SCHEMA IF NOT EXISTS mx_testing_schema AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS mx_testing_schema.mx_test_table CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('mx_testing_schema.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('mx_testing_schema.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('mx_testing_schema.mx_test_table_col_3_seq'::regclass,'mx_testing_schema.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (2, 8, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_testing_schema']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['mx_testing_schema', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_testing_schema', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 2, 100001), (1310002, 0, 1, 100002), (1310003, 0, 2, 100003), (1310004, 0, 1, 100004), (1310005, 0, 2, 100005), (1310006, 0, 1, 100006), (1310007, 0, 2, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('mx_testing_schema.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('mx_testing_schema.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('mx_testing_schema.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('mx_testing_schema.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('mx_testing_schema.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('mx_testing_schema.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('mx_testing_schema.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(54 rows) - --- Show that range distributed tables are not included in the activate node snapshot -UPDATE pg_dist_partition SET partmethod='r' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) - CREATE SCHEMA IF NOT EXISTS mx_testing_schema AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS mx_testing_schema.mx_test_table CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE),(3, 2, 'localhost', 57638, 'default', FALSE, FALSE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('mx_testing_schema.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('mx_testing_schema.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('mx_testing_schema.mx_test_table_col_3_seq'::regclass,'mx_testing_schema.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (2, 8, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_testing_schema']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['mx_testing_schema', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_testing_schema', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 2, 100001), (1310002, 0, 1, 100002), (1310003, 0, 2, 100003), (1310004, 0, 1, 100004), (1310005, 0, 2, 100005), (1310006, 0, 1, 100006), (1310007, 0, 2, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('mx_testing_schema.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('mx_testing_schema.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('mx_testing_schema.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('mx_testing_schema.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('mx_testing_schema.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('mx_testing_schema.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('mx_testing_schema.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(54 rows) - --- Test start_metadata_sync_to_node and citus_activate_node UDFs --- Ensure that hasmetadata=false for all nodes except for the coordinator node -SELECT count(*) FROM pg_dist_node WHERE hasmetadata=true; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- Show that metadata can not be synced on secondary node -SELECT groupid AS worker_1_group FROM pg_dist_node WHERE nodeport = :worker_1_port \gset -SELECT master_add_node('localhost', 8888, groupid => :worker_1_group, noderole => 'secondary'); - master_add_node ---------------------------------------------------------------------- - 5 -(1 row) - -SELECT start_metadata_sync_to_node('localhost', 8888); - start_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT hasmetadata FROM pg_dist_node WHERE nodeport = 8888; - hasmetadata ---------------------------------------------------------------------- - f -(1 row) - -SELECT stop_metadata_sync_to_node('localhost', 8888); -NOTICE: (localhost,8888) is a secondary node: to clear the metadata, you should clear metadata from the primary node - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT hasmetadata FROM pg_dist_node WHERE nodeport = 8888; - hasmetadata ---------------------------------------------------------------------- - f -(1 row) - --- Add a node to another cluster to make sure it's also synced -SELECT master_add_secondary_node('localhost', 8889, 'localhost', :worker_1_port, nodecluster => 'second-cluster'); - master_add_secondary_node ---------------------------------------------------------------------- - 6 -(1 row) - -\c - - - :master_port --- Run start_metadata_sync_to_node and citus_activate_node and check that it marked hasmetadata for that worker -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT nodeid, hasmetadata FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_1_port; - nodeid | hasmetadata ---------------------------------------------------------------------- - 2 | t -(1 row) - --- Check that the metadata has been copied to the worker -\c - - - :worker_1_port -SELECT * FROM pg_dist_local_group; - groupid ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT * FROM pg_dist_node ORDER BY nodeid; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards ---------------------------------------------------------------------- - 1 | 0 | localhost | 57636 | default | t | t | primary | default | t | f - 2 | 1 | localhost | 57637 | default | t | t | primary | default | t | t - 3 | 2 | localhost | 57638 | default | f | t | primary | default | f | t - 5 | 1 | localhost | 8888 | default | f | t | secondary | default | f | t - 6 | 1 | localhost | 8889 | default | f | t | secondary | second-cluster | f | t -(5 rows) - -SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; - logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted ---------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f -(1 row) - -SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue ---------------------------------------------------------------------- - mx_testing_schema.mx_test_table | 1310000 | t | -2147483648 | -1610612737 - mx_testing_schema.mx_test_table | 1310001 | t | -1610612736 | -1073741825 - mx_testing_schema.mx_test_table | 1310002 | t | -1073741824 | -536870913 - mx_testing_schema.mx_test_table | 1310003 | t | -536870912 | -1 - mx_testing_schema.mx_test_table | 1310004 | t | 0 | 536870911 - mx_testing_schema.mx_test_table | 1310005 | t | 536870912 | 1073741823 - mx_testing_schema.mx_test_table | 1310006 | t | 1073741824 | 1610612735 - mx_testing_schema.mx_test_table | 1310007 | t | 1610612736 | 2147483647 -(8 rows) - -SELECT * FROM pg_dist_shard_placement WHERE shardid IN (SELECT shardid FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%') ORDER BY shardid, nodename, nodeport; - shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------------------------------------------------------------- - 1310000 | 1 | 0 | localhost | 57637 | 100000 - 1310001 | 1 | 0 | localhost | 57638 | 100001 - 1310002 | 1 | 0 | localhost | 57637 | 100002 - 1310003 | 1 | 0 | localhost | 57638 | 100003 - 1310004 | 1 | 0 | localhost | 57637 | 100004 - 1310005 | 1 | 0 | localhost | 57638 | 100005 - 1310006 | 1 | 0 | localhost | 57637 | 100006 - 1310007 | 1 | 0 | localhost | 57638 | 100007 -(8 rows) - -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col_1 | integer | - col_2 | text | not null - col_3 | bigint | not null default nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) - col_4 | bigint | default nextval('user_defined_seq'::regclass) -(4 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_testing_schema.mx_test_table_col_1_key'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_1 | integer | col_1 -(1 row) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_testing_schema.mx_index'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_2 | text | col_2 -(1 row) - --- Check that pg_dist_colocation is synced -SELECT * FROM pg_dist_colocation ORDER BY colocationid; - colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation ---------------------------------------------------------------------- - 2 | 8 | 1 | 23 | 0 -(1 row) - --- Make sure that truncate trigger has been set for the MX table on worker -SELECT count(*) FROM pg_trigger WHERE tgrelid='mx_testing_schema.mx_test_table'::regclass; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- Make sure that citus_activate_node considers foreign key constraints -\c - - - :master_port --- Since we're superuser, we can set the replication model to 'streaming' to --- create some MX tables -SET citus.shard_replication_factor TO 1; -CREATE SCHEMA mx_testing_schema_2; -CREATE TABLE mx_testing_schema.fk_test_1 (col1 int, col2 text, col3 int, UNIQUE(col1, col3)); -CREATE TABLE mx_testing_schema_2.fk_test_2 (col1 int, col2 int, col3 text, - FOREIGN KEY (col1, col2) REFERENCES mx_testing_schema.fk_test_1 (col1, col3)); -SELECT create_distributed_table('mx_testing_schema.fk_test_1', 'col1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('mx_testing_schema_2.fk_test_2', 'col1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- Check that foreign key metadata exists on the worker -\c - - - :worker_1_port -SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_testing_schema_2.fk_test_2'::regclass; - Constraint | Definition ---------------------------------------------------------------------- - fk_test_2_col1_col2_fkey | FOREIGN KEY (col1, col2) REFERENCES mx_testing_schema.fk_test_1(col1, col3) -(1 row) - -\c - - - :master_port -DROP TABLE mx_testing_schema_2.fk_test_2; -DROP TABLE mx_testing_schema.fk_test_1; -RESET citus.shard_replication_factor; --- Check that repeated calls to citus_activate_node has no side effects -\c - - - :master_port -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -\c - - - :worker_1_port -SELECT * FROM pg_dist_local_group; - groupid ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT * FROM pg_dist_node ORDER BY nodeid; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards ---------------------------------------------------------------------- - 1 | 0 | localhost | 57636 | default | t | t | primary | default | t | f - 2 | 1 | localhost | 57637 | default | t | t | primary | default | t | t - 3 | 2 | localhost | 57638 | default | f | t | primary | default | f | t - 5 | 1 | localhost | 8888 | default | f | t | secondary | default | f | t - 6 | 1 | localhost | 8889 | default | f | t | secondary | second-cluster | f | t -(5 rows) - -SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; - logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted ---------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f -(1 row) - -SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue ---------------------------------------------------------------------- - mx_testing_schema.mx_test_table | 1310000 | t | -2147483648 | -1610612737 - mx_testing_schema.mx_test_table | 1310001 | t | -1610612736 | -1073741825 - mx_testing_schema.mx_test_table | 1310002 | t | -1073741824 | -536870913 - mx_testing_schema.mx_test_table | 1310003 | t | -536870912 | -1 - mx_testing_schema.mx_test_table | 1310004 | t | 0 | 536870911 - mx_testing_schema.mx_test_table | 1310005 | t | 536870912 | 1073741823 - mx_testing_schema.mx_test_table | 1310006 | t | 1073741824 | 1610612735 - mx_testing_schema.mx_test_table | 1310007 | t | 1610612736 | 2147483647 -(8 rows) - -SELECT * FROM pg_dist_shard_placement WHERE shardid IN (SELECT shardid FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%') ORDER BY shardid, nodename, nodeport; - shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------------------------------------------------------------- - 1310000 | 1 | 0 | localhost | 57637 | 100000 - 1310001 | 1 | 0 | localhost | 57638 | 100001 - 1310002 | 1 | 0 | localhost | 57637 | 100002 - 1310003 | 1 | 0 | localhost | 57638 | 100003 - 1310004 | 1 | 0 | localhost | 57637 | 100004 - 1310005 | 1 | 0 | localhost | 57638 | 100005 - 1310006 | 1 | 0 | localhost | 57637 | 100006 - 1310007 | 1 | 0 | localhost | 57638 | 100007 -(8 rows) - -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col_1 | integer | - col_2 | text | not null - col_3 | bigint | not null default nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) - col_4 | bigint | default nextval('user_defined_seq'::regclass) -(4 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_testing_schema.mx_test_table_col_1_key'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_1 | integer | col_1 -(1 row) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_testing_schema.mx_index'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_2 | text | col_2 -(1 row) - -SELECT count(*) FROM pg_trigger WHERE tgrelid='mx_testing_schema.mx_test_table'::regclass; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- Make sure that citus_activate_node can be called inside a transaction and rollbacked -\c - - - :master_port -BEGIN; -SELECT 1 FROM citus_activate_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -ROLLBACK; -SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_port; - hasmetadata ---------------------------------------------------------------------- - f -(1 row) - --- Check that the distributed table can be queried from the worker -\c - - - :master_port -SET citus.shard_replication_factor TO 1; -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -CREATE TABLE mx_query_test (a int, b text, c int); -SELECT create_distributed_table('mx_query_test', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE single_shard_tbl(a int); -SELECT create_distributed_table('single_shard_tbl', null); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO single_shard_tbl VALUES (1); -SELECT repmodel FROM pg_dist_partition WHERE logicalrelid='mx_query_test'::regclass; - repmodel ---------------------------------------------------------------------- - s -(1 row) - -INSERT INTO mx_query_test VALUES (1, 'one', 1); -INSERT INTO mx_query_test VALUES (2, 'two', 4); -INSERT INTO mx_query_test VALUES (3, 'three', 9); -INSERT INTO mx_query_test VALUES (4, 'four', 16); -INSERT INTO mx_query_test VALUES (5, 'five', 24); -\c - - - :worker_1_port -SELECT * FROM mx_query_test ORDER BY a; - a | b | c ---------------------------------------------------------------------- - 1 | one | 1 - 2 | two | 4 - 3 | three | 9 - 4 | four | 16 - 5 | five | 24 -(5 rows) - -INSERT INTO mx_query_test VALUES (6, 'six', 36); -UPDATE mx_query_test SET c = 25 WHERE a = 5; -SELECT * FROM single_shard_tbl ORDER BY a; - a ---------------------------------------------------------------------- - 1 -(1 row) - -INSERT INTO single_shard_tbl VALUES (2); -\c - - - :master_port -SELECT * FROM mx_query_test ORDER BY a; - a | b | c ---------------------------------------------------------------------- - 1 | one | 1 - 2 | two | 4 - 3 | three | 9 - 4 | four | 16 - 5 | five | 25 - 6 | six | 36 -(6 rows) - -SELECT * FROM single_shard_tbl ORDER BY a; - a ---------------------------------------------------------------------- - 1 - 2 -(2 rows) - -\c - - - :master_port -DROP TABLE mx_query_test; -DROP TABLE single_shard_tbl; --- Check that stop_metadata_sync_to_node function sets hasmetadata of the node to false -\c - - - :master_port -SELECT start_metadata_sync_to_node('localhost', :worker_1_port); - start_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_1_port; - hasmetadata ---------------------------------------------------------------------- - t -(1 row) - -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -NOTICE: dropping metadata on the node (localhost,57637) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_1_port; - hasmetadata ---------------------------------------------------------------------- - f -(1 row) - --- Test DDL propagation in MX tables -SELECT start_metadata_sync_to_node('localhost', :worker_1_port); - start_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SET citus.shard_count = 5; -CREATE SCHEMA mx_test_schema_1; -CREATE SCHEMA mx_test_schema_2; --- Create MX tables -SET citus.shard_replication_factor TO 1; -CREATE TABLE mx_test_schema_1.mx_table_1 (col1 int UNIQUE, col2 text); -CREATE INDEX mx_index_1 ON mx_test_schema_1.mx_table_1 (col1); -CREATE TABLE mx_test_schema_2.mx_table_2 (col1 int, col2 text); -CREATE INDEX mx_index_2 ON mx_test_schema_2.mx_table_2 (col2); -ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY(col1) REFERENCES mx_test_schema_1.mx_table_1(col1); -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col1 | integer | - col2 | text | -(2 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_1.mx_table_1_col1_key'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col1 | integer | col1 -(1 row) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_1.mx_index_1'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col1 | integer | col1 -(1 row) - -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_2.mx_table_2'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col1 | integer | - col2 | text | -(2 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_2.mx_index_2'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col2 | text | col2 -(1 row) - -SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_2.mx_table_2'::regclass; - Constraint | Definition ---------------------------------------------------------------------- - mx_fk_constraint | FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) -(1 row) - -SELECT create_distributed_table('mx_test_schema_1.mx_table_1', 'col1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('mx_test_schema_2.mx_table_2', 'col1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- Check that created tables are marked as streaming replicated tables -SELECT - logicalrelid, repmodel -FROM - pg_dist_partition -WHERE - logicalrelid = 'mx_test_schema_1.mx_table_1'::regclass - OR logicalrelid = 'mx_test_schema_2.mx_table_2'::regclass -ORDER BY - logicalrelid::text; - logicalrelid | repmodel ---------------------------------------------------------------------- - mx_test_schema_1.mx_table_1 | s - mx_test_schema_2.mx_table_2 | s -(2 rows) - --- See the shards and placements of the mx tables -SELECT - logicalrelid, shardid, nodename, nodeport -FROM - pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE - logicalrelid = 'mx_test_schema_1.mx_table_1'::regclass - OR logicalrelid = 'mx_test_schema_2.mx_table_2'::regclass -ORDER BY - logicalrelid::text, shardid; - logicalrelid | shardid | nodename | nodeport ---------------------------------------------------------------------- - mx_test_schema_1.mx_table_1 | 1310022 | localhost | 57637 - mx_test_schema_1.mx_table_1 | 1310023 | localhost | 57638 - mx_test_schema_1.mx_table_1 | 1310024 | localhost | 57637 - mx_test_schema_1.mx_table_1 | 1310025 | localhost | 57638 - mx_test_schema_1.mx_table_1 | 1310026 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310027 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310028 | localhost | 57638 - mx_test_schema_2.mx_table_2 | 1310029 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310030 | localhost | 57638 - mx_test_schema_2.mx_table_2 | 1310031 | localhost | 57637 -(10 rows) - --- Check that metadata of MX tables exist on the metadata worker -\c - - - :worker_1_port --- Check that tables are created -\dt mx_test_schema_?.mx_table_? - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - mx_test_schema_1 | mx_table_1 | table | postgres - mx_test_schema_2 | mx_table_2 | table | postgres -(2 rows) - --- Check that table metadata are created -SELECT - logicalrelid, repmodel -FROM - pg_dist_partition -WHERE - logicalrelid = 'mx_test_schema_1.mx_table_1'::regclass - OR logicalrelid = 'mx_test_schema_2.mx_table_2'::regclass -ORDER BY - logicalrelid::text; - logicalrelid | repmodel ---------------------------------------------------------------------- - mx_test_schema_1.mx_table_1 | s - mx_test_schema_2.mx_table_2 | s -(2 rows) - --- Check that shard and placement data are created -SELECT - logicalrelid, shardid, nodename, nodeport -FROM - pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE - logicalrelid = 'mx_test_schema_1.mx_table_1'::regclass - OR logicalrelid = 'mx_test_schema_2.mx_table_2'::regclass -ORDER BY - logicalrelid::text, shardid; - logicalrelid | shardid | nodename | nodeport ---------------------------------------------------------------------- - mx_test_schema_1.mx_table_1 | 1310022 | localhost | 57637 - mx_test_schema_1.mx_table_1 | 1310023 | localhost | 57638 - mx_test_schema_1.mx_table_1 | 1310024 | localhost | 57637 - mx_test_schema_1.mx_table_1 | 1310025 | localhost | 57638 - mx_test_schema_1.mx_table_1 | 1310026 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310027 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310028 | localhost | 57638 - mx_test_schema_2.mx_table_2 | 1310029 | localhost | 57637 - mx_test_schema_2.mx_table_2 | 1310030 | localhost | 57638 - mx_test_schema_2.mx_table_2 | 1310031 | localhost | 57637 -(10 rows) - --- Check that metadata of MX tables don't exist on the non-metadata worker -\c - - - :worker_2_port -\d mx_test_schema_1.mx_table_1 -\d mx_test_schema_2.mx_table_2 -SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_test_schema%'; - logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_test_schema%'; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; - shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------------------------------------------------------------- -(0 rows) - --- Check that CREATE INDEX statement is propagated -\c - - - :master_port -SET client_min_messages TO 'ERROR'; -CREATE INDEX mx_index_3 ON mx_test_schema_2.mx_table_2 USING hash (col1); -ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_table_2_col1_key UNIQUE (col1); -\c - - - :worker_1_port -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_2.mx_index_3'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col1 | integer | col1 -(1 row) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_2.mx_table_2_col1_key'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col1 | integer | col1 -(1 row) - --- Check that DROP INDEX statement is propagated -\c - - - :master_port -DROP INDEX mx_test_schema_2.mx_index_3; -\c - - - :worker_1_port -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_test_schema_2.mx_index_3'::regclass; -ERROR: relation "mx_test_schema_2.mx_index_3" does not exist --- Check that ALTER TABLE statements are propagated -\c - - - :master_port -ALTER TABLE mx_test_schema_1.mx_table_1 ADD COLUMN col3 NUMERIC; -ALTER TABLE mx_test_schema_1.mx_table_1 ALTER COLUMN col3 SET DATA TYPE INT; -ALTER TABLE - mx_test_schema_1.mx_table_1 -ADD CONSTRAINT - mx_fk_constraint -FOREIGN KEY - (col1) -REFERENCES - mx_test_schema_2.mx_table_2(col1); -\c - - - :worker_1_port -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col1 | integer | - col2 | text | - col3 | integer | -(3 rows) - -SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; - Constraint | Definition ---------------------------------------------------------------------- - mx_fk_constraint | FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) -(1 row) - --- Check that foreign key constraint with NOT VALID works as well -\c - - - :master_port -ALTER TABLE mx_test_schema_1.mx_table_1 DROP CONSTRAINT mx_fk_constraint; -ALTER TABLE - mx_test_schema_1.mx_table_1 -ADD CONSTRAINT - mx_fk_constraint_2 -FOREIGN KEY - (col1) -REFERENCES - mx_test_schema_2.mx_table_2(col1) -NOT VALID; -\c - - - :worker_1_port -SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; - Constraint | Definition ---------------------------------------------------------------------- - mx_fk_constraint_2 | FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) -(1 row) - --- Check that update_distributed_table_colocation call propagates the changes to the workers -\c - - - :master_port -SELECT nextval('pg_catalog.pg_dist_colocationid_seq') AS last_colocation_id \gset -ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 10000; -SET citus.shard_count TO 7; -SET citus.shard_replication_factor TO 1; -CREATE TABLE mx_colocation_test_1 (a int); -SELECT create_distributed_table('mx_colocation_test_1', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE mx_colocation_test_2 (a int); -SELECT create_distributed_table('mx_colocation_test_2', 'a', colocate_with:='none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- Reset the colocation IDs of the test tables -DELETE FROM - pg_dist_colocation -WHERE EXISTS ( - SELECT 1 - FROM pg_dist_partition - WHERE - colocationid = pg_dist_partition.colocationid - AND pg_dist_partition.logicalrelid = 'mx_colocation_test_1'::regclass); --- Check the colocation IDs of the created tables -SELECT - logicalrelid, colocationid -FROM - pg_dist_partition -WHERE - logicalrelid = 'mx_colocation_test_1'::regclass - OR logicalrelid = 'mx_colocation_test_2'::regclass -ORDER BY logicalrelid::text; - logicalrelid | colocationid ---------------------------------------------------------------------- - mx_colocation_test_1 | 10000 - mx_colocation_test_2 | 10001 -(2 rows) - --- Update colocation and see the changes on the master and the worker -SELECT update_distributed_table_colocation('mx_colocation_test_1', colocate_with => 'mx_colocation_test_2'); - update_distributed_table_colocation ---------------------------------------------------------------------- - -(1 row) - -SELECT - logicalrelid, colocationid -FROM - pg_dist_partition -WHERE - logicalrelid = 'mx_colocation_test_1'::regclass - OR logicalrelid = 'mx_colocation_test_2'::regclass -ORDER BY - logicalrelid::text; - logicalrelid | colocationid ---------------------------------------------------------------------- - mx_colocation_test_1 | 10001 - mx_colocation_test_2 | 10001 -(2 rows) - -\c - - - :worker_1_port -SELECT - logicalrelid, colocationid -FROM - pg_dist_partition -WHERE - logicalrelid = 'mx_colocation_test_1'::regclass - OR logicalrelid = 'mx_colocation_test_2'::regclass -ORDER BY - logicalrelid::text; - logicalrelid | colocationid ---------------------------------------------------------------------- - mx_colocation_test_1 | 10001 - mx_colocation_test_2 | 10001 -(2 rows) - -\c - - - :master_port --- Check that DROP TABLE on MX tables works -DROP TABLE mx_colocation_test_1; -DROP TABLE mx_colocation_test_2; -\d mx_colocation_test_1 -\d mx_colocation_test_2 -\c - - - :worker_1_port -\d mx_colocation_test_1 -\d mx_colocation_test_2 --- Check that dropped MX table can be recreated again -\c - - - :master_port -SET citus.shard_count TO 7; -SET citus.shard_replication_factor TO 1; -CREATE TABLE mx_temp_drop_test (a int); -SELECT create_distributed_table('mx_temp_drop_test', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT logicalrelid, repmodel FROM pg_dist_partition WHERE logicalrelid = 'mx_temp_drop_test'::regclass; - logicalrelid | repmodel ---------------------------------------------------------------------- - mx_temp_drop_test | s -(1 row) - -DROP TABLE mx_temp_drop_test; -CREATE TABLE mx_temp_drop_test (a int); -SELECT create_distributed_table('mx_temp_drop_test', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT logicalrelid, repmodel FROM pg_dist_partition WHERE logicalrelid = 'mx_temp_drop_test'::regclass; - logicalrelid | repmodel ---------------------------------------------------------------------- - mx_temp_drop_test | s -(1 row) - -DROP TABLE mx_temp_drop_test; --- Check that MX tables can be created with SERIAL columns -\c - - - :master_port -SET citus.shard_count TO 3; -SET citus.shard_replication_factor TO 1; -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -NOTICE: dropping metadata on the node (localhost,57637) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); -NOTICE: dropping metadata on the node (localhost,57638) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - --- sync table with serial column after create_distributed_table -CREATE TABLE mx_table_with_small_sequence(a int, b SERIAL, c SMALLSERIAL); -SELECT create_distributed_table('mx_table_with_small_sequence', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -DROP TABLE mx_table_with_small_sequence; --- Show that create_distributed_table works with a serial column -CREATE TABLE mx_table_with_small_sequence(a int, b SERIAL, c SMALLSERIAL); -SELECT create_distributed_table('mx_table_with_small_sequence', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO mx_table_with_small_sequence VALUES (0); -\c - - - :worker_1_port --- Insert doesn't work because the defaults are of type int and smallint -INSERT INTO mx_table_with_small_sequence VALUES (1), (3); -ERROR: nextval(sequence) calls in worker nodes are not supported for column defaults of type int or smallint -\c - - - :master_port -SET citus.shard_replication_factor TO 1; --- Create an MX table with (BIGSERIAL) sequences -CREATE TABLE mx_table_with_sequence(a int, b BIGSERIAL, c BIGSERIAL); -SELECT create_distributed_table('mx_table_with_sequence', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO mx_table_with_sequence VALUES (0); -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - a | integer | - b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) - c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) -(3 rows) - -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_b_seq | sequence | postgres -(1 row) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_c_seq | sequence | postgres -(1 row) - --- Check that the sequences created on the metadata worker as well -\c - - - :worker_1_port -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - a | integer | - b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) - c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) -(3 rows) - -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_b_seq | sequence | postgres -(1 row) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_c_seq | sequence | postgres -(1 row) - --- Insert works because the defaults are of type bigint -INSERT INTO mx_table_with_sequence VALUES (1), (3); --- check that pg_depend records exist on the worker -SELECT refobjsubid FROM pg_depend -WHERE objid = 'mx_table_with_sequence_b_seq'::regclass AND refobjid = 'mx_table_with_sequence'::regclass; - refobjsubid ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT refobjsubid FROM pg_depend -WHERE objid = 'mx_table_with_sequence_c_seq'::regclass AND refobjid = 'mx_table_with_sequence'::regclass; - refobjsubid ---------------------------------------------------------------------- - 3 -(1 row) - --- Check that the sequences on the worker have their own space -SELECT nextval('mx_table_with_sequence_b_seq'); - nextval ---------------------------------------------------------------------- - 281474976710659 -(1 row) - -SELECT nextval('mx_table_with_sequence_c_seq'); - nextval ---------------------------------------------------------------------- - 281474976710659 -(1 row) - --- Check that adding a new metadata node sets the sequence space correctly -\c - - - :master_port -SELECT 1 FROM citus_activate_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -\c - - - :worker_2_port -SELECT groupid FROM pg_dist_local_group; - groupid ---------------------------------------------------------------------- - 2 -(1 row) - -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - a | integer | - b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) - c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) -(3 rows) - -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_b_seq | sequence | postgres -(1 row) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_table_with_sequence_c_seq | sequence | postgres -(1 row) - -SELECT nextval('mx_table_with_sequence_b_seq'); - nextval ---------------------------------------------------------------------- - 562949953421313 -(1 row) - -SELECT nextval('mx_table_with_sequence_c_seq'); - nextval ---------------------------------------------------------------------- - 562949953421313 -(1 row) - --- Insert doesn't work because the defaults are of type int and smallint -INSERT INTO mx_table_with_small_sequence VALUES (2), (4); -ERROR: nextval(sequence) calls in worker nodes are not supported for column defaults of type int or smallint --- Insert works because the defaults are of type bigint -INSERT INTO mx_table_with_sequence VALUES (2), (4); --- Check that dropping the mx table with sequences works as expected -\c - - - :master_port --- check our small sequence values -SELECT a, b, c FROM mx_table_with_small_sequence ORDER BY a,b,c; - a | b | c ---------------------------------------------------------------------- - 0 | 1 | 1 -(1 row) - ---check our bigint sequence values -SELECT a, b, c FROM mx_table_with_sequence ORDER BY a,b,c; - a | b | c ---------------------------------------------------------------------- - 0 | 1 | 1 - 1 | 281474976710657 | 281474976710657 - 2 | 562949953421314 | 562949953421314 - 3 | 281474976710658 | 281474976710658 - 4 | 562949953421315 | 562949953421315 -(5 rows) - --- Check that dropping the mx table with sequences works as expected -DROP TABLE mx_table_with_small_sequence, mx_table_with_sequence; -\d mx_table_with_sequence -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - --- Check that the sequences are dropped from the workers -\c - - - :worker_1_port -\d mx_table_with_sequence -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - --- Check that the sequences are dropped from the workers -\c - - - :worker_2_port -\ds mx_table_with_sequence_b_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - -\ds mx_table_with_sequence_c_seq - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- -(0 rows) - --- Check that MX sequences play well with non-super users -\c - - - :master_port --- Remove a node so that shards and sequences won't be created on table creation. Therefore, --- we can test that citus_activate_node can actually create the sequence with proper --- owner -CREATE TABLE pg_dist_placement_temp AS SELECT * FROM pg_dist_placement; -CREATE TABLE pg_dist_partition_temp AS SELECT * FROM pg_dist_partition; -CREATE TABLE pg_dist_object_temp AS SELECT * FROM pg_catalog.pg_dist_object; -DELETE FROM pg_dist_placement; -DELETE FROM pg_dist_partition; -DELETE FROM pg_catalog.pg_dist_object; -SELECT groupid AS old_worker_2_group FROM pg_dist_node WHERE nodeport = :worker_2_port \gset -SELECT master_remove_node('localhost', :worker_2_port); - master_remove_node ---------------------------------------------------------------------- - -(1 row) - - -- the master user needs superuser permissions to change the replication model -CREATE USER mx_user WITH SUPERUSER; -\c - mx_user - :master_port --- Create an mx table as a different user -CREATE TABLE mx_table (a int, b BIGSERIAL); -SET citus.shard_replication_factor TO 1; -SELECT create_distributed_table('mx_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -\c - postgres - :master_port -SELECT master_add_node('localhost', :worker_2_port); - master_add_node ---------------------------------------------------------------------- - 7 -(1 row) - -\c - mx_user - :worker_1_port -SELECT nextval('mx_table_b_seq'); - nextval ---------------------------------------------------------------------- - 281474976710657 -(1 row) - -INSERT INTO mx_table (a) VALUES (37); -INSERT INTO mx_table (a) VALUES (38); -SELECT * FROM mx_table ORDER BY a; - a | b ---------------------------------------------------------------------- - 37 | 281474976710658 - 38 | 281474976710659 -(2 rows) - -\c - mx_user - :worker_2_port -SELECT nextval('mx_table_b_seq'); - nextval ---------------------------------------------------------------------- - 1125899906842625 -(1 row) - -INSERT INTO mx_table (a) VALUES (39); -INSERT INTO mx_table (a) VALUES (40); -SELECT * FROM mx_table ORDER BY a; - a | b ---------------------------------------------------------------------- - 37 | 281474976710658 - 38 | 281474976710659 - 39 | 1125899906842626 - 40 | 1125899906842627 -(4 rows) - -\c - mx_user - :master_port -DROP TABLE mx_table; --- put the metadata back into a consistent state -\c - postgres - :master_port -INSERT INTO pg_dist_placement SELECT * FROM pg_dist_placement_temp; -INSERT INTO pg_dist_partition SELECT * FROM pg_dist_partition_temp; -INSERT INTO pg_catalog.pg_dist_object SELECT * FROM pg_dist_object_temp ON CONFLICT ON CONSTRAINT pg_dist_object_pkey DO NOTHING; -DROP TABLE pg_dist_placement_temp; -DROP TABLE pg_dist_partition_temp; -DROP TABLE pg_dist_object_temp; -UPDATE pg_dist_placement - SET groupid = (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_2_port) - WHERE groupid = :old_worker_2_group; -\c - - - :worker_1_port -UPDATE pg_dist_placement - SET groupid = (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_2_port) - WHERE groupid = :old_worker_2_group; -\c - - - :worker_2_port -UPDATE pg_dist_placement - SET groupid = (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_2_port) - WHERE groupid = :old_worker_2_group; -\c - - - :master_port -SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); -NOTICE: dropping metadata on the node (localhost,57638) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -DROP USER mx_user; --- Check that create_reference_table creates the metadata on workers -\c - - - :master_port -CREATE TABLE mx_ref (col_1 int, col_2 text); -SELECT create_reference_table('mx_ref'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - --- make sure that adding/removing nodes doesn't cause --- multiple colocation entries for reference tables -SELECT count(*) FROM pg_dist_colocation WHERE distributioncolumntype = 0; - count ---------------------------------------------------------------------- - 1 -(1 row) - -\dt mx_ref - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_ref | table | postgres -(1 row) - -\c - - - :worker_1_port -\dt mx_ref - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_ref | table | postgres -(1 row) - -SELECT - logicalrelid, partmethod, repmodel, shardid, placementid, nodename, nodeport -FROM - pg_dist_partition - NATURAL JOIN pg_dist_shard - NATURAL JOIN pg_dist_shard_placement -WHERE - logicalrelid = 'mx_ref'::regclass -ORDER BY - nodeport; - logicalrelid | partmethod | repmodel | shardid | placementid | nodename | nodeport ---------------------------------------------------------------------- - mx_ref | n | t | 1310074 | 100074 | localhost | 57636 - mx_ref | n | t | 1310074 | 100075 | localhost | 57637 - mx_ref | n | t | 1310074 | 100076 | localhost | 57638 -(3 rows) - -SELECT shardid AS ref_table_shardid FROM pg_dist_shard WHERE logicalrelid='mx_ref'::regclass \gset --- make sure we have the pg_dist_colocation record on the worker -SELECT count(*) FROM pg_dist_colocation WHERE distributioncolumntype = 0; - count ---------------------------------------------------------------------- - 1 -(1 row) - --- Check that DDL commands are propagated to reference tables on workers -\c - - - :master_port -ALTER TABLE mx_ref ADD COLUMN col_3 NUMERIC DEFAULT 0; -CREATE INDEX mx_ref_index ON mx_ref(col_1); -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col_1 | integer | - col_2 | text | - col_3 | numeric | default 0 -(3 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_ref_index'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_1 | integer | col_1 -(1 row) - -\c - - - :worker_1_port -SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; - Column | Type | Modifiers ---------------------------------------------------------------------- - col_1 | integer | - col_2 | text | - col_3 | numeric | default 0 -(3 rows) - -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_ref_index'::regclass; - Column | Type | Definition ---------------------------------------------------------------------- - col_1 | integer | col_1 -(1 row) - --- Check that metada is cleaned successfully upon drop table -\c - - - :master_port -DROP TABLE mx_ref; -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_ref_index'::regclass; -ERROR: relation "mx_ref_index" does not exist -\c - - - :worker_1_port -SELECT "Column", "Type", "Definition" FROM index_attrs WHERE - relid = 'mx_ref_index'::regclass; -ERROR: relation "mx_ref_index" does not exist -SELECT * FROM pg_dist_shard WHERE shardid=:ref_table_shardid; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue ---------------------------------------------------------------------- -(0 rows) - -SELECT * FROM pg_dist_shard_placement WHERE shardid=:ref_table_shardid; - shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------------------------------------------------------------- -(0 rows) - --- Check that master_add_node propagates the metadata about new placements of a reference table -\c - - - :master_port -SELECT groupid AS old_worker_2_group - FROM pg_dist_node WHERE nodeport = :worker_2_port \gset -CREATE TABLE tmp_placement AS - SELECT * FROM pg_dist_placement WHERE groupid = :old_worker_2_group; -DELETE FROM pg_dist_placement - WHERE groupid = :old_worker_2_group; -SELECT master_remove_node('localhost', :worker_2_port); -WARNING: could not find any shard placements for shardId 1310001 -WARNING: could not find any shard placements for shardId 1310023 -WARNING: could not find any shard placements for shardId 1310028 - master_remove_node ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE mx_ref (col_1 int, col_2 text); -SELECT create_reference_table('mx_ref'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -SELECT shardid, nodename, nodeport -FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE logicalrelid='mx_ref'::regclass; - shardid | nodename | nodeport ---------------------------------------------------------------------- - 1310075 | localhost | 57636 - 1310075 | localhost | 57637 -(2 rows) - -\c - - - :worker_1_port -SELECT shardid, nodename, nodeport -FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE logicalrelid='mx_ref'::regclass; - shardid | nodename | nodeport ---------------------------------------------------------------------- - 1310075 | localhost | 57636 - 1310075 | localhost | 57637 -(2 rows) - -\c - - - :master_port -SET client_min_messages TO ERROR; -SELECT master_add_node('localhost', :worker_2_port); - master_add_node ---------------------------------------------------------------------- - 8 -(1 row) - -RESET client_min_messages; -SELECT shardid, nodename, nodeport -FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE logicalrelid='mx_ref'::regclass -ORDER BY shardid, nodeport; - shardid | nodename | nodeport ---------------------------------------------------------------------- - 1310075 | localhost | 57636 - 1310075 | localhost | 57637 -(2 rows) - -\c - - - :worker_1_port -SELECT shardid, nodename, nodeport -FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement -WHERE logicalrelid='mx_ref'::regclass -ORDER BY shardid, nodeport; - shardid | nodename | nodeport ---------------------------------------------------------------------- - 1310075 | localhost | 57636 - 1310075 | localhost | 57637 -(2 rows) - --- Get the metadata back into a consistent state -\c - - - :master_port -INSERT INTO pg_dist_placement (SELECT * FROM tmp_placement); -DROP TABLE tmp_placement; -UPDATE pg_dist_placement - SET groupid = (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_2_port) - WHERE groupid = :old_worker_2_group; -\c - - - :worker_1_port -UPDATE pg_dist_placement - SET groupid = (SELECT groupid FROM pg_dist_node WHERE nodeport = :worker_2_port) - WHERE groupid = :old_worker_2_group; --- Confirm that shouldhaveshards is 'true' -\c - - - :master_port -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - t -(1 row) - -\c - postgres - :worker_1_port -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - t -(1 row) - --- Check that setting shouldhaveshards to false is correctly transferred to other mx nodes -\c - - - :master_port -SELECT * from master_set_node_property('localhost', 8888, 'shouldhaveshards', false); - master_set_node_property ---------------------------------------------------------------------- - -(1 row) - -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - f -(1 row) - -\c - postgres - :worker_1_port -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - f -(1 row) - --- Check that setting shouldhaveshards to true is correctly transferred to other mx nodes -\c - postgres - :master_port -SELECT * from master_set_node_property('localhost', 8888, 'shouldhaveshards', true); - master_set_node_property ---------------------------------------------------------------------- - -(1 row) - -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - t -(1 row) - -\c - postgres - :worker_1_port -select shouldhaveshards from pg_dist_node where nodeport = 8888; - shouldhaveshards ---------------------------------------------------------------------- - t -(1 row) - -\c - - - :master_port --- --- Check that metadata commands error out if any nodes are out-of-sync --- --- increase metadata_sync intervals to avoid metadata sync while we test -ALTER SYSTEM SET citus.metadata_sync_interval TO 300000; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO 300000; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SET citus.shard_replication_factor TO 1; -CREATE TABLE dist_table_1(a int); -SELECT create_distributed_table('dist_table_1', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -UPDATE pg_dist_node SET metadatasynced=false WHERE nodeport=:worker_1_port; -SELECT hasmetadata, metadatasynced FROM pg_dist_node WHERE nodeport=:worker_1_port; - hasmetadata | metadatasynced ---------------------------------------------------------------------- - t | f -(1 row) - -CREATE TABLE dist_table_2(a int); -SELECT create_distributed_table('dist_table_2', 'a'); -ERROR: localhost:xxxxx is a metadata node, but is out of sync -HINT: If the node is up, wait until metadata gets synced to it and try again. -SELECT create_reference_table('dist_table_2'); -ERROR: localhost:xxxxx is a metadata node, but is out of sync -HINT: If the node is up, wait until metadata gets synced to it and try again. -ALTER TABLE dist_table_1 ADD COLUMN b int; -ERROR: localhost:xxxxx is a metadata node, but is out of sync -HINT: If the node is up, wait until metadata gets synced to it and try again. -SELECT citus_disable_node_and_wait('localhost', :worker_1_port); -ERROR: disabling the first worker node in the metadata is not allowed -DETAIL: Citus uses the first worker node in the metadata for certain internal operations when replicated tables are modified. Synchronous mode ensures that all nodes have the same view of the first worker node, which is used for certain locking operations. -HINT: You can force disabling node, SELECT citus_disable_node('localhost', 57637, synchronous:=true); -CONTEXT: SQL statement "SELECT pg_catalog.citus_disable_node(nodename, nodeport, force)" -PL/pgSQL function citus_disable_node_and_wait(text,integer,boolean) line XX at PERFORM -SELECT citus_disable_node_and_wait('localhost', :worker_2_port); -ERROR: cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx -DETAIL: One of the table(s) that prevents the operation complete successfully is mx_testing_schema.mx_test_table -HINT: To proceed, either drop the tables or use undistribute_table() function to convert them to local tables -CONTEXT: SQL statement "SELECT pg_catalog.citus_disable_node(nodename, nodeport, force)" -PL/pgSQL function citus_disable_node_and_wait(text,integer,boolean) line XX at PERFORM -SELECT master_remove_node('localhost', :worker_1_port); -ERROR: cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx -DETAIL: One of the table(s) that prevents the operation complete successfully is mx_testing_schema.mx_test_table -HINT: To proceed, either drop the tables or use undistribute_table() function to convert them to local tables -SELECT master_remove_node('localhost', :worker_2_port); -ERROR: cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx -DETAIL: One of the table(s) that prevents the operation complete successfully is mx_testing_schema.mx_test_table -HINT: To proceed, either drop the tables or use undistribute_table() function to convert them to local tables --- master_update_node should succeed -SELECT nodeid AS worker_2_nodeid FROM pg_dist_node WHERE nodeport=:worker_2_port \gset -SELECT master_update_node(:worker_2_nodeid, 'localhost', 4444); - master_update_node ---------------------------------------------------------------------- - -(1 row) - -SELECT master_update_node(:worker_2_nodeid, 'localhost', :worker_2_port); - master_update_node ---------------------------------------------------------------------- - -(1 row) - -ALTER SYSTEM SET citus.metadata_sync_interval TO DEFAULT; -ALTER SYSTEM SET citus.metadata_sync_retry_interval TO DEFAULT; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - --- make sure that all the nodes have valid metadata before moving forward -SELECT wait_until_metadata_sync(60000); - wait_until_metadata_sync ---------------------------------------------------------------------- - -(1 row) - -SELECT master_add_node('localhost', :worker_2_port); - master_add_node ---------------------------------------------------------------------- - 8 -(1 row) - -CREATE SEQUENCE mx_test_sequence_0; -CREATE SEQUENCE mx_test_sequence_1; --- test create_distributed_table -CREATE TABLE test_table (id int DEFAULT nextval('mx_test_sequence_0')); -SELECT create_distributed_table('test_table', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- shouldn't work since it's partition column -ALTER TABLE test_table ALTER COLUMN id SET DEFAULT nextval('mx_test_sequence_1'); -ERROR: cannot execute ALTER TABLE command involving partition column --- test different plausible commands -ALTER TABLE test_table ADD COLUMN id2 int DEFAULT nextval('mx_test_sequence_1'); -ALTER TABLE test_table ALTER COLUMN id2 DROP DEFAULT; -ALTER TABLE test_table ALTER COLUMN id2 SET DEFAULT nextval('mx_test_sequence_1'); -SELECT unnest(activate_node_snapshot()) order by 1; - unnest ---------------------------------------------------------------------- - ALTER DATABASE regression OWNER TO postgres; - ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - ALTER SEQUENCE public.mx_test_sequence_0 OWNER TO postgres - ALTER SEQUENCE public.mx_test_sequence_1 OWNER TO postgres - ALTER SEQUENCE public.user_defined_seq OWNER TO postgres - ALTER TABLE mx_test_schema_1.mx_table_1 ADD CONSTRAINT mx_fk_constraint_2 FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) NOT VALID - ALTER TABLE mx_test_schema_1.mx_table_1 ADD CONSTRAINT mx_table_1_col1_key UNIQUE (col1) - ALTER TABLE mx_test_schema_1.mx_table_1 OWNER TO postgres - ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) - ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_table_2_col1_key UNIQUE (col1) - ALTER TABLE mx_test_schema_2.mx_table_2 OWNER TO postgres - ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - ALTER TABLE public.dist_table_1 OWNER TO postgres - ALTER TABLE public.mx_ref OWNER TO postgres - ALTER TABLE public.test_table OWNER TO postgres - CALL pg_catalog.worker_drop_all_shell_tables(true) - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) - CREATE INDEX mx_index_1 ON mx_test_schema_1.mx_table_1 USING btree (col1) - CREATE INDEX mx_index_2 ON mx_test_schema_2.mx_table_2 USING btree (col2) - CREATE SCHEMA IF NOT EXISTS mx_test_schema_1 AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS mx_test_schema_2 AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS mx_testing_schema AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS mx_testing_schema_2 AUTHORIZATION postgres - CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres - CREATE TABLE mx_test_schema_1.mx_table_1 (col1 integer, col2 text, col3 integer) USING heap - CREATE TABLE mx_test_schema_2.mx_table_2 (col1 integer, col2 text) USING heap - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL, col_4 bigint DEFAULT nextval('public.user_defined_seq'::regclass)) USING heap - CREATE TABLE public.dist_table_1 (a integer) USING heap - CREATE TABLE public.mx_ref (col_1 integer, col_2 text) USING heap - CREATE TABLE public.test_table (id integer DEFAULT worker_nextval('public.mx_test_sequence_0'::regclass), id2 integer DEFAULT worker_nextval('public.mx_test_sequence_1'::regclass)) USING heap - DELETE FROM pg_catalog.pg_dist_colocation - DELETE FROM pg_catalog.pg_dist_object - DELETE FROM pg_catalog.pg_dist_schema - DELETE FROM pg_dist_node - DELETE FROM pg_dist_partition - DELETE FROM pg_dist_placement - DELETE FROM pg_dist_shard - DROP TABLE IF EXISTS mx_test_schema_1.mx_table_1 CASCADE - DROP TABLE IF EXISTS mx_test_schema_2.mx_table_2 CASCADE - DROP TABLE IF EXISTS mx_testing_schema.mx_test_table CASCADE - DROP TABLE IF EXISTS public.dist_table_1 CASCADE - DROP TABLE IF EXISTS public.mx_ref CASCADE - DROP TABLE IF EXISTS public.test_table CASCADE - GRANT CREATE ON SCHEMA public TO PUBLIC; - GRANT CREATE ON SCHEMA public TO postgres; - GRANT USAGE ON SCHEMA public TO PUBLIC; - GRANT USAGE ON SCHEMA public TO postgres; - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, metadatasynced, isactive, noderole, nodecluster, shouldhaveshards) VALUES (5, 1, 'localhost', 8888, 'default', FALSE, FALSE, TRUE, 'secondary'::noderole, 'default', TRUE),(6, 1, 'localhost', 8889, 'default', FALSE, FALSE, TRUE, 'secondary'::noderole, 'second-cluster', TRUE),(1, 0, 'localhost', 57636, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', FALSE),(2, 1, 'localhost', 57637, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', TRUE),(8, 5, 'localhost', 57638, 'default', TRUE, TRUE, TRUE, 'primary'::noderole, 'default', TRUE) - RESET ROLE - RESET ROLE - SELECT alter_role_if_exists('postgres', 'ALTER ROLE postgres SET lc_messages = ''C''') - SELECT citus_internal_add_partition_metadata ('mx_test_schema_1.mx_table_1'::regclass, 'h', 'col1', 7, 's') - SELECT citus_internal_add_partition_metadata ('mx_test_schema_2.mx_table_2'::regclass, 'h', 'col1', 7, 's') - SELECT citus_internal_add_partition_metadata ('mx_testing_schema.mx_test_table'::regclass, 'h', 'col_1', 2, 's') - SELECT citus_internal_add_partition_metadata ('public.dist_table_1'::regclass, 'h', 'a', 10010, 's') - SELECT citus_internal_add_partition_metadata ('public.mx_ref'::regclass, 'n', NULL, 10009, 't') - SELECT citus_internal_add_partition_metadata ('public.test_table'::regclass, 'h', 'id', 10010, 's') - SELECT pg_catalog.worker_drop_sequence_dependency('mx_test_schema_1.mx_table_1'); - SELECT pg_catalog.worker_drop_sequence_dependency('mx_test_schema_2.mx_table_2'); - SELECT pg_catalog.worker_drop_sequence_dependency('mx_testing_schema.mx_test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency('public.dist_table_1'); - SELECT pg_catalog.worker_drop_sequence_dependency('public.mx_ref'); - SELECT pg_catalog.worker_drop_sequence_dependency('public.test_table'); - SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition - SELECT pg_catalog.worker_record_sequence_dependency('mx_testing_schema.mx_test_table_col_3_seq'::regclass,'mx_testing_schema.mx_test_table'::regclass,'col_3') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.mx_test_sequence_0 AS integer INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1 NO CYCLE','integer') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.mx_test_sequence_1 AS integer INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1 NO CYCLE','integer') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS public.user_defined_seq AS bigint INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1 NO CYCLE','bigint') - SELECT worker_create_or_alter_role('postgres', 'CREATE ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''', 'ALTER ROLE postgres SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN REPLICATION BYPASSRLS CONNECTION LIMIT 0 PASSWORD ''md5c53670dddfc3bb4b5675c7872bc2249a'' VALID UNTIL ''2052-05-05 00:00:00-07''') - SELECT worker_create_truncate_trigger('mx_test_schema_1.mx_table_1') - SELECT worker_create_truncate_trigger('mx_test_schema_2.mx_table_2') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') - SELECT worker_create_truncate_trigger('public.dist_table_1') - SELECT worker_create_truncate_trigger('public.mx_ref') - SELECT worker_create_truncate_trigger('public.test_table') - SET ROLE postgres - SET ROLE postgres - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'off' - SET citus.enable_ddl_propagation TO 'on' - SET citus.enable_ddl_propagation TO 'on' - UPDATE pg_dist_local_group SET groupid = 1 - UPDATE pg_dist_node SET hasmetadata = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET isactive = TRUE WHERE nodeid = 2 - UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid = 2 - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (10009, 1, -1, 0, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH colocation_group_data (colocationid, shardcount, replicationfactor, distributioncolumntype, distributioncolumncollationname, distributioncolumncollationschema) AS (VALUES (10010, 4, 1, 'integer'::regtype, NULL, NULL)) SELECT pg_catalog.citus_internal_add_colocation_metadata(colocationid, shardcount, replicationfactor, distributioncolumntype, coalesce(c.oid, 0)) FROM colocation_group_data d LEFT JOIN pg_collation c ON (d.distributioncolumncollationname = c.collname AND d.distributioncolumncollationschema::regnamespace = c.collnamespace) - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('database', ARRAY['regression']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('role', ARRAY['postgres']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_test_schema_1']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_test_schema_2']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_testing_schema']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['mx_testing_schema_2']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['mx_testing_schema', 'mx_test_table_col_3_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'mx_test_sequence_0']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'mx_test_sequence_1']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('sequence', ARRAY['public', 'user_defined_seq']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_test_schema_1', 'mx_table_1']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_test_schema_2', 'mx_table_2']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['mx_testing_schema', 'mx_test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'dist_table_1']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'mx_ref']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('table', ARRAY['public', 'test_table']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal_add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310000, 0, 1, 100000), (1310001, 0, 5, 100001), (1310002, 0, 1, 100002), (1310003, 0, 5, 100003), (1310004, 0, 1, 100004), (1310005, 0, 5, 100005), (1310006, 0, 1, 100006), (1310007, 0, 5, 100007)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310022, 0, 1, 100022), (1310023, 0, 5, 100023), (1310024, 0, 1, 100024), (1310025, 0, 5, 100025), (1310026, 0, 1, 100026)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310027, 0, 1, 100027), (1310028, 0, 5, 100028), (1310029, 0, 1, 100029), (1310030, 0, 5, 100030), (1310031, 0, 1, 100031)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310075, 0, 0, 100077), (1310075, 0, 1, 100078), (1310075, 0, 5, 100079)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310076, 0, 1, 100080), (1310077, 0, 5, 100081), (1310078, 0, 1, 100082), (1310079, 0, 5, 100083)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH placement_data(shardid, shardlength, groupid, placementid) AS (VALUES (1310085, 0, 1, 100091), (1310086, 0, 5, 100092), (1310087, 0, 1, 100093), (1310088, 0, 5, 100094)) SELECT citus_internal_add_placement_metadata(shardid, shardlength, groupid, placementid) FROM placement_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_test_schema_1.mx_table_1'::regclass, 1310022, 't'::"char", '-2147483648', '-1288490190'), ('mx_test_schema_1.mx_table_1'::regclass, 1310023, 't'::"char", '-1288490189', '-429496731'), ('mx_test_schema_1.mx_table_1'::regclass, 1310024, 't'::"char", '-429496730', '429496728'), ('mx_test_schema_1.mx_table_1'::regclass, 1310025, 't'::"char", '429496729', '1288490187'), ('mx_test_schema_1.mx_table_1'::regclass, 1310026, 't'::"char", '1288490188', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_test_schema_2.mx_table_2'::regclass, 1310027, 't'::"char", '-2147483648', '-1288490190'), ('mx_test_schema_2.mx_table_2'::regclass, 1310028, 't'::"char", '-1288490189', '-429496731'), ('mx_test_schema_2.mx_table_2'::regclass, 1310029, 't'::"char", '-429496730', '429496728'), ('mx_test_schema_2.mx_table_2'::regclass, 1310030, 't'::"char", '429496729', '1288490187'), ('mx_test_schema_2.mx_table_2'::regclass, 1310031, 't'::"char", '1288490188', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't'::"char", '-2147483648', '-1610612737'), ('mx_testing_schema.mx_test_table'::regclass, 1310001, 't'::"char", '-1610612736', '-1073741825'), ('mx_testing_schema.mx_test_table'::regclass, 1310002, 't'::"char", '-1073741824', '-536870913'), ('mx_testing_schema.mx_test_table'::regclass, 1310003, 't'::"char", '-536870912', '-1'), ('mx_testing_schema.mx_test_table'::regclass, 1310004, 't'::"char", '0', '536870911'), ('mx_testing_schema.mx_test_table'::regclass, 1310005, 't'::"char", '536870912', '1073741823'), ('mx_testing_schema.mx_test_table'::regclass, 1310006, 't'::"char", '1073741824', '1610612735'), ('mx_testing_schema.mx_test_table'::regclass, 1310007, 't'::"char", '1610612736', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.dist_table_1'::regclass, 1310076, 't'::"char", '-2147483648', '-1073741825'), ('public.dist_table_1'::regclass, 1310077, 't'::"char", '-1073741824', '-1'), ('public.dist_table_1'::regclass, 1310078, 't'::"char", '0', '1073741823'), ('public.dist_table_1'::regclass, 1310079, 't'::"char", '1073741824', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.mx_ref'::regclass, 1310075, 't'::"char", NULL, NULL)) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; - WITH shard_data(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) AS (VALUES ('public.test_table'::regclass, 1310085, 't'::"char", '-2147483648', '-1073741825'), ('public.test_table'::regclass, 1310086, 't'::"char", '-1073741824', '-1'), ('public.test_table'::regclass, 1310087, 't'::"char", '0', '1073741823'), ('public.test_table'::regclass, 1310088, 't'::"char", '1073741824', '2147483647')) SELECT citus_internal_add_shard_metadata(relationname, shardid, storagetype, shardminvalue, shardmaxvalue) FROM shard_data; -(118 rows) - --- shouldn't work since test_table is MX -ALTER TABLE test_table ADD COLUMN id3 bigserial; -ERROR: cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers --- shouldn't work since the above operations should be the only subcommands -ALTER TABLE test_table ADD COLUMN id4 int DEFAULT nextval('mx_test_sequence_1') CHECK (id4 > 0); -ERROR: cannot execute ADD COLUMN .. DEFAULT nextval('..') command with other subcommands/constraints -HINT: You can issue each subcommand separately -ALTER TABLE test_table ADD COLUMN id4 int, ADD COLUMN id5 int DEFAULT nextval('mx_test_sequence_1'); -ERROR: cannot execute ADD COLUMN .. DEFAULT nextval('..') command with other subcommands/constraints -HINT: You can issue each subcommand separately -ALTER TABLE test_table ALTER COLUMN id1 SET DEFAULT nextval('mx_test_sequence_1'), ALTER COLUMN id2 DROP DEFAULT; -ERROR: cannot execute ALTER COLUMN COLUMN .. SET DEFAULT nextval('..') command with other subcommands -HINT: You can issue each subcommand separately -ALTER TABLE test_table ADD COLUMN id4 bigserial CHECK (id4 > 0); -ERROR: cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers -\c - - - :worker_1_port -\ds - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_test_sequence_0 | sequence | postgres - public | mx_test_sequence_1 | sequence | postgres - public | mx_test_table_col_3_seq | sequence | postgres - public | sequence_rollback | sequence | postgres - public | sequence_rollback(citus_backup_0) | sequence | postgres - public | user_defined_seq | sequence | postgres -(6 rows) - -\c - - - :master_port -CREATE SEQUENCE local_sequence; --- verify that DROP SEQUENCE will propagate the command to workers for --- the distributed sequences mx_test_sequence_0 and mx_test_sequence_1 -DROP SEQUENCE mx_test_sequence_0, mx_test_sequence_1, local_sequence CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to default value for column id2 of table test_table -drop cascades to default value for column id of table test_table -\c - - - :worker_1_port -\ds - List of relations - Schema | Name | Type | Owner ---------------------------------------------------------------------- - public | mx_test_table_col_3_seq | sequence | postgres - public | sequence_rollback | sequence | postgres - public | sequence_rollback(citus_backup_0) | sequence | postgres - public | user_defined_seq | sequence | postgres -(4 rows) - -\c - - - :master_port -DROP TABLE test_table CASCADE; --- Cleanup -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -NOTICE: dropping metadata on the node (localhost,57637) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); -NOTICE: dropping metadata on the node (localhost,57638) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -DROP TABLE mx_test_schema_2.mx_table_2 CASCADE; -NOTICE: drop cascades to constraint mx_fk_constraint_2 on table mx_test_schema_1.mx_table_1 -DROP TABLE mx_test_schema_1.mx_table_1 CASCADE; -DROP TABLE mx_testing_schema.mx_test_table; -DROP TABLE mx_ref; -DROP TABLE dist_table_1, dist_table_2; -SET client_min_messages TO ERROR; -SET citus.enable_ddl_propagation TO off; -- for enterprise -CREATE USER non_super_metadata_user; -SET citus.enable_ddl_propagation TO on; -RESET client_min_messages; -SELECT run_command_on_workers('CREATE USER non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,"CREATE ROLE") - (localhost,57638,t,"CREATE ROLE") -(2 rows) - -GRANT EXECUTE ON FUNCTION start_metadata_sync_to_node(text,int) TO non_super_metadata_user; -GRANT EXECUTE ON FUNCTION stop_metadata_sync_to_node(text,int,bool) TO non_super_metadata_user; -GRANT ALL ON pg_dist_node TO non_super_metadata_user; -GRANT ALL ON pg_dist_local_group TO non_super_metadata_user; -GRANT ALL ON SCHEMA citus TO non_super_metadata_user; -GRANT INSERT ON ALL TABLES IN SCHEMA citus TO non_super_metadata_user; -GRANT USAGE ON SCHEMA mx_testing_schema TO non_super_metadata_user; -GRANT USAGE ON SCHEMA mx_testing_schema_2 TO non_super_metadata_user; -GRANT USAGE ON SCHEMA mx_test_schema_1 TO non_super_metadata_user; -GRANT USAGE ON SCHEMA mx_test_schema_2 TO non_super_metadata_user; -SELECT run_command_on_workers('GRANT ALL ON pg_dist_node TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT ALL ON pg_dist_local_group TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT ALL ON SCHEMA citus TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('ALTER SEQUENCE user_defined_seq OWNER TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,"ALTER SEQUENCE") - (localhost,57638,t,"ALTER SEQUENCE") -(2 rows) - -SELECT run_command_on_workers('GRANT ALL ON ALL TABLES IN SCHEMA citus TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT USAGE ON SCHEMA mx_testing_schema TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT USAGE ON SCHEMA mx_testing_schema_2 TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT USAGE ON SCHEMA mx_test_schema_1 TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SELECT run_command_on_workers('GRANT USAGE ON SCHEMA mx_test_schema_2 TO non_super_metadata_user'); - run_command_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,GRANT) - (localhost,57638,t,GRANT) -(2 rows) - -SET ROLE non_super_metadata_user; --- user must be super user stop/start metadata -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -ERROR: operation is not allowed -HINT: Run the command with a superuser. -SELECT start_metadata_sync_to_node('localhost', :worker_1_port); -ERROR: operation is not allowed -HINT: Run the command with a superuser. -RESET ROLE; -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); -NOTICE: dropping metadata on the node (localhost,57637) - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -SELECT start_metadata_sync_to_node('localhost', :worker_1_port); - start_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - -RESET citus.shard_count; -RESET citus.shard_replication_factor; -ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART :last_group_id; -ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART :last_node_id; -ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART :last_colocation_id; -ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART :last_placement_id; --- Activate them at the end -SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT 1 FROM citus_activate_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - diff --git a/src/test/regress/expected/multi_mx_insert_select_repartition.out b/src/test/regress/expected/multi_mx_insert_select_repartition.out index 62f197c30..6be05bacd 100644 --- a/src/test/regress/expected/multi_mx_insert_select_repartition.out +++ b/src/test/regress/expected/multi_mx_insert_select_repartition.out @@ -3,17 +3,6 @@ -- -- Test behaviour of repartitioned INSERT ... SELECT in MX setup -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA multi_mx_insert_select_repartition; SET search_path TO multi_mx_insert_select_repartition; SET citus.next_shard_id TO 4213581; diff --git a/src/test/regress/expected/multi_mx_insert_select_repartition_0.out b/src/test/regress/expected/multi_mx_insert_select_repartition_0.out deleted file mode 100644 index 15deba0c0..000000000 --- a/src/test/regress/expected/multi_mx_insert_select_repartition_0.out +++ /dev/null @@ -1,166 +0,0 @@ --- --- MULTI_MX_INSERT_SELECT_REPARTITION --- --- Test behaviour of repartitioned INSERT ... SELECT in MX setup --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA multi_mx_insert_select_repartition; -SET search_path TO multi_mx_insert_select_repartition; -SET citus.next_shard_id TO 4213581; -SET citus.shard_replication_factor TO 1; -SET citus.shard_count TO 4; -CREATE TABLE source_table(a int, b int); -SELECT create_distributed_table('source_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO source_table SELECT floor(i/4), i*i FROM generate_series(1, 20) i; -SET citus.shard_count TO 3; -CREATE TABLE target_table(a int, b int); -SELECT create_distributed_table('target_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE FUNCTION square(int) RETURNS INT - AS $$ SELECT $1 * $1 $$ - LANGUAGE SQL; -select create_distributed_function('square(int)'); -NOTICE: procedure multi_mx_insert_select_repartition.square is already distributed -DETAIL: Citus distributes procedures with CREATE [PROCEDURE|FUNCTION|AGGREGATE] commands - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - -select public.colocate_proc_with_table('square', 'source_table'::regclass, 0); - colocate_proc_with_table ---------------------------------------------------------------------- - -(1 row) - --- Test along with function delegation --- function delegation only happens for "SELECT f()", and we don't use --- repartitioned INSERT/SELECT when task count is 1, so the following --- should go via coordinator -EXPLAIN (costs off) INSERT INTO target_table(a) SELECT square(4); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: pull to coordinator - -> Result -(3 rows) - -INSERT INTO target_table(a) SELECT square(4); -SELECT * FROM target_table; - a | b ---------------------------------------------------------------------- - 16 | -(1 row) - -TRUNCATE target_table; --- --- Test repartitioned INSERT/SELECT from MX worker --- -\c - - - :worker_1_port -SET search_path TO multi_mx_insert_select_repartition; -EXPLAIN (costs off) INSERT INTO target_table SELECT a, max(b) FROM source_table GROUP BY a; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus INSERT ... SELECT) - INSERT/SELECT method: repartition - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Group Key: a - -> Seq Scan on source_table_4213581 source_table -(10 rows) - -INSERT INTO target_table SELECT a, max(b) FROM source_table GROUP BY a; -SET citus.log_local_commands to on; --- INSERT .. SELECT via repartitioning with local execution -BEGIN; - select count(*) from source_table WHERE a = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM multi_mx_insert_select_repartition.source_table_4213581 source_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 4 -(1 row) - - insert into target_table SELECT a*2 FROM source_table RETURNING a; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_4213581_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_4213581_to','SELECT (a OPERATOR(pg_catalog.*) 2) AS a FROM multi_mx_insert_select_repartition.source_table_4213581 source_table WHERE true',0,'hash','{-2147483648,-715827883,715827882}'::text[],'{-715827884,715827881,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_4213583_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_4213583_to','SELECT (a OPERATOR(pg_catalog.*) 2) AS a FROM multi_mx_insert_select_repartition.source_table_4213583 source_table WHERE true',0,'hash','{-2147483648,-715827883,715827882}'::text[],'{-715827884,715827881,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT bytes FROM fetch_intermediate_results(ARRAY['repartitioned_results_xxxxx_from_4213582_to_0','repartitioned_results_xxxxx_from_4213584_to_0']::text[],'localhost',57638) bytes -NOTICE: executing the command locally: INSERT INTO multi_mx_insert_select_repartition.target_table_4213585 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213581_to_0,repartitioned_results_xxxxx_from_4213582_to_0,repartitioned_results_xxxxx_from_4213584_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) RETURNING citus_table_alias.a -NOTICE: executing the command locally: INSERT INTO multi_mx_insert_select_repartition.target_table_4213587 AS citus_table_alias (a) SELECT a FROM read_intermediate_results('{repartitioned_results_xxxxx_from_4213581_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer) RETURNING citus_table_alias.a - a ---------------------------------------------------------------------- - 0 - 0 - 0 - 2 - 2 - 2 - 2 - 4 - 4 - 4 - 4 - 6 - 6 - 6 - 6 - 8 - 8 - 8 - 8 - 10 -(20 rows) - -ROLLBACK; -BEGIN; - select count(*) from source_table WHERE a = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM multi_mx_insert_select_repartition.source_table_4213581 source_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 4 -(1 row) - - insert into target_table SELECT a FROM source_table LIMIT 10; -NOTICE: executing the command locally: SELECT a FROM multi_mx_insert_select_repartition.source_table_4213581 source_table WHERE true LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a FROM multi_mx_insert_select_repartition.source_table_4213583 source_table WHERE true LIMIT '10'::bigint -NOTICE: executing the copy locally for shard xxxxx -ROLLBACK; -\c - - - :master_port -SET search_path TO multi_mx_insert_select_repartition; -SELECT * FROM target_table ORDER BY a; - a | b ---------------------------------------------------------------------- - 0 | 9 - 1 | 49 - 2 | 121 - 3 | 225 - 4 | 361 - 5 | 400 -(6 rows) - -RESET client_min_messages; -\set VERBOSITY terse -DROP SCHEMA multi_mx_insert_select_repartition CASCADE; -NOTICE: drop cascades to 3 other objects diff --git a/src/test/regress/expected/mx_coordinator_shouldhaveshards.out b/src/test/regress/expected/mx_coordinator_shouldhaveshards.out index 547300460..b985dd330 100644 --- a/src/test/regress/expected/mx_coordinator_shouldhaveshards.out +++ b/src/test/regress/expected/mx_coordinator_shouldhaveshards.out @@ -1,17 +1,6 @@ -- -- MX_COORDINATOR_SHOULDHAVESHARDS -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA mx_coordinator_shouldhaveshards; SET search_path TO mx_coordinator_shouldhaveshards; SET citus.shard_replication_factor to 1; diff --git a/src/test/regress/expected/mx_coordinator_shouldhaveshards_0.out b/src/test/regress/expected/mx_coordinator_shouldhaveshards_0.out deleted file mode 100644 index 15cd69068..000000000 --- a/src/test/regress/expected/mx_coordinator_shouldhaveshards_0.out +++ /dev/null @@ -1,335 +0,0 @@ --- --- MX_COORDINATOR_SHOULDHAVESHARDS --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA mx_coordinator_shouldhaveshards; -SET search_path TO mx_coordinator_shouldhaveshards; -SET citus.shard_replication_factor to 1; -SET client_min_messages TO WARNING; -SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -RESET client_min_messages; -SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- issue 4508 table_1 and table_2 are used to test some edge cases --- around intermediate result pruning -CREATE TABLE table_1 (key int, value text); -SELECT create_distributed_table('table_1', 'key', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE table_2 (key int, value text); -SELECT create_distributed_table('table_2', 'key', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO table_1 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); -INSERT INTO table_2 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'); -SET citus.shard_replication_factor to 2; -CREATE TABLE table_1_rep (key int, value text); -SELECT create_distributed_table('table_1_rep', 'key', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE table_2_rep (key int, value text); -SELECT create_distributed_table('table_2_rep', 'key', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO table_1_rep VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); -INSERT INTO table_2_rep VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'); -set citus.log_intermediate_results TO ON; -set client_min_messages to debug1; -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count | key ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -INSERT INTO table_1 SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -WITH stats AS ( - SELECT count(key) m FROM table_1 -), -inserts AS ( - INSERT INTO table_2 - SELECT key, count(*) - FROM table_1 - WHERE key >= (SELECT m FROM stats) - GROUP BY key - HAVING count(*) <= (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM mx_coordinator_shouldhaveshards.table_1 -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO mx_coordinator_shouldhaveshards.table_2 (key, value) SELECT key, count(*) AS count FROM mx_coordinator_shouldhaveshards.table_1 WHERE (key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2.key, table_2.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator - count ---------------------------------------------------------------------- - 0 -(1 row) - -WITH a AS (SELECT * FROM table_1_rep ORDER BY 1,2 DESC LIMIT 1) -SELECT count(*), -key -FROM a JOIN table_2_rep USING (key) -GROUP BY key -HAVING (max(table_2_rep.value) >= (SELECT value FROM a)); -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1_rep ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2_rep USING (key)) GROUP BY a.key HAVING (max(table_2_rep.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count | key ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH a AS (SELECT * FROM table_1_rep ORDER BY 1,2 DESC LIMIT 1) -INSERT INTO table_1_rep SELECT count(*), -key -FROM a JOIN table_2_rep USING (key) -GROUP BY key -HAVING (max(table_2_rep.value) >= (SELECT value FROM a)); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1_rep ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2_rep USING (key)) GROUP BY a.key HAVING (max(table_2_rep.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -WITH stats AS ( - SELECT count(key) m FROM table_1_rep -), -inserts AS ( - INSERT INTO table_2_rep - SELECT key, count(*) - FROM table_1_rep - WHERE key >= (SELECT m FROM stats) - GROUP BY key - HAVING count(*) <= (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM mx_coordinator_shouldhaveshards.table_1_rep -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO mx_coordinator_shouldhaveshards.table_2_rep (key, value) SELECT key, count(*) AS count FROM mx_coordinator_shouldhaveshards.table_1_rep WHERE (key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2_rep.key, table_2_rep.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator - count ---------------------------------------------------------------------- - 0 -(1 row) - -\c - - - :worker_1_port -SET search_path TO mx_coordinator_shouldhaveshards; -set citus.log_intermediate_results TO ON; -set client_min_messages to debug1; -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count | key ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH a AS (SELECT * FROM table_1 ORDER BY 1,2 DESC LIMIT 1) -INSERT INTO table_1 SELECT count(*), -key -FROM a JOIN table_2 USING (key) -GROUP BY key -HAVING (max(table_2.value) >= (SELECT value FROM a)); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1 ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2 USING (key)) GROUP BY a.key HAVING (max(table_2.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -WITH stats AS ( - SELECT count(key) m FROM table_1 -), -inserts AS ( - INSERT INTO table_2 - SELECT key, count(*) - FROM table_1 - WHERE key >= (SELECT m FROM stats) - GROUP BY key - HAVING count(*) <= (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM mx_coordinator_shouldhaveshards.table_1 -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO mx_coordinator_shouldhaveshards.table_2 (key, value) SELECT key, count(*) AS count FROM mx_coordinator_shouldhaveshards.table_1 WHERE (key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2.key, table_2.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator - count ---------------------------------------------------------------------- - 0 -(1 row) - -WITH a AS (SELECT * FROM table_1_rep ORDER BY 1,2 DESC LIMIT 1) -SELECT count(*), -key -FROM a JOIN table_2_rep USING (key) -GROUP BY key -HAVING (max(table_2_rep.value) >= (SELECT value FROM a)); -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1_rep ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count, a.key FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2_rep USING (key)) GROUP BY a.key HAVING (max(table_2_rep.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx - count | key ---------------------------------------------------------------------- - 1 | 1 -(1 row) - -WITH a AS (SELECT * FROM table_1_rep ORDER BY 1,2 DESC LIMIT 1) -INSERT INTO table_1_rep SELECT count(*), -key -FROM a JOIN table_2_rep USING (key) -GROUP BY key -HAVING (max(table_2_rep.value) >= (SELECT value FROM a)); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value FROM mx_coordinator_shouldhaveshards.table_1_rep ORDER BY key, value DESC LIMIT 1 -DEBUG: push down of limit count: 1 -DEBUG: generating subplan XXX_2 for subquery SELECT int4(count(*)) AS auto_coerced_by_citus_0, (a.key)::text AS auto_coerced_by_citus_1 FROM ((SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a JOIN mx_coordinator_shouldhaveshards.table_2_rep USING (key)) GROUP BY a.key HAVING (max(table_2_rep.value) OPERATOR(pg_catalog.>=) (SELECT a_1.value FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) a_1)) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT auto_coerced_by_citus_0 AS key, auto_coerced_by_citus_1 AS value FROM (SELECT intermediate_result.auto_coerced_by_citus_0, intermediate_result.auto_coerced_by_citus_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(auto_coerced_by_citus_0 integer, auto_coerced_by_citus_1 text)) citus_insert_select_subquery -DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -WITH stats AS ( - SELECT count(key) m FROM table_1_rep -), -inserts AS ( - INSERT INTO table_2_rep - SELECT key, count(*) - FROM table_1_rep - WHERE key >= (SELECT m FROM stats) - GROUP BY key - HAVING count(*) <= (SELECT m FROM stats) - LIMIT 1 - RETURNING * -) SELECT count(*) FROM inserts; -DEBUG: generating subplan XXX_1 for CTE stats: SELECT count(key) AS m FROM mx_coordinator_shouldhaveshards.table_1_rep -DEBUG: generating subplan XXX_2 for CTE inserts: INSERT INTO mx_coordinator_shouldhaveshards.table_2_rep (key, value) SELECT key, count(*) AS count FROM mx_coordinator_shouldhaveshards.table_1_rep WHERE (key OPERATOR(pg_catalog.>=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.<=) (SELECT stats.m FROM (SELECT intermediate_result.m FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(m bigint)) stats)) LIMIT 1 RETURNING table_2_rep.key, table_2_rep.value -DEBUG: cannot push down this subquery -DETAIL: Limit clause is currently unsupported when a subquery references a column from another query -DEBUG: push down of limit count: 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text)) inserts -DEBUG: Subplan XXX_1 will be written to local file -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -DEBUG: Subplan XXX_2 will be written to local file -DEBUG: Collecting INSERT ... SELECT results on coordinator - count ---------------------------------------------------------------------- - 0 -(1 row) - -\c - - - :master_port -SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', false); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SET client_min_messages TO ERROR; -DROP SCHEMA mx_coordinator_shouldhaveshards CASCADE; -SELECT master_remove_node('localhost', :master_port); - master_remove_node ---------------------------------------------------------------------- - -(1 row) - diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index d945a07d6..45f6454f5 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1,13 +1,6 @@ -- -- PG15 -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE SCHEMA pg15; SET search_path TO pg15; SET citus.next_shard_id TO 960000; diff --git a/src/test/regress/expected/pg15_0.out b/src/test/regress/expected/pg15_0.out deleted file mode 100644 index b1ed9cc5b..000000000 --- a/src/test/regress/expected/pg15_0.out +++ /dev/null @@ -1,9 +0,0 @@ --- --- PG15 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/pg15_jsonpath.out b/src/test/regress/expected/pg15_jsonpath.out index 335a7fbba..a3bdf8a55 100644 --- a/src/test/regress/expected/pg15_jsonpath.out +++ b/src/test/regress/expected/pg15_jsonpath.out @@ -2,13 +2,6 @@ -- PG15 jsonpath tests -- Relevant pg commit: e26114c817b610424010cfbe91a743f591246ff1 -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE SCHEMA jsonpath; SET search_path TO jsonpath; CREATE TABLE jsonpath_test (id serial, sample text); diff --git a/src/test/regress/expected/pg15_jsonpath_0.out b/src/test/regress/expected/pg15_jsonpath_0.out deleted file mode 100644 index b496f64dc..000000000 --- a/src/test/regress/expected/pg15_jsonpath_0.out +++ /dev/null @@ -1,10 +0,0 @@ --- --- PG15 jsonpath tests --- Relevant pg commit: e26114c817b610424010cfbe91a743f591246ff1 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/pgmerge.out b/src/test/regress/expected/pgmerge.out index 0c2f9b741..9057cac6b 100644 --- a/src/test/regress/expected/pgmerge.out +++ b/src/test/regress/expected/pgmerge.out @@ -1,10 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif -- -- MERGE test from PG community (adapted to Citus by converting all tables to Citus local) -- diff --git a/src/test/regress/expected/pgmerge_0.out b/src/test/regress/expected/pgmerge_0.out deleted file mode 100644 index a7e3fbf20..000000000 --- a/src/test/regress/expected/pgmerge_0.out +++ /dev/null @@ -1,6 +0,0 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index c761efb3e..c727cb0d6 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -258,17 +258,6 @@ DROP PUBLICATION "pub-all-insertupdateonly"; DROP PUBLICATION "pub-all"; DROP PUBLICATION pubpartitioned; DROP PUBLICATION pubnotdistributed; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -SET client_min_messages TO ERROR; -DROP SCHEMA publication CASCADE; -DROP SCHEMA "publication-1" CASCADE; -DROP SCHEMA citus_schema_1 CASCADE; -\q -\endif -- recreate a mixed publication CREATE PUBLICATION pubtables FOR TABLE test, "publication-1"."test-pubs", citus_schema_1.test; -- operations on an existing distributed table diff --git a/src/test/regress/expected/publication_0.out b/src/test/regress/expected/publication_0.out deleted file mode 100644 index 14fa94d17..000000000 --- a/src/test/regress/expected/publication_0.out +++ /dev/null @@ -1,270 +0,0 @@ -CREATE SCHEMA publication; -CREATE SCHEMA "publication-1"; -SET search_path TO publication; -SET citus.shard_replication_factor TO 1; -CREATE OR REPLACE FUNCTION activate_node_snapshot() - RETURNS text[] - LANGUAGE C STRICT - AS 'citus'; -COMMENT ON FUNCTION activate_node_snapshot() - IS 'commands to activate node snapshot'; -\c - - - :worker_1_port -SET citus.enable_ddl_propagation TO off; -CREATE OR REPLACE FUNCTION activate_node_snapshot() - RETURNS text[] - LANGUAGE C STRICT - AS 'citus'; -COMMENT ON FUNCTION activate_node_snapshot() - IS 'commands to activate node snapshot'; -\c - - - :worker_2_port -SET citus.enable_ddl_propagation TO off; -CREATE OR REPLACE FUNCTION activate_node_snapshot() - RETURNS text[] - LANGUAGE C STRICT - AS 'citus'; -COMMENT ON FUNCTION activate_node_snapshot() - IS 'commands to activate node snapshot'; --- create some publications with conflicting names on worker node --- publication will be different from coordinator -CREATE PUBLICATION "pub-all"; --- publication will be same as coordinator -CREATE PUBLICATION "pub-all-insertupdateonly" FOR ALL TABLES WITH (publish = 'insert, update');; -\c - - - :master_port -SET search_path TO publication; -SET citus.shard_replication_factor TO 1; --- do not create publications on worker 2 initially -SELECT citus_remove_node('localhost', :worker_2_port); - citus_remove_node ---------------------------------------------------------------------- - -(1 row) - --- create a non-distributed publication -SET citus.enable_ddl_propagation TO off; -CREATE PUBLICATION pubnotdistributed WITH (publish = 'delete'); -RESET citus.enable_ddl_propagation; -ALTER PUBLICATION pubnotdistributed SET (publish = 'truncate'); --- create regular, distributed publications -CREATE PUBLICATION pubempty; -CREATE PUBLICATION pubinsertonly WITH (publish = 'insert'); -CREATE PUBLICATION "pub-all" FOR ALL TABLES; -CREATE PUBLICATION "pub-all-insertupdateonly" FOR ALL TABLES WITH (publish = 'insert, update'); --- add worker 2 with publications -SELECT 1 FROM citus_add_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- Check publications on all the nodes, if we see the same publication name twice then its definition differs --- Note that publications are special in the sense that the coordinator object might differ from --- worker objects due to the presence of regular tables. -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLICATION%' ORDER BY 1) s$$) - ORDER BY c) s; - c ---------------------------------------------------------------------- - SELECT worker_create_or_replace_object('CREATE PUBLICATION "pub-all" FOR ALL TABLES WITH (publish_via_partition_root = ''false'', publish = ''insert, update, delete, truncate'')'); - SELECT worker_create_or_replace_object('CREATE PUBLICATION "pub-all-insertupdateonly" FOR ALL TABLES WITH (publish_via_partition_root = ''false'', publish = ''insert, update'')'); - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubempty WITH (publish_via_partition_root = ''false'', publish = ''insert, update, delete, truncate'')'); - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubinsertonly WITH (publish_via_partition_root = ''false'', publish = ''insert'')'); -(4 rows) - -CREATE TABLE test (x int primary key, y int, "column-1" int, doc xml); -CREATE TABLE "test-pubs" (x int primary key, y int, "column-1" int); -CREATE TABLE "publication-1"."test-pubs" (x int primary key, y int, "column-1" int); --- various operations on a publication with only local tables -CREATE PUBLICATION pubtables_orig FOR TABLE test, "test-pubs", "publication-1"."test-pubs" WITH (publish = 'insert, truncate'); -ALTER PUBLICATION pubtables_orig DROP TABLE test; -ALTER PUBLICATION pubtables_orig ADD TABLE test; --- publication will be empty on worker nodes, since all tables are local -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLICATION%' AND c LIKE '%pubtables%' ORDER BY 1) s$$) - ORDER BY c) s; - c ---------------------------------------------------------------------- - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubtables_orig WITH (publish_via_partition_root = ''false'', publish = ''insert, truncate'')'); -(1 row) - --- distribute a table and create a tenant schema, creating a mixed publication -SELECT create_distributed_table('test','x', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET citus.enable_schema_based_sharding TO ON; -CREATE SCHEMA citus_schema_1; -CREATE TABLE citus_schema_1.test (x int primary key, y int, "column-1" int, doc xml); -SET citus.enable_schema_based_sharding TO OFF; -ALTER PUBLICATION pubtables_orig ADD TABLE citus_schema_1.test; --- some generic operations -ALTER PUBLICATION pubtables_orig RENAME TO pubtables; -ALTER PUBLICATION pubtables SET (publish = 'insert, update, delete'); -ALTER PUBLICATION pubtables OWNER TO postgres; -ALTER PUBLICATION pubtables SET (publish = 'inert, update, delete'); -ERROR: unrecognized "publish" value: "inert" -ALTER PUBLICATION pubtables ADD TABLE notexist; -ERROR: relation "notexist" does not exist --- operations with a distributed table -ALTER PUBLICATION pubtables DROP TABLE test; -ALTER PUBLICATION pubtables ADD TABLE test; -ALTER PUBLICATION pubtables SET TABLE test, "test-pubs", "publication-1"."test-pubs", citus_schema_1.test; --- operations with a tenant schema table -ALTER PUBLICATION pubtables DROP TABLE citus_schema_1.test; -ALTER PUBLICATION pubtables ADD TABLE citus_schema_1.test; -ALTER PUBLICATION pubtables SET TABLE test, "test-pubs", "publication-1"."test-pubs", citus_schema_1.test; --- operations with a local table in a mixed publication -ALTER PUBLICATION pubtables DROP TABLE "test-pubs"; -ALTER PUBLICATION pubtables ADD TABLE "test-pubs"; -SELECT create_distributed_table('"test-pubs"', 'x'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- test and test-pubs will show up in worker nodes -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLICATION%' AND c LIKE '%pubtables%' ORDER BY 1) s$$) - ORDER BY c) s; - c ---------------------------------------------------------------------- - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubtables FOR TABLE publication.test, citus_schema_1.test, publication."test-pubs" WITH (publish_via_partition_root = ''false'', publish = ''insert, update, delete'')'); -(1 row) - --- operations with a strangely named distributed table in a mixed publication -ALTER PUBLICATION pubtables DROP TABLE "test-pubs"; -ALTER PUBLICATION pubtables ADD TABLE "test-pubs"; --- create a publication with distributed and local tables -DROP PUBLICATION pubtables; -CREATE PUBLICATION pubtables FOR TABLE test, "test-pubs", "publication-1"."test-pubs", citus_schema_1.test; --- change distributed tables -SELECT alter_distributed_table('test', shard_count := 5, cascade_to_colocated := true); -NOTICE: creating a new table for publication.test -NOTICE: moving the data of publication.test -NOTICE: dropping the old publication.test -NOTICE: renaming the new table to publication.test -NOTICE: creating a new table for publication."test-pubs" -NOTICE: moving the data of publication."test-pubs" -NOTICE: dropping the old publication."test-pubs" -NOTICE: renaming the new table to publication."test-pubs" - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT undistribute_table('test'); -NOTICE: creating a new table for publication.test -NOTICE: moving the data of publication.test -NOTICE: dropping the old publication.test -NOTICE: renaming the new table to publication.test - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -SELECT citus_add_local_table_to_metadata('test'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table_concurrently('test', 'x'); - create_distributed_table_concurrently ---------------------------------------------------------------------- - -(1 row) - -SELECT undistribute_table('"test-pubs"'); -NOTICE: creating a new table for publication."test-pubs" -NOTICE: moving the data of publication."test-pubs" -NOTICE: dropping the old publication."test-pubs" -NOTICE: renaming the new table to publication."test-pubs" - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_reference_table('"test-pubs"'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - --- publications are unchanged despite various tranformations -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLICATION%' AND c LIKE '%pubtables%' ORDER BY 1) s$$) - ORDER BY c) s; - c ---------------------------------------------------------------------- - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubtables FOR TABLE citus_schema_1.test, publication.test, publication."test-pubs" WITH (publish_via_partition_root = ''false'', publish = ''insert, update, delete, truncate'')'); -(1 row) - --- partitioned table -CREATE TABLE testpub_partitioned (a int, b text, c text) PARTITION BY RANGE (a); -CREATE TABLE testpub_partitioned_0 PARTITION OF testpub_partitioned FOR VALUES FROM (1) TO (10); -ALTER TABLE testpub_partitioned_0 ADD PRIMARY KEY (a); -ALTER TABLE testpub_partitioned_0 REPLICA IDENTITY USING INDEX testpub_partitioned_0_pkey; -CREATE TABLE testpub_partitioned_1 PARTITION OF testpub_partitioned FOR VALUES FROM (11) TO (20); -ALTER TABLE testpub_partitioned_1 ADD PRIMARY KEY (a); -ALTER TABLE testpub_partitioned_1 REPLICA IDENTITY USING INDEX testpub_partitioned_1_pkey; -CREATE PUBLICATION pubpartitioned FOR TABLE testpub_partitioned WITH (publish_via_partition_root = 'true'); -SELECT create_distributed_table('testpub_partitioned', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLICATION%' AND c LIKE '%pubpartitioned%' ORDER BY 1) s$$) - ORDER BY c) s; - c ---------------------------------------------------------------------- - SELECT worker_create_or_replace_object('CREATE PUBLICATION pubpartitioned FOR TABLE publication.testpub_partitioned WITH (publish_via_partition_root = ''true'', publish = ''insert, update, delete, truncate'')'); -(1 row) - -DROP PUBLICATION pubpartitioned; -CREATE PUBLICATION pubpartitioned FOR TABLE testpub_partitioned WITH (publish_via_partition_root = 'true'); --- add a partition -ALTER PUBLICATION pubpartitioned ADD TABLE testpub_partitioned_1; -SELECT DISTINCT c FROM ( - SELECT unnest(result::text[]) c - FROM run_command_on_workers($$ - SELECT array_agg(c) FROM (SELECT c FROM unnest(activate_node_snapshot()) c WHERE c LIKE '%CREATE PUBLIATION%' AND c LIKE '%pubpartitioned%' ORDER BY 1) s$$) - ORDER BY c) s; -ERROR: malformed array literal: "" -DETAIL: Array value must start with "{" or dimension information. --- make sure we can sync all the publication metadata -SELECT start_metadata_sync_to_all_nodes(); - start_metadata_sync_to_all_nodes ---------------------------------------------------------------------- - t -(1 row) - -DROP PUBLICATION pubempty; -DROP PUBLICATION pubtables; -DROP PUBLICATION pubinsertonly; -DROP PUBLICATION "pub-all-insertupdateonly"; -DROP PUBLICATION "pub-all"; -DROP PUBLICATION pubpartitioned; -DROP PUBLICATION pubnotdistributed; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -SET client_min_messages TO ERROR; -DROP SCHEMA publication CASCADE; -DROP SCHEMA "publication-1" CASCADE; -DROP SCHEMA citus_schema_1 CASCADE; -\q diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index f485763c5..dd8d6fe7b 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -1,17 +1,6 @@ -- -- SINGLE_NODE -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - t -(1 row) - CREATE SCHEMA single_node; SET search_path TO single_node; SET citus.shard_count TO 4; diff --git a/src/test/regress/expected/single_node_0.out b/src/test/regress/expected/single_node_0.out deleted file mode 100644 index 321d283f8..000000000 --- a/src/test/regress/expected/single_node_0.out +++ /dev/null @@ -1,2581 +0,0 @@ --- --- SINGLE_NODE --- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; - server_version_ge_15 ---------------------------------------------------------------------- - f -(1 row) - -CREATE SCHEMA single_node; -SET search_path TO single_node; -SET citus.shard_count TO 4; -SET citus.shard_replication_factor TO 1; -SET citus.next_shard_id TO 90630500; --- Ensure tuple data in explain analyze output is the same on all PG versions -SET citus.enable_binary_protocol = TRUE; --- do not cache any connections for now, will enable it back soon -ALTER SYSTEM SET citus.max_cached_conns_per_worker TO 0; --- adding the coordinator as inactive is disallowed -SELECT 1 FROM master_add_inactive_node('localhost', :master_port, groupid => 0); -ERROR: coordinator node cannot be added as inactive node --- before adding a node we are not officially a coordinator -SELECT citus_is_coordinator(); - citus_is_coordinator ---------------------------------------------------------------------- - f -(1 row) - --- idempotently add node to allow this test to run without add_coordinator -SET client_min_messages TO WARNING; -SELECT 1 FROM citus_set_coordinator_host('localhost', :master_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- after adding a node we are officially a coordinator -SELECT citus_is_coordinator(); - citus_is_coordinator ---------------------------------------------------------------------- - t -(1 row) - --- coordinator cannot be disabled -SELECT 1 FROM citus_disable_node('localhost', :master_port); -ERROR: cannot change "isactive" field of the coordinator node -RESET client_min_messages; -SELECT 1 FROM master_remove_node('localhost', :master_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SELECT count(*) FROM pg_dist_node; - count ---------------------------------------------------------------------- - 0 -(1 row) - --- there are no workers now, but we should still be able to create Citus tables --- force local execution when creating the index -ALTER SYSTEM SET citus.local_shared_pool_size TO -1; --- Postmaster might not ack SIGHUP signal sent by pg_reload_conf() immediately, --- so we need to sleep for some amount of time to do our best to ensure that --- postmaster reflects GUC changes. -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(0.1); - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE failover_to_local (a int); -SELECT create_distributed_table('failover_to_local', 'a', shard_count=>32); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE INDEX CONCURRENTLY ON failover_to_local(a); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. - Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: Consider using a higher value for max_connections --- reset global GUC changes -ALTER SYSTEM RESET citus.local_shared_pool_size; -ALTER SYSTEM RESET citus.max_cached_conns_per_worker; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -CREATE TABLE single_node_nullkey_c1(a int, b int); -SELECT create_distributed_table('single_node_nullkey_c1', null, colocate_with=>'none', distribution_type=>null); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE single_node_nullkey_c2(a int, b int); -SELECT create_distributed_table('single_node_nullkey_c2', null, colocate_with=>'none', distribution_type=>null); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- created on different colocation groups .. -SELECT -( - SELECT colocationid FROM pg_dist_partition - WHERE logicalrelid = 'single_node.single_node_nullkey_c1'::regclass -) -!= -( - SELECT colocationid FROM pg_dist_partition - WHERE logicalrelid = 'single_node.single_node_nullkey_c2'::regclass -); - ?column? ---------------------------------------------------------------------- - t -(1 row) - --- .. but both are associated to coordinator -SELECT groupid = 0 FROM pg_dist_placement -WHERE shardid = ( - SELECT shardid FROM pg_dist_shard - WHERE logicalrelid = 'single_node.single_node_nullkey_c1'::regclass -); - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SELECT groupid = 0 FROM pg_dist_placement -WHERE shardid = ( - SELECT shardid FROM pg_dist_shard - WHERE logicalrelid = 'single_node.single_node_nullkey_c2'::regclass -); - ?column? ---------------------------------------------------------------------- - t -(1 row) - --- try creating a single-shard table from a shard relation -SELECT shardid AS round_robin_test_c1_shard_id FROM pg_dist_shard WHERE logicalrelid = 'single_node.single_node_nullkey_c1'::regclass \gset -SELECT create_distributed_table('single_node_nullkey_c1_' || :round_robin_test_c1_shard_id , null, colocate_with=>'none', distribution_type=>null); -ERROR: relation "single_node_nullkey_c1_90630532" is a shard relation --- create a tenant schema on single node setup -SET citus.enable_schema_based_sharding TO ON; -CREATE SCHEMA tenant_1; -CREATE TABLE tenant_1.tbl_1 (a int); --- verify that we recorded tenant_1 in pg_dist_schema -SELECT COUNT(*)=1 FROM pg_dist_schema WHERE schemaid::regnamespace::text = 'tenant_1'; - ?column? ---------------------------------------------------------------------- - t -(1 row) - --- verify that tenant_1.tbl_1 is recorded in pg_dist_partition, as a single-shard table -SELECT COUNT(*)=1 FROM pg_dist_partition -WHERE logicalrelid = 'tenant_1.tbl_1'::regclass AND - partmethod = 'n' AND repmodel = 's' AND colocationid IS NOT NULL; - ?column? ---------------------------------------------------------------------- - t -(1 row) - -RESET citus.enable_schema_based_sharding; --- Test lazy conversion from Citus local to single-shard tables --- and reference tables, on single node. This means that no shard --- replication should be needed. -CREATE TABLE ref_table_conversion_test ( - a int PRIMARY KEY -); -SELECT citus_add_local_table_to_metadata('ref_table_conversion_test'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - --- save old shardid and placementid -SELECT get_shard_id_for_distribution_column('single_node.ref_table_conversion_test') AS ref_table_conversion_test_old_shard_id \gset -SELECT placementid AS ref_table_conversion_test_old_coord_placement_id FROM pg_dist_placement WHERE shardid = :ref_table_conversion_test_old_shard_id \gset -SELECT create_reference_table('ref_table_conversion_test'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -SELECT public.verify_pg_dist_partition_for_reference_table('single_node.ref_table_conversion_test'); - verify_pg_dist_partition_for_reference_table ---------------------------------------------------------------------- - t -(1 row) - -SELECT public.verify_shard_placements_for_reference_table('single_node.ref_table_conversion_test', - :ref_table_conversion_test_old_shard_id, - :ref_table_conversion_test_old_coord_placement_id); - verify_shard_placements_for_reference_table ---------------------------------------------------------------------- - t -(1 row) - -CREATE TABLE single_shard_conversion_test_1 ( - int_col_1 int PRIMARY KEY, - text_col_1 text UNIQUE, - int_col_2 int -); -SELECT citus_add_local_table_to_metadata('single_shard_conversion_test_1'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - --- save old shardid -SELECT get_shard_id_for_distribution_column('single_node.single_shard_conversion_test_1') AS single_shard_conversion_test_1_old_shard_id \gset -SELECT create_distributed_table('single_shard_conversion_test_1', null, colocate_with=>'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT public.verify_pg_dist_partition_for_single_shard_table('single_node.single_shard_conversion_test_1'); - verify_pg_dist_partition_for_single_shard_table ---------------------------------------------------------------------- - t -(1 row) - -SELECT public.verify_shard_placement_for_single_shard_table('single_node.single_shard_conversion_test_1', :single_shard_conversion_test_1_old_shard_id, true); - verify_shard_placement_for_single_shard_table ---------------------------------------------------------------------- - t -(1 row) - -CREATE TABLE single_shard_conversion_test_2 ( - int_col_1 int -); -SELECT citus_add_local_table_to_metadata('single_shard_conversion_test_2'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - --- save old shardid -SELECT get_shard_id_for_distribution_column('single_node.single_shard_conversion_test_2') AS single_shard_conversion_test_2_old_shard_id \gset -SELECT create_distributed_table('single_shard_conversion_test_2', null, colocate_with=>'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT public.verify_pg_dist_partition_for_single_shard_table('single_node.single_shard_conversion_test_2'); - verify_pg_dist_partition_for_single_shard_table ---------------------------------------------------------------------- - t -(1 row) - -SELECT public.verify_shard_placement_for_single_shard_table('single_node.single_shard_conversion_test_2', :single_shard_conversion_test_2_old_shard_id, true); - verify_shard_placement_for_single_shard_table ---------------------------------------------------------------------- - t -(1 row) - --- make sure that they're created on different colocation groups -SELECT -( - SELECT colocationid FROM pg_dist_partition - WHERE logicalrelid = 'single_node.single_shard_conversion_test_1'::regclass -) -!= -( - SELECT colocationid FROM pg_dist_partition - WHERE logicalrelid = 'single_node.single_shard_conversion_test_2'::regclass -); - ?column? ---------------------------------------------------------------------- - t -(1 row) - -SET client_min_messages TO WARNING; -DROP TABLE failover_to_local, single_node_nullkey_c1, single_node_nullkey_c2, ref_table_conversion_test, single_shard_conversion_test_1, single_shard_conversion_test_2; -DROP SCHEMA tenant_1 CASCADE; -RESET client_min_messages; --- so that we don't have to update rest of the test output -SET citus.next_shard_id TO 90630500; -CREATE TABLE ref(x int, y int); -SELECT create_reference_table('ref'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -SELECT groupid, nodename, nodeport, isactive, shouldhaveshards, hasmetadata, metadatasynced FROM pg_dist_node; - groupid | nodename | nodeport | isactive | shouldhaveshards | hasmetadata | metadatasynced ---------------------------------------------------------------------- - 0 | localhost | 57636 | t | t | t | t -(1 row) - -DROP TABLE ref; --- remove the coordinator to try again with create_reference_table -SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node WHERE groupid = 0; - master_remove_node ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE loc(x int, y int); -SELECT citus_add_local_table_to_metadata('loc'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - -SELECT groupid, nodename, nodeport, isactive, shouldhaveshards, hasmetadata, metadatasynced FROM pg_dist_node; - groupid | nodename | nodeport | isactive | shouldhaveshards | hasmetadata | metadatasynced ---------------------------------------------------------------------- - 0 | localhost | 57636 | t | t | t | t -(1 row) - -DROP TABLE loc; --- remove the coordinator to try again with create_distributed_table -SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node WHERE groupid = 0; - master_remove_node ---------------------------------------------------------------------- - -(1 row) - --- verify the coordinator gets auto added with the localhost guc -ALTER SYSTEM SET citus.local_hostname TO '127.0.0.1'; --although not a hostname, should work for connecting locally -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(.1); -- wait to make sure the config has changed before running the GUC - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE test(x int, y int); -SELECT create_distributed_table('test','x'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT groupid, nodename, nodeport, isactive, shouldhaveshards, hasmetadata, metadatasynced FROM pg_dist_node; - groupid | nodename | nodeport | isactive | shouldhaveshards | hasmetadata | metadatasynced ---------------------------------------------------------------------- - 0 | 127.0.0.1 | 57636 | t | t | t | t -(1 row) - -DROP TABLE test; --- remove the coordinator to try again -SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node WHERE groupid = 0; - master_remove_node ---------------------------------------------------------------------- - -(1 row) - -ALTER SYSTEM RESET citus.local_hostname; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(.1); -- wait to make sure the config has changed before running the GUC - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE test(x int, y int); -SELECT create_distributed_table('test','x'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT groupid, nodename, nodeport, isactive, shouldhaveshards, hasmetadata, metadatasynced FROM pg_dist_node; - groupid | nodename | nodeport | isactive | shouldhaveshards | hasmetadata | metadatasynced ---------------------------------------------------------------------- - 0 | localhost | 57636 | t | t | t | t -(1 row) - -BEGIN; - -- we should not enable MX for this temporary node just because - -- it'd spawn a bg worker targeting this node - -- and that changes the connection count specific tests - -- here - SET LOCAL citus.enable_metadata_sync TO OFF; - -- cannot add workers with specific IP as long as I have a placeholder coordinator record - SELECT 1 FROM master_add_node('127.0.0.1', :worker_1_port); -ERROR: cannot add a worker node when the coordinator hostname is set to localhost -DETAIL: Worker nodes need to be able to connect to the coordinator to transfer data. -HINT: Use SELECT citus_set_coordinator_host('') to configure the coordinator hostname -COMMIT; -BEGIN; - -- we should not enable MX for this temporary node just because - -- it'd spawn a bg worker targeting this node - -- and that changes the connection count specific tests - -- here - SET LOCAL citus.enable_metadata_sync TO OFF; - -- adding localhost workers is ok - SELECT 1 FROM master_add_node('localhost', :worker_1_port); -NOTICE: shards are still on the coordinator after adding the new node -HINT: Use SELECT rebalance_table_shards(); to balance shards data between workers and coordinator or SELECT citus_drain_node('localhost',57636); to permanently move shards away from the coordinator. - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -COMMIT; --- we don't need this node anymore -SELECT 1 FROM master_remove_node('localhost', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- set the coordinator host to something different than localhost -SELECT 1 FROM citus_set_coordinator_host('127.0.0.1'); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -BEGIN; - -- we should not enable MX for this temporary node just because - -- it'd spawn a bg worker targeting this node - -- and that changes the connection count specific tests - -- here - SET LOCAL citus.enable_metadata_sync TO OFF; - -- adding workers with specific IP is ok now - SELECT 1 FROM master_add_node('127.0.0.1', :worker_1_port); -NOTICE: shards are still on the coordinator after adding the new node -HINT: Use SELECT rebalance_table_shards(); to balance shards data between workers and coordinator or SELECT citus_drain_node('127.0.0.1',57636); to permanently move shards away from the coordinator. - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -COMMIT; --- we don't need this node anymore -SELECT 1 FROM master_remove_node('127.0.0.1', :worker_1_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- set the coordinator host back to localhost for the remainder of tests -SELECT 1 FROM citus_set_coordinator_host('localhost'); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- should have shards setting should not really matter for a single node -SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', true); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -CREATE TYPE new_type AS (n int, m text); -CREATE TABLE test_2(x int, y int, z new_type); -SELECT create_distributed_table('test_2','x'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE ref(a int, b int); -SELECT create_reference_table('ref'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE local(c int, d int); -CREATE TABLE public.another_schema_table(a int, b int); -SELECT create_distributed_table('public.another_schema_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE non_binary_copy_test (key int PRIMARY KEY, value new_type); -SELECT create_distributed_table('non_binary_copy_test', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO non_binary_copy_test SELECT i, (i, 'citus9.5')::new_type FROM generate_series(0,1000)i; --- Confirm the basics work -INSERT INTO test VALUES (1, 2), (3, 4), (5, 6), (2, 7), (4, 5); -SELECT * FROM test WHERE x = 1; - x | y ---------------------------------------------------------------------- - 1 | 2 -(1 row) - -SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 5 -(1 row) - -SELECT * FROM test ORDER BY x; - x | y ---------------------------------------------------------------------- - 1 | 2 - 2 | 7 - 3 | 4 - 4 | 5 - 5 | 6 -(5 rows) - -UPDATE test SET y = y + 1 RETURNING *; - x | y ---------------------------------------------------------------------- - 1 | 3 - 2 | 8 - 3 | 5 - 4 | 6 - 5 | 7 -(5 rows) - -WITH cte_1 AS (UPDATE test SET y = y - 1 RETURNING *) SELECT * FROM cte_1 ORDER BY 1,2; - x | y ---------------------------------------------------------------------- - 1 | 2 - 2 | 7 - 3 | 4 - 4 | 5 - 5 | 6 -(5 rows) - --- show that we can filter remote commands --- given that citus.grep_remote_commands, we log all commands -SET citus.log_local_commands to true; -SELECT count(*) FROM public.another_schema_table WHERE a = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM public.another_schema_table_90630515 another_schema_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - --- grep matches all commands -SET citus.grep_remote_commands TO "%%"; -SELECT count(*) FROM public.another_schema_table WHERE a = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM public.another_schema_table_90630515 another_schema_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 0 -(1 row) - --- only filter a specific shard for the local execution -BEGIN; - SET LOCAL citus.grep_remote_commands TO "%90630515%"; - SELECT count(*) FROM public.another_schema_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM public.another_schema_table_90630515 another_schema_table WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- match nothing - SET LOCAL citus.grep_remote_commands TO '%nothing%'; - SELECT count(*) FROM public.another_schema_table; - count ---------------------------------------------------------------------- - 0 -(1 row) - -COMMIT; --- only filter a specific shard for the remote execution -BEGIN; - SET LOCAL citus.enable_local_execution TO FALSE; - SET LOCAL citus.grep_remote_commands TO '%90630515%'; - SET LOCAL citus.log_remote_commands TO ON; - SELECT count(*) FROM public.another_schema_table; -NOTICE: issuing SELECT count(*) AS count FROM public.another_schema_table_90630515 another_schema_table WHERE true -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- match nothing - SET LOCAL citus.grep_remote_commands TO '%nothing%'; - SELECT count(*) FROM public.another_schema_table; - count ---------------------------------------------------------------------- - 0 -(1 row) - -COMMIT; -RESET citus.log_local_commands; -RESET citus.grep_remote_commands; --- Test upsert with constraint -CREATE TABLE upsert_test -( - part_key int UNIQUE, - other_col int, - third_col int -); --- distribute the table -SELECT create_distributed_table('upsert_test', 'part_key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- do a regular insert -INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1), (2, 2) RETURNING *; - part_key | other_col | third_col ---------------------------------------------------------------------- - 1 | 1 | - 2 | 2 | -(2 rows) - -SET citus.log_remote_commands to true; --- observe that there is a conflict and the following query does nothing -INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT DO NOTHING RETURNING *; -NOTICE: executing the command locally: INSERT INTO single_node.upsert_test_90630523 AS citus_table_alias (part_key, other_col) VALUES (1, 1) ON CONFLICT DO NOTHING RETURNING part_key, other_col, third_col - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - --- same as the above with different syntax -INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO NOTHING RETURNING *; -NOTICE: executing the command locally: INSERT INTO single_node.upsert_test_90630523 AS citus_table_alias (part_key, other_col) VALUES (1, 1) ON CONFLICT(part_key) DO NOTHING RETURNING part_key, other_col, third_col - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - --- again the same query with another syntax -INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT upsert_test_part_key_key DO NOTHING RETURNING *; -NOTICE: executing the command locally: INSERT INTO single_node.upsert_test_90630523 AS citus_table_alias (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT upsert_test_part_key_key_90630523 DO NOTHING RETURNING part_key, other_col, third_col - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - -BEGIN; --- force local execution -SELECT count(*) FROM upsert_test WHERE part_key = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.upsert_test_90630523 upsert_test WHERE (part_key OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - -SET citus.log_remote_commands to false; --- multi-shard pushdown query that goes through local execution -INSERT INTO upsert_test (part_key, other_col) SELECT part_key, other_col FROM upsert_test ON CONFLICT ON CONSTRAINT upsert_test_part_key_key DO NOTHING RETURNING *; - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - --- multi-shard pull-to-coordinator query that goes through local execution -INSERT INTO upsert_test (part_key, other_col) SELECT part_key, other_col FROM upsert_test LIMIT 100 ON CONFLICT ON CONSTRAINT upsert_test_part_key_key DO NOTHING RETURNING *; - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - -COMMIT; --- to test citus local tables -select undistribute_table('upsert_test'); -NOTICE: creating a new table for single_node.upsert_test -NOTICE: moving the data of single_node.upsert_test -NOTICE: dropping the old single_node.upsert_test -NOTICE: renaming the new table to single_node.upsert_test - undistribute_table ---------------------------------------------------------------------- - -(1 row) - --- create citus local table -select citus_add_local_table_to_metadata('upsert_test'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - --- test the constraint with local execution -INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT upsert_test_part_key_key DO NOTHING RETURNING *; - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - -DROP TABLE upsert_test; -CREATE TABLE relation_tracking_table_1(id int, nonid int); -SELECT create_distributed_table('relation_tracking_table_1', 'id', colocate_with := 'none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO relation_tracking_table_1 select generate_series(6, 10000, 1), 0; -CREATE or REPLACE function foo() -returns setof relation_tracking_table_1 -AS $$ -BEGIN -RETURN query select * from relation_tracking_table_1 order by 1 limit 10; -end; -$$ language plpgsql; -CREATE TABLE relation_tracking_table_2 (id int, nonid int); --- use the relation-access in this session -select foo(); - foo ---------------------------------------------------------------------- - (6,0) - (7,0) - (8,0) - (9,0) - (10,0) - (11,0) - (12,0) - (13,0) - (14,0) - (15,0) -(10 rows) - --- we should be able to use sequential mode, as the previous multi-shard --- relation access has been cleaned-up -BEGIN; -SET LOCAL citus.multi_shard_modify_mode TO sequential; -INSERT INTO relation_tracking_table_2 select generate_series(6, 1000, 1), 0; -SELECT create_distributed_table('relation_tracking_table_2', 'id', colocate_with := 'none'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$single_node.relation_tracking_table_2$$) - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) FROM relation_tracking_table_2; - count ---------------------------------------------------------------------- - 995 -(1 row) - -ROLLBACK; -BEGIN; -INSERT INTO relation_tracking_table_2 select generate_series(6, 1000, 1), 0; -SELECT create_distributed_table('relation_tracking_table_2', 'id', colocate_with := 'none'); -NOTICE: Copying data from local table... -NOTICE: copying the data has completed -DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$single_node.relation_tracking_table_2$$) - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT count(*) FROM relation_tracking_table_2; - count ---------------------------------------------------------------------- - 995 -(1 row) - -COMMIT; -SET client_min_messages TO ERROR; -DROP TABLE relation_tracking_table_2, relation_tracking_table_1 CASCADE; -RESET client_min_messages; -CREATE SCHEMA "Quoed.Schema"; -SET search_path TO "Quoed.Schema"; -CREATE TABLE "long_constraint_upsert\_test" -( - part_key int, - other_col int, - third_col int, - CONSTRAINT "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" UNIQUE (part_key) -); -NOTICE: identifier "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" will be truncated to "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted " --- distribute the table and create shards -SELECT create_distributed_table('"long_constraint_upsert\_test"', 'part_key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO "long_constraint_upsert\_test" (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" DO NOTHING RETURNING *; -NOTICE: identifier "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" will be truncated to "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted " - part_key | other_col | third_col ---------------------------------------------------------------------- - 1 | 1 | -(1 row) - -ALTER TABLE "long_constraint_upsert\_test" RENAME TO simple_table_name; -INSERT INTO simple_table_name (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" DO NOTHING RETURNING *; -NOTICE: identifier "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" will be truncated to "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted " - part_key | other_col | third_col ---------------------------------------------------------------------- -(0 rows) - --- this is currently not supported, but once we support --- make sure that the following query also works fine -ALTER TABLE simple_table_name RENAME CONSTRAINT "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" TO simple_constraint_name; -NOTICE: identifier "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted \aconstraint" will be truncated to "looo oooo ooooo ooooooooooooooooo oooooooo oooooooo ng quoted " -ERROR: renaming constraints belonging to distributed tables is currently unsupported ---INSERT INTO simple_table_name (part_key, other_col) VALUES (1, 1) ON CONFLICT ON CONSTRAINT simple_constraint_name DO NOTHING RETURNING *; -SET search_path TO single_node; -SET client_min_messages TO ERROR; -DROP SCHEMA "Quoed.Schema" CASCADE; -RESET client_min_messages; --- test partitioned index creation with long name -CREATE TABLE test_index_creation1 -( - tenant_id integer NOT NULL, - timeperiod timestamp without time zone NOT NULL, - field1 integer NOT NULL, - inserted_utc timestamp without time zone NOT NULL DEFAULT now(), - PRIMARY KEY(tenant_id, timeperiod) -) PARTITION BY RANGE (timeperiod); -CREATE TABLE test_index_creation1_p2020_09_26 -PARTITION OF test_index_creation1 FOR VALUES FROM ('2020-09-26 00:00:00') TO ('2020-09-27 00:00:00'); -CREATE TABLE test_index_creation1_p2020_09_27 -PARTITION OF test_index_creation1 FOR VALUES FROM ('2020-09-27 00:00:00') TO ('2020-09-28 00:00:00'); -select create_distributed_table('test_index_creation1', 'tenant_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- should be able to create indexes with INCLUDE/WHERE -CREATE INDEX ix_test_index_creation5 ON test_index_creation1 - USING btree(tenant_id, timeperiod) - INCLUDE (field1) WHERE (tenant_id = 100); --- test if indexes are created -SELECT 1 AS created WHERE EXISTS(SELECT * FROM pg_indexes WHERE indexname LIKE '%test_index_creation%'); - created ---------------------------------------------------------------------- - 1 -(1 row) - --- test citus size functions in transaction with modification -CREATE TABLE test_citus_size_func (a int); -SELECT create_distributed_table('test_citus_size_func', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO test_citus_size_func VALUES(1), (2); -BEGIN; - -- DDL with citus_table_size - ALTER TABLE test_citus_size_func ADD COLUMN newcol INT; - SELECT citus_table_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- DDL with citus_relation_size - ALTER TABLE test_citus_size_func ADD COLUMN newcol INT; - SELECT citus_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- DDL with citus_total_relation_size - ALTER TABLE test_citus_size_func ADD COLUMN newcol INT; - SELECT citus_total_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- single shard insert with citus_table_size - INSERT INTO test_citus_size_func VALUES (3); - SELECT citus_table_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- multi shard modification with citus_table_size - INSERT INTO test_citus_size_func SELECT * FROM test_citus_size_func; - SELECT citus_table_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- single shard insert with citus_relation_size - INSERT INTO test_citus_size_func VALUES (3); - SELECT citus_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- multi shard modification with citus_relation_size - INSERT INTO test_citus_size_func SELECT * FROM test_citus_size_func; - SELECT citus_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- single shard insert with citus_total_relation_size - INSERT INTO test_citus_size_func VALUES (3); - SELECT citus_total_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; -BEGIN; - -- multi shard modification with citus_total_relation_size - INSERT INTO test_citus_size_func SELECT * FROM test_citus_size_func; - SELECT citus_total_relation_size('test_citus_size_func'); -ERROR: citus size functions cannot be called in transaction blocks which contain multi-shard data modifications -ROLLBACK; --- we should be able to limit intermediate results -BEGIN; - SET LOCAL citus.max_intermediate_result_size TO 0; - WITH cte_1 AS (SELECT * FROM test OFFSET 0) SELECT * FROM cte_1; -ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 0 kB) -DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. -HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. -ROLLBACK; --- the first cte (cte_1) does not exceed the limit --- but the second (cte_2) exceeds, so we error out -BEGIN; - SET LOCAL citus.max_intermediate_result_size TO '1kB'; - INSERT INTO test SELECT i,i from generate_series(0,1000)i; - -- only pulls 1 row, should not hit the limit - WITH cte_1 AS (SELECT * FROM test LIMIT 1) SELECT count(*) FROM cte_1; - count ---------------------------------------------------------------------- - 1 -(1 row) - - -- cte_1 only pulls 1 row, but cte_2 all rows - WITH cte_1 AS (SELECT * FROM test LIMIT 1), - cte_2 AS (SELECT * FROM test OFFSET 0) - SELECT count(*) FROM cte_1, cte_2; -ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 1 kB) -DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. -HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. -ROLLBACK; --- single shard and multi-shard delete --- inside a transaction block -BEGIN; - DELETE FROM test WHERE y = 5; - INSERT INTO test VALUES (4, 5); - DELETE FROM test WHERE x = 1; - INSERT INTO test VALUES (1, 2); -COMMIT; -CREATE INDEX single_node_i1 ON test(x); -CREATE INDEX single_node_i2 ON test(x,y); -REINDEX SCHEMA single_node; -REINDEX SCHEMA CONCURRENTLY single_node; --- keep one of the indexes --- drop w/wout tx blocks -BEGIN; - DROP INDEX single_node_i2; -ROLLBACK; -DROP INDEX single_node_i2; --- change the schema w/wout TX block -BEGIN; - ALTER TABLE public.another_schema_table SET SCHEMA single_node; -ROLLBACK; -ALTER TABLE public.another_schema_table SET SCHEMA single_node; -BEGIN; - TRUNCATE test; - SELECT * FROM test; - x | y ---------------------------------------------------------------------- -(0 rows) - -ROLLBACK; -VACUUM test; -VACUUM test, test_2; -VACUUM ref, test; -VACUUM ANALYZE test(x); -ANALYZE ref; -ANALYZE test_2; -VACUUM local; -VACUUM local, ref, test, test_2; -VACUUM FULL test, ref; -BEGIN; - ALTER TABLE test ADD COLUMN z INT DEFAULT 66; - SELECT count(*) FROM test WHERE z = 66; - count ---------------------------------------------------------------------- - 5 -(1 row) - -ROLLBACK; --- explain analyze should work on a single node -EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) - SELECT * FROM test; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=5 loops=1) - Task Count: 4 - Tuple data received from nodes: 40 bytes - Tasks Shown: One of 4 - -> Task - Tuple data received from node: 16 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on test_90630506 test (actual rows=2 loops=1) -(8 rows) - --- common utility command -SELECT pg_size_pretty(citus_relation_size('test'::regclass)); - pg_size_pretty ---------------------------------------------------------------------- - 24 kB -(1 row) - --- basic view queries -CREATE VIEW single_node_view AS - SELECT count(*) as cnt FROM test t1 JOIN test t2 USING (x); -SELECT * FROM single_node_view; - cnt ---------------------------------------------------------------------- - 5 -(1 row) - -SELECT * FROM single_node_view, test WHERE test.x = single_node_view.cnt; - cnt | x | y ---------------------------------------------------------------------- - 5 | 5 | 6 -(1 row) - --- copy in/out -BEGIN; - COPY test(x) FROM PROGRAM 'seq 32'; - SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 37 -(1 row) - - COPY (SELECT count(DISTINCT x) FROM test) TO STDOUT; -32 - INSERT INTO test SELECT i,i FROM generate_series(0,100)i; -ROLLBACK; --- master_create_empty_shard on coordinator -BEGIN; -CREATE TABLE append_table (a INT, b INT); -SELECT create_distributed_table('append_table','a','append'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT master_create_empty_shard('append_table'); -NOTICE: Creating placements for the append partitioned tables on the coordinator is not supported, skipping coordinator ... -ERROR: could only create 0 of 1 of required shard replicas -END; --- alter table inside a tx block -BEGIN; - ALTER TABLE test ADD COLUMN z single_node.new_type; - INSERT INTO test VALUES (99, 100, (1, 'onder')::new_type) RETURNING *; - x | y | z ---------------------------------------------------------------------- - 99 | 100 | (1,onder) -(1 row) - -ROLLBACK; --- prepared statements with custom types -PREPARE single_node_prepare_p1(int, int, new_type) AS - INSERT INTO test_2 VALUES ($1, $2, $3); -EXECUTE single_node_prepare_p1(1, 1, (95, 'citus9.5')::new_type); -EXECUTE single_node_prepare_p1(2 ,2, (94, 'citus9.4')::new_type); -EXECUTE single_node_prepare_p1(3 ,2, (93, 'citus9.3')::new_type); -EXECUTE single_node_prepare_p1(4 ,2, (92, 'citus9.2')::new_type); -EXECUTE single_node_prepare_p1(5 ,2, (91, 'citus9.1')::new_type); -EXECUTE single_node_prepare_p1(6 ,2, (90, 'citus9.0')::new_type); -PREPARE use_local_query_cache(int) AS SELECT count(*) FROM test_2 WHERE x = $1; -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -SET client_min_messages TO DEBUG2; --- the 6th execution will go through the planner --- the 7th execution will skip the planner as it uses the cache -EXECUTE use_local_query_cache(1); -DEBUG: Deferred pruning for a fast-path router query -DEBUG: Creating router plan - count ---------------------------------------------------------------------- - 1 -(1 row) - -EXECUTE use_local_query_cache(1); - count ---------------------------------------------------------------------- - 1 -(1 row) - -RESET client_min_messages; --- partitioned table should be fine, adding for completeness -CREATE TABLE collections_list ( - key bigint, - ts timestamptz DEFAULT now(), - collection_id integer, - value numeric, - PRIMARY KEY(key, collection_id) -) PARTITION BY LIST (collection_id ); -SELECT create_distributed_table('collections_list', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE collections_list_0 - PARTITION OF collections_list (key, ts, collection_id, value) - FOR VALUES IN ( 0 ); -CREATE TABLE collections_list_1 - PARTITION OF collections_list (key, ts, collection_id, value) - FOR VALUES IN ( 1 ); -INSERT INTO collections_list SELECT i, '2011-01-01', i % 2, i * i FROM generate_series(0, 100) i; -SELECT count(*) FROM collections_list WHERE key < 10 AND collection_id = 1; - count ---------------------------------------------------------------------- - 5 -(1 row) - -SELECT count(*) FROM collections_list_0 WHERE key < 10 AND collection_id = 1; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM collections_list_1 WHERE key = 11; - count ---------------------------------------------------------------------- - 1 -(1 row) - -ALTER TABLE collections_list DROP COLUMN ts; -SELECT * FROM collections_list, collections_list_0 WHERE collections_list.key=collections_list_0.key ORDER BY 1 DESC,2 DESC,3 DESC,4 DESC LIMIT 1; - key | collection_id | value | key | collection_id | value ---------------------------------------------------------------------- - 100 | 0 | 10000 | 100 | 0 | 10000 -(1 row) - --- test hash distribution using INSERT with generate_series() function -CREATE OR REPLACE FUNCTION part_hashint4_noop(value int4, seed int8) -RETURNS int8 AS $$ -SELECT value + seed; -$$ LANGUAGE SQL IMMUTABLE; -CREATE OPERATOR CLASS part_test_int4_ops -FOR TYPE int4 -USING HASH AS -operator 1 =, -function 2 part_hashint4_noop(int4, int8); -CREATE TABLE hash_parted ( - a int, - b int -) PARTITION BY HASH (a part_test_int4_ops); -CREATE TABLE hpart0 PARTITION OF hash_parted FOR VALUES WITH (modulus 4, remainder 0); -CREATE TABLE hpart1 PARTITION OF hash_parted FOR VALUES WITH (modulus 4, remainder 1); -CREATE TABLE hpart2 PARTITION OF hash_parted FOR VALUES WITH (modulus 4, remainder 2); -CREATE TABLE hpart3 PARTITION OF hash_parted FOR VALUES WITH (modulus 4, remainder 3); --- Disable metadata sync since citus doesn't support distributing --- operator class for now. -SET citus.enable_metadata_sync TO OFF; -SELECT create_distributed_table('hash_parted ', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO hash_parted VALUES (1, generate_series(1, 10)); -SELECT * FROM hash_parted ORDER BY 1, 2; - a | b ---------------------------------------------------------------------- - 1 | 1 - 1 | 2 - 1 | 3 - 1 | 4 - 1 | 5 - 1 | 6 - 1 | 7 - 1 | 8 - 1 | 9 - 1 | 10 -(10 rows) - -ALTER TABLE hash_parted DETACH PARTITION hpart0; -ALTER TABLE hash_parted DETACH PARTITION hpart1; -ALTER TABLE hash_parted DETACH PARTITION hpart2; -ALTER TABLE hash_parted DETACH PARTITION hpart3; -RESET citus.enable_metadata_sync; --- test range partition without creating partitions and inserting with generate_series() --- should error out even in plain PG since no partition of relation "parent_tab" is found for row --- in Citus it errors out because it fails to evaluate partition key in insert -CREATE TABLE parent_tab (id int) PARTITION BY RANGE (id); -SELECT create_distributed_table('parent_tab', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO parent_tab VALUES (generate_series(0, 3)); -ERROR: failed to evaluate partition key in insert -HINT: try using constant values for partition column --- now it should work -CREATE TABLE parent_tab_1_2 PARTITION OF parent_tab FOR VALUES FROM (1) to (2); -ALTER TABLE parent_tab ADD COLUMN b int; -INSERT INTO parent_tab VALUES (1, generate_series(0, 3)); -SELECT * FROM parent_tab ORDER BY 1, 2; - id | b ---------------------------------------------------------------------- - 1 | 0 - 1 | 1 - 1 | 2 - 1 | 3 -(4 rows) - --- make sure that parallel accesses are good -SET citus.force_max_query_parallelization TO ON; -SELECT * FROM test_2 ORDER BY 1 DESC; - x | y | z ---------------------------------------------------------------------- - 6 | 2 | (90,citus9.0) - 5 | 2 | (91,citus9.1) - 4 | 2 | (92,citus9.2) - 3 | 2 | (93,citus9.3) - 2 | 2 | (94,citus9.4) - 1 | 1 | (95,citus9.5) -(6 rows) - -DELETE FROM test_2 WHERE y = 1000 RETURNING *; - x | y | z ---------------------------------------------------------------------- -(0 rows) - -RESET citus.force_max_query_parallelization ; -BEGIN; - INSERT INTO test_2 VALUES (7 ,2, (83, 'citus8.3')::new_type); - SAVEPOINT s1; - INSERT INTO test_2 VALUES (9 ,1, (82, 'citus8.2')::new_type); - SAVEPOINT s2; - ROLLBACK TO SAVEPOINT s1; - SELECT * FROM test_2 WHERE z = (83, 'citus8.3')::new_type OR z = (82, 'citus8.2')::new_type; - x | y | z ---------------------------------------------------------------------- - 7 | 2 | (83,citus8.3) -(1 row) - - RELEASE SAVEPOINT s1; -COMMIT; -SELECT * FROM test_2 WHERE z = (83, 'citus8.3')::new_type OR z = (82, 'citus8.2')::new_type; - x | y | z ---------------------------------------------------------------------- - 7 | 2 | (83,citus8.3) -(1 row) - --- final query is only intermediate result --- we want PG 11/12/13 behave consistently, the CTEs should be MATERIALIZED -WITH cte_1 AS (SELECT * FROM test_2) SELECT * FROM cte_1 ORDER BY 1,2; - x | y | z ---------------------------------------------------------------------- - 1 | 1 | (95,citus9.5) - 2 | 2 | (94,citus9.4) - 3 | 2 | (93,citus9.3) - 4 | 2 | (92,citus9.2) - 5 | 2 | (91,citus9.1) - 6 | 2 | (90,citus9.0) - 7 | 2 | (83,citus8.3) -(7 rows) - --- final query is router query -WITH cte_1 AS (SELECT * FROM test_2) SELECT * FROM cte_1, test_2 WHERE test_2.x = cte_1.x AND test_2.x = 7 ORDER BY 1,2; - x | y | z | x | y | z ---------------------------------------------------------------------- - 7 | 2 | (83,citus8.3) | 7 | 2 | (83,citus8.3) -(1 row) - --- final query is a distributed query -WITH cte_1 AS (SELECT * FROM test_2) SELECT * FROM cte_1, test_2 WHERE test_2.x = cte_1.x AND test_2.y != 2 ORDER BY 1,2; - x | y | z | x | y | z ---------------------------------------------------------------------- - 1 | 1 | (95,citus9.5) | 1 | 1 | (95,citus9.5) -(1 row) - --- query pushdown should work -SELECT - * -FROM - (SELECT x, count(*) FROM test_2 GROUP BY x) as foo, - (SELECT x, count(*) FROM test_2 GROUP BY x) as bar -WHERE - foo.x = bar.x -ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC -LIMIT 1; - x | count | x | count ---------------------------------------------------------------------- - 7 | 1 | 7 | 1 -(1 row) - --- make sure that foreign keys work fine -ALTER TABLE test_2 ADD CONSTRAINT first_pkey PRIMARY KEY (x); -ALTER TABLE test ADD CONSTRAINT foreign_key FOREIGN KEY (x) REFERENCES test_2(x) ON DELETE CASCADE; --- show that delete on test_2 cascades to test -SELECT * FROM test WHERE x = 5; - x | y ---------------------------------------------------------------------- - 5 | 6 -(1 row) - -DELETE FROM test_2 WHERE x = 5; -SELECT * FROM test WHERE x = 5; - x | y ---------------------------------------------------------------------- -(0 rows) - -INSERT INTO test_2 VALUES (5 ,2, (91, 'citus9.1')::new_type); -INSERT INTO test VALUES (5, 6); -INSERT INTO ref VALUES (1, 2), (5, 6), (7, 8); -SELECT count(*) FROM ref; - count ---------------------------------------------------------------------- - 3 -(1 row) - -SELECT * FROM ref ORDER BY a; - a | b ---------------------------------------------------------------------- - 1 | 2 - 5 | 6 - 7 | 8 -(3 rows) - -SELECT * FROM test, ref WHERE x = a ORDER BY x; - x | y | a | b ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 - 5 | 6 | 5 | 6 -(2 rows) - -INSERT INTO local VALUES (1, 2), (3, 4), (7, 8); -SELECT count(*) FROM local; - count ---------------------------------------------------------------------- - 3 -(1 row) - -SELECT * FROM local ORDER BY c; - c | d ---------------------------------------------------------------------- - 1 | 2 - 3 | 4 - 7 | 8 -(3 rows) - -SELECT * FROM ref, local WHERE a = c ORDER BY a; - a | b | c | d ---------------------------------------------------------------------- - 1 | 2 | 1 | 2 - 7 | 8 | 7 | 8 -(2 rows) - --- Check repartition joins are supported -SET citus.enable_repartition_joins TO ON; -SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET citus.enable_single_hash_repartition_joins TO ON; -SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET search_path TO public; -SET citus.enable_single_hash_repartition_joins TO OFF; -SELECT * FROM single_node.test t1, single_node.test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET citus.enable_single_hash_repartition_joins TO ON; -SELECT * FROM single_node.test t1, single_node.test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET search_path TO single_node; -SET citus.task_assignment_policy TO 'round-robin'; -SET citus.enable_single_hash_repartition_joins TO ON; -SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET citus.task_assignment_policy TO 'greedy'; -SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -SET citus.task_assignment_policy TO 'first-replica'; -SELECT * FROM test t1, test t2 WHERE t1.x = t2.y ORDER BY t1.x; - x | y | x | y ---------------------------------------------------------------------- - 2 | 7 | 1 | 2 - 4 | 5 | 3 | 4 - 5 | 6 | 4 | 5 -(3 rows) - -RESET citus.enable_repartition_joins; -RESET citus.enable_single_hash_repartition_joins; --- INSERT SELECT router -BEGIN; -INSERT INTO test(x, y) SELECT x, y FROM test WHERE x = 1; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- INSERT SELECT pushdown -BEGIN; -INSERT INTO test(x, y) SELECT x, y FROM test; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 10 -(1 row) - -ROLLBACK; --- INSERT SELECT analytical query -BEGIN; -INSERT INTO test(x, y) SELECT count(x), max(y) FROM test; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- INSERT SELECT repartition -BEGIN; -INSERT INTO test(x, y) SELECT y, x FROM test; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 10 -(1 row) - -ROLLBACK; --- INSERT SELECT from reference table into distributed -BEGIN; -INSERT INTO test(x, y) SELECT a, b FROM ref; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from local table into distributed -BEGIN; -INSERT INTO test(x, y) SELECT c, d FROM local; -SELECT count(*) from test; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO ref(a, b) SELECT x, y FROM test; -SELECT count(*) from ref; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO ref(a, b) SELECT c, d FROM local; -SELECT count(*) from ref; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO local(c, d) SELECT x, y FROM test; -SELECT count(*) from local; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO local(c, d) SELECT a, b FROM ref; -SELECT count(*) from local; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- Confirm that dummy placements work -SELECT count(*) FROM test WHERE false; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y); - count ---------------------------------------------------------------------- -(0 rows) - --- Confirm that they work with round-robin task assignment policy -SET citus.task_assignment_policy TO 'round-robin'; -SELECT count(*) FROM test WHERE false; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM test WHERE false GROUP BY GROUPING SETS (x,y); - count ---------------------------------------------------------------------- -(0 rows) - -RESET citus.task_assignment_policy; -SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 5 -(1 row) - --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO ref(a, b) SELECT x, y FROM test; -SELECT count(*) from ref; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO ref(a, b) SELECT c, d FROM local; -SELECT count(*) from ref; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO local(c, d) SELECT x, y FROM test; -SELECT count(*) from local; - count ---------------------------------------------------------------------- - 8 -(1 row) - -ROLLBACK; --- INSERT SELECT from distributed table to local table -BEGIN; -INSERT INTO local(c, d) SELECT a, b FROM ref; -SELECT count(*) from local; - count ---------------------------------------------------------------------- - 6 -(1 row) - -ROLLBACK; --- query fails on the shards should be handled --- nicely -SELECT x/0 FROM test; -ERROR: division by zero -CONTEXT: while executing command on localhost:xxxxx --- Add "fake" pg_dist_transaction records and run recovery --- to show that it is recovered --- Temporarily disable automatic 2PC recovery -ALTER SYSTEM SET citus.recover_2pc_interval TO -1; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -BEGIN; -CREATE TABLE should_commit (value int); -PREPARE TRANSACTION 'citus_0_should_commit'; --- zero is the coordinator's group id, so we can hard code it -INSERT INTO pg_dist_transaction VALUES (0, 'citus_0_should_commit'); -SELECT recover_prepared_transactions(); - recover_prepared_transactions ---------------------------------------------------------------------- - 1 -(1 row) - --- the table should be seen -SELECT * FROM should_commit; - value ---------------------------------------------------------------------- -(0 rows) - --- set the original back -ALTER SYSTEM RESET citus.recover_2pc_interval; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -RESET citus.task_executor_type; --- make sure undistribute table works fine -ALTER TABLE test DROP CONSTRAINT foreign_key; -SELECT undistribute_table('test_2'); -NOTICE: creating a new table for single_node.test_2 -NOTICE: moving the data of single_node.test_2 -NOTICE: dropping the old single_node.test_2 -NOTICE: renaming the new table to single_node.test_2 - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -SELECT * FROM pg_dist_partition WHERE logicalrelid = 'test_2'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted ---------------------------------------------------------------------- -(0 rows) - -CREATE TABLE reference_table_1 (col_1 INT UNIQUE, col_2 INT UNIQUE, UNIQUE (col_2, col_1)); -SELECT create_reference_table('reference_table_1'); - create_reference_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE distributed_table_1 (col_1 INT UNIQUE); -SELECT create_distributed_table('distributed_table_1', 'col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE citus_local_table_1 (col_1 INT UNIQUE); -SELECT citus_add_local_table_to_metadata('citus_local_table_1'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE partitioned_table_1 (col_1 INT UNIQUE, col_2 INT) PARTITION BY RANGE (col_1); -CREATE TABLE partitioned_table_1_100_200 PARTITION OF partitioned_table_1 FOR VALUES FROM (100) TO (200); -CREATE TABLE partitioned_table_1_200_300 PARTITION OF partitioned_table_1 FOR VALUES FROM (200) TO (300); -SELECT create_distributed_table('partitioned_table_1', 'col_1'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_1 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); -ALTER TABLE reference_table_1 ADD CONSTRAINT fkey_2 FOREIGN KEY (col_2) REFERENCES reference_table_1(col_1); -ALTER TABLE distributed_table_1 ADD CONSTRAINT fkey_3 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1); -ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_4 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); -ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_5 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); -SELECT undistribute_table('partitioned_table_1', cascade_via_foreign_keys=>true); -NOTICE: converting the partitions of single_node.partitioned_table_1 -NOTICE: creating a new table for single_node.partitioned_table_1 -NOTICE: dropping the old single_node.partitioned_table_1 -NOTICE: renaming the new table to single_node.partitioned_table_1 -NOTICE: creating a new table for single_node.reference_table_1 -NOTICE: moving the data of single_node.reference_table_1 -NOTICE: dropping the old single_node.reference_table_1 -NOTICE: renaming the new table to single_node.reference_table_1 -NOTICE: creating a new table for single_node.distributed_table_1 -NOTICE: moving the data of single_node.distributed_table_1 -NOTICE: dropping the old single_node.distributed_table_1 -NOTICE: renaming the new table to single_node.distributed_table_1 -NOTICE: creating a new table for single_node.citus_local_table_1 -NOTICE: moving the data of single_node.citus_local_table_1 -NOTICE: dropping the old single_node.citus_local_table_1 -NOTICE: renaming the new table to single_node.citus_local_table_1 -NOTICE: creating a new table for single_node.partitioned_table_1_100_200 -NOTICE: moving the data of single_node.partitioned_table_1_100_200 -NOTICE: dropping the old single_node.partitioned_table_1_100_200 -NOTICE: renaming the new table to single_node.partitioned_table_1_100_200 -NOTICE: creating a new table for single_node.partitioned_table_1_200_300 -NOTICE: moving the data of single_node.partitioned_table_1_200_300 -NOTICE: dropping the old single_node.partitioned_table_1_200_300 -NOTICE: renaming the new table to single_node.partitioned_table_1_200_300 - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -CREATE TABLE local_table_1 (col_1 INT UNIQUE); -CREATE TABLE local_table_2 (col_1 INT UNIQUE); -CREATE TABLE local_table_3 (col_1 INT UNIQUE); -ALTER TABLE local_table_2 ADD CONSTRAINT fkey_6 FOREIGN KEY (col_1) REFERENCES local_table_1(col_1); -ALTER TABLE local_table_3 ADD CONSTRAINT fkey_7 FOREIGN KEY (col_1) REFERENCES local_table_1(col_1); -ALTER TABLE local_table_1 ADD CONSTRAINT fkey_8 FOREIGN KEY (col_1) REFERENCES local_table_1(col_1); -SELECT citus_add_local_table_to_metadata('local_table_2', cascade_via_foreign_keys=>true); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - -CREATE PROCEDURE call_delegation(x int) LANGUAGE plpgsql AS $$ -BEGIN - INSERT INTO test (x) VALUES ($1); -END;$$; -SELECT * FROM pg_dist_node; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards ---------------------------------------------------------------------- - 5 | 0 | localhost | 57636 | default | t | t | primary | default | t | t -(1 row) - -SELECT create_distributed_function('call_delegation(int)', '$1', 'test'); - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - -CREATE FUNCTION function_delegation(int) RETURNS void AS $$ -BEGIN -UPDATE test SET y = y + 1 WHERE x < $1; -END; -$$ LANGUAGE plpgsql; -SELECT create_distributed_function('function_delegation(int)', '$1', 'test'); - create_distributed_function ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO DEBUG1; -CALL call_delegation(1); -DEBUG: not pushing down procedure to the same node -SELECT function_delegation(1); -DEBUG: not pushing down function to the same node - function_delegation ---------------------------------------------------------------------- - -(1 row) - -SET client_min_messages TO WARNING; -DROP TABLE test CASCADE; -CREATE OR REPLACE FUNCTION pg_catalog.get_all_active_client_backend_count() - RETURNS bigint - LANGUAGE C STRICT - AS 'citus', $$get_all_active_client_backend_count$$; --- set the cached connections to zero --- and execute a distributed query so that --- we end up with zero cached connections afterwards -ALTER SYSTEM SET citus.max_cached_conns_per_worker TO 0; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - --- disable deadlock detection and re-trigger 2PC recovery --- once more when citus.max_cached_conns_per_worker is zero --- so that we can be sure that the connections established for --- maintanince daemon is closed properly. --- this is to prevent random failures in the tests (otherwise, we --- might see connections established for this operations) -ALTER SYSTEM SET citus.distributed_deadlock_detection_factor TO -1; -ALTER SYSTEM SET citus.recover_2pc_interval TO '1ms'; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(0.1); - pg_sleep ---------------------------------------------------------------------- - -(1 row) - --- now that last 2PC recovery is done, we're good to disable it -ALTER SYSTEM SET citus.recover_2pc_interval TO '-1'; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - --- test alter_distributed_table UDF -CREATE TABLE adt_table (a INT, b INT); -CREATE TABLE adt_col (a INT UNIQUE, b INT); -CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); -SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); -INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); -INSERT INTO adt_ref VALUES (3), (5); -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 4 - adt_ref | distributed | a | 4 - adt_table | distributed | a | 4 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref, adt_table -(1 row) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 6 - adt_ref | distributed | a | 6 - adt_table | distributed | a | 6 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref, adt_table -(1 row) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text LIKE 'adt%'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_col | distributed | a | 6 - adt_ref | distributed | a | 6 - adt_table | distributed | b | 6 -(3 rows) - -SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE table_name::text LIKE 'adt%' GROUP BY colocation_id ORDER BY 1; - Colocation Groups ---------------------------------------------------------------------- - adt_col, adt_ref - adt_table -(2 rows) - -SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; - Referencing Table | Definition ---------------------------------------------------------------------- - adt_col | UNIQUE (a) - adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) -(2 rows) - -SELECT * FROM adt_table ORDER BY 1; - a | b ---------------------------------------------------------------------- - 1 | 2 - 3 | 4 - 5 | 6 -(3 rows) - -SELECT * FROM adt_col ORDER BY 1; - a | b ---------------------------------------------------------------------- - 3 | 4 - 5 | 6 - 7 | 8 -(3 rows) - -SELECT * FROM adt_ref ORDER BY 1; - a ---------------------------------------------------------------------- - 3 - 5 -(2 rows) - --- make sure that COPY (e.g., INSERT .. SELECT) and --- alter_distributed_table works in the same TX -BEGIN; -SET LOCAL citus.enable_local_execution=OFF; -INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; -SELECT alter_distributed_table('adt_table', distribution_column:='a'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -ROLLBACK; -BEGIN; -INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; -SELECT alter_distributed_table('adt_table', distribution_column:='a'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT COUNT(*) FROM adt_table; - count ---------------------------------------------------------------------- - 1003 -(1 row) - -END; -SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables WHERE table_name::text = 'adt_table'; - table_name | citus_table_type | distribution_column | shard_count ---------------------------------------------------------------------- - adt_table | distributed | a | 6 -(1 row) - -\c - - - :master_port --- sometimes Postgres is a little slow to terminate the backends --- even if PGFinish is sent. So, to prevent any flaky tests, sleep -SELECT pg_sleep(0.1); - pg_sleep ---------------------------------------------------------------------- - -(1 row) - --- since max_cached_conns_per_worker == 0 at this point, the --- backend(s) that execute on the shards will be terminated --- so show that there no internal backends -SET search_path TO single_node; -SET citus.next_shard_id TO 90730500; -SELECT count(*) from should_commit; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'citus_internal%'; - count ---------------------------------------------------------------------- - 0 -(1 row) - -SELECT get_all_active_client_backend_count(); - get_all_active_client_backend_count ---------------------------------------------------------------------- - 1 -(1 row) - -BEGIN; - SET LOCAL citus.shard_count TO 32; - SET LOCAL citus.force_max_query_parallelization TO ON; - SET LOCAL citus.enable_local_execution TO false; - CREATE TABLE test (a int); - SET citus.shard_replication_factor TO 1; - SELECT create_distributed_table('test', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - - SELECT count(*) FROM test; - count ---------------------------------------------------------------------- - 0 -(1 row) - - -- now, we should have additional 32 connections - SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'citus_internal%'; - count ---------------------------------------------------------------------- - 32 -(1 row) - - -- single external connection - SELECT get_all_active_client_backend_count(); - get_all_active_client_backend_count ---------------------------------------------------------------------- - 1 -(1 row) - -ROLLBACK; -\c - - - :master_port -SET search_path TO single_node; -SET citus.next_shard_id TO 90830500; --- simulate that even if there is no connection slots --- to connect, Citus can switch to local execution -SET citus.force_max_query_parallelization TO false; -SET citus.log_remote_commands TO ON; -ALTER SYSTEM SET citus.local_shared_pool_size TO -1; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - -SELECT pg_sleep(0.1); - pg_sleep ---------------------------------------------------------------------- - -(1 row) - -SET citus.executor_slow_start_interval TO 10; -SELECT count(*) from another_schema_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630515 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630516 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630517 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630518 another_schema_table WHERE true - count ---------------------------------------------------------------------- - 0 -(1 row) - -UPDATE another_schema_table SET b = b; -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630515 another_schema_table SET b = b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630516 another_schema_table SET b = b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630517 another_schema_table SET b = b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630518 another_schema_table SET b = b --- INSERT .. SELECT pushdown and INSERT .. SELECT via repartitioning --- not that we ignore INSERT .. SELECT via coordinator as it relies on --- COPY command -INSERT INTO another_schema_table SELECT * FROM another_schema_table; -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE (a IS NOT NULL) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE (a IS NOT NULL) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE (a IS NOT NULL) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE (a IS NOT NULL) -INSERT INTO another_schema_table SELECT b::int, a::int FROM another_schema_table; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630515_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630515_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630516_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630516_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630517_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630517_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630518_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630518_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 --- multi-row INSERTs -INSERT INTO another_schema_table VALUES (1,1), (2,2), (3,3), (4,4), (5,5),(6,6),(7,7); -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) VALUES (1,1), (5,5) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) VALUES (3,3), (4,4), (7,7) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) VALUES (6,6) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) VALUES (2,2) --- INSERT..SELECT with re-partitioning when using local execution -BEGIN; -INSERT INTO another_schema_table VALUES (1,100); -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 (a, b) VALUES (1, 100) -INSERT INTO another_schema_table VALUES (2,100); -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 (a, b) VALUES (2, 100) -INSERT INTO another_schema_table SELECT b::int, a::int FROM another_schema_table; -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630515_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630515_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630516_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630516_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630517_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630517_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: SELECT partition_index, 'repartitioned_results_xxxxx_from_90630518_to' || '_' || partition_index::text , rows_written FROM worker_partition_query_result('repartitioned_results_xxxxx_from_90630518_to','SELECT b AS a, a AS b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true',0,'hash','{-2147483648,-1073741824,0,1073741824}'::text[],'{-1073741825,-1,1073741823,2147483647}'::text[],true) WHERE rows_written > 0 -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_90630515_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_90630516_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_90630515_to_2,repartitioned_results_xxxxx_from_90630517_to_2,repartitioned_results_xxxxx_from_90630518_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_results('{repartitioned_results_xxxxx_from_90630518_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(a integer, b integer) -SELECT * FROM another_schema_table WHERE a = 100 ORDER BY b; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE (a OPERATOR(pg_catalog.=) 100) ORDER BY b - a | b ---------------------------------------------------------------------- - 100 | 1 - 100 | 2 -(2 rows) - -ROLLBACK; --- intermediate results -WITH cte_1 AS (SELECT * FROM another_schema_table LIMIT 1000) - SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte_1 - count ---------------------------------------------------------------------- - 7 -(1 row) - --- this is to get ready for the next tests -TRUNCATE another_schema_table; -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE --- copy can use local execution even if there is no connection available -COPY another_schema_table(a) FROM PROGRAM 'seq 32'; -NOTICE: executing the copy locally for shard xxxxx -CONTEXT: COPY another_schema_table, line 1: "1" -NOTICE: executing the copy locally for shard xxxxx -CONTEXT: COPY another_schema_table, line 2: "2" -NOTICE: executing the copy locally for shard xxxxx -CONTEXT: COPY another_schema_table, line 3: "3" -NOTICE: executing the copy locally for shard xxxxx -CONTEXT: COPY another_schema_table, line 6: "6" --- INSERT .. SELECT with co-located intermediate results -SET citus.log_remote_commands to false; -CREATE UNIQUE INDEX another_schema_table_pk ON another_schema_table(a); -SET citus.log_local_commands to true; -INSERT INTO another_schema_table SELECT * FROM another_schema_table LIMIT 10000 ON CONFLICT(a) DO NOTHING; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630515'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630516'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630517'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630518'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING -INSERT INTO another_schema_table SELECT * FROM another_schema_table ORDER BY a LIMIT 10 ON CONFLICT(a) DO UPDATE SET b = EXCLUDED.b + 1 RETURNING *; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true ORDER BY a LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true ORDER BY a LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true ORDER BY a LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true ORDER BY a LIMIT '10'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630515'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO UPDATE SET b = (excluded.b OPERATOR(pg_catalog.+) 1) RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630516'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO UPDATE SET b = (excluded.b OPERATOR(pg_catalog.+) 1) RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630517'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO UPDATE SET b = (excluded.b OPERATOR(pg_catalog.+) 1) RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630518'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO UPDATE SET b = (excluded.b OPERATOR(pg_catalog.+) 1) RETURNING citus_table_alias.a, citus_table_alias.b - a | b ---------------------------------------------------------------------- - 1 | - 2 | - 3 | - 4 | - 5 | - 6 | - 7 | - 8 | - 9 | - 10 | -(10 rows) - --- INSERT .. SELECT with co-located intermediate result for non-binary input -WITH cte_1 AS -(INSERT INTO non_binary_copy_test SELECT * FROM non_binary_copy_test LIMIT 10000 ON CONFLICT (key) DO UPDATE SET value = (0, 'citus0')::new_type RETURNING value) -SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT key, value FROM single_node.non_binary_copy_test_90630519 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value FROM single_node.non_binary_copy_test_90630520 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value FROM single_node.non_binary_copy_test_90630521 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value FROM single_node.non_binary_copy_test_90630522 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630519 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_result('insert_select_XXX_90630519'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.value -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630520 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_result('insert_select_XXX_90630520'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.value -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630521 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_result('insert_select_XXX_90630521'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.value -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630522 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_result('insert_select_XXX_90630522'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.value -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'text'::citus_copy_format) intermediate_result(value single_node.new_type)) cte_1 - count ---------------------------------------------------------------------- - 1001 -(1 row) - --- test with NULL columns -ALTER TABLE non_binary_copy_test ADD COLUMN z INT; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90630519, 'single_node', 'ALTER TABLE non_binary_copy_test ADD COLUMN z integer;') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90630520, 'single_node', 'ALTER TABLE non_binary_copy_test ADD COLUMN z integer;') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90630521, 'single_node', 'ALTER TABLE non_binary_copy_test ADD COLUMN z integer;') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90630522, 'single_node', 'ALTER TABLE non_binary_copy_test ADD COLUMN z integer;') -WITH cte_1 AS -(INSERT INTO non_binary_copy_test SELECT * FROM non_binary_copy_test LIMIT 10000 ON CONFLICT (key) DO UPDATE SET value = (0, 'citus0')::new_type RETURNING z) -SELECT bool_and(z is null) FROM cte_1; -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630519 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630520 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630521 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630522 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630519 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630519'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630520 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630520'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630521 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630521'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630522 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630522'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: SELECT bool_and((z IS NULL)) AS bool_and FROM (SELECT intermediate_result.z FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(z integer)) cte_1 - bool_and ---------------------------------------------------------------------- - t -(1 row) - --- test with type coersion (int -> text) and also NULL values with coersion -WITH cte_1 AS -(INSERT INTO non_binary_copy_test SELECT * FROM non_binary_copy_test LIMIT 10000 ON CONFLICT (key) DO UPDATE SET value = (0, 'citus0')::new_type RETURNING key, z) -SELECT count(DISTINCT key::text), count(DISTINCT z::text) FROM cte_1; -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630519 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630520 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630521 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630522 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630519 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630519'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.key, citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630520 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630520'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.key, citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630521 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630521'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.key, citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630522 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630522'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.key, citus_table_alias.z -NOTICE: executing the command locally: SELECT count(DISTINCT (key)::text) AS count, count(DISTINCT (z)::text) AS count FROM (SELECT intermediate_result.key, intermediate_result.z FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, z integer)) cte_1 - count | count ---------------------------------------------------------------------- - 1001 | 0 -(1 row) - --- test disabling drop and truncate for known shards -SET citus.shard_replication_factor TO 1; -CREATE TABLE test_disabling_drop_and_truncate (a int); -SELECT create_distributed_table('test_disabling_drop_and_truncate', 'a'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830500, 'single_node', 'CREATE TABLE single_node.test_disabling_drop_and_truncate (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830500, 'single_node', 'ALTER TABLE single_node.test_disabling_drop_and_truncate OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830501, 'single_node', 'CREATE TABLE single_node.test_disabling_drop_and_truncate (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830501, 'single_node', 'ALTER TABLE single_node.test_disabling_drop_and_truncate OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830502, 'single_node', 'CREATE TABLE single_node.test_disabling_drop_and_truncate (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830502, 'single_node', 'ALTER TABLE single_node.test_disabling_drop_and_truncate OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830503, 'single_node', 'CREATE TABLE single_node.test_disabling_drop_and_truncate (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830503, 'single_node', 'ALTER TABLE single_node.test_disabling_drop_and_truncate OWNER TO postgres') - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SET citus.enable_manual_changes_to_shards TO off; --- these should error out -DROP TABLE test_disabling_drop_and_truncate_90830500; -ERROR: cannot modify "test_disabling_drop_and_truncate_90830500" because it is a shard of a distributed table -HINT: Use the distributed table or set citus.enable_manual_changes_to_shards to on to modify shards directly -TRUNCATE TABLE test_disabling_drop_and_truncate_90830500; -ERROR: cannot modify "test_disabling_drop_and_truncate_90830500" because it is a shard of a distributed table -HINT: Use the distributed table or set citus.enable_manual_changes_to_shards to on to modify shards directly -RESET citus.enable_manual_changes_to_shards ; --- these should work as expected -TRUNCATE TABLE test_disabling_drop_and_truncate_90830500; -DROP TABLE test_disabling_drop_and_truncate_90830500; -DROP TABLE test_disabling_drop_and_truncate; --- test creating distributed or reference tables from shards -CREATE TABLE test_creating_distributed_relation_table_from_shard (a int); -SELECT create_distributed_table('test_creating_distributed_relation_table_from_shard', 'a'); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830504, 'single_node', 'CREATE TABLE single_node.test_creating_distributed_relation_table_from_shard (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830504, 'single_node', 'ALTER TABLE single_node.test_creating_distributed_relation_table_from_shard OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830505, 'single_node', 'CREATE TABLE single_node.test_creating_distributed_relation_table_from_shard (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830505, 'single_node', 'ALTER TABLE single_node.test_creating_distributed_relation_table_from_shard OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830506, 'single_node', 'CREATE TABLE single_node.test_creating_distributed_relation_table_from_shard (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830506, 'single_node', 'ALTER TABLE single_node.test_creating_distributed_relation_table_from_shard OWNER TO postgres') -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (90830507, 'single_node', 'CREATE TABLE single_node.test_creating_distributed_relation_table_from_shard (a integer) USING heap');SELECT worker_apply_shard_ddl_command (90830507, 'single_node', 'ALTER TABLE single_node.test_creating_distributed_relation_table_from_shard OWNER TO postgres') - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- these should error because shards cannot be used to: --- create distributed table -SELECT create_distributed_table('test_creating_distributed_relation_table_from_shard_90830504', 'a'); -ERROR: relation "test_creating_distributed_relation_table_from_shard_90830504" is a shard relation --- create reference table -SELECT create_reference_table('test_creating_distributed_relation_table_from_shard_90830504'); -ERROR: relation "test_creating_distributed_relation_table_from_shard_90830504" is a shard relation -RESET citus.shard_replication_factor; -DROP TABLE test_creating_distributed_relation_table_from_shard; --- lets flush the copy often to make sure everyhing is fine -SET citus.local_copy_flush_threshold TO 1; -TRUNCATE another_schema_table; -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -NOTICE: executing the command locally: TRUNCATE TABLE single_node.another_schema_table_xxxxx CASCADE -INSERT INTO another_schema_table(a) SELECT i from generate_Series(0,10000)i; -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -NOTICE: executing the copy locally for shard xxxxx -WITH cte_1 AS -(INSERT INTO another_schema_table SELECT * FROM another_schema_table ORDER BY a LIMIT 10000 ON CONFLICT(a) DO NOTHING RETURNING *) -SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true ORDER BY a LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true ORDER BY a LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true ORDER BY a LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true ORDER BY a LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630515 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630515'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630516 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630516'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630517 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630517'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: INSERT INTO single_node.another_schema_table_90630518 AS citus_table_alias (a, b) SELECT a, b FROM read_intermediate_result('insert_select_XXX_90630518'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer) ON CONFLICT(a) DO NOTHING RETURNING citus_table_alias.a, citus_table_alias.b -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte_1 - count ---------------------------------------------------------------------- - 0 -(1 row) - -WITH cte_1 AS -(INSERT INTO non_binary_copy_test SELECT * FROM non_binary_copy_test LIMIT 10000 ON CONFLICT (key) DO UPDATE SET value = (0, 'citus0')::new_type RETURNING z) -SELECT bool_and(z is null) FROM cte_1; -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630519 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630520 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630521 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the command locally: SELECT key, value, z FROM single_node.non_binary_copy_test_90630522 non_binary_copy_test WHERE true LIMIT '10000'::bigint -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the copy locally for colocated file with shard xxxxx -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630519 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630519'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630520 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630520'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630521 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630521'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: INSERT INTO single_node.non_binary_copy_test_90630522 AS citus_table_alias (key, value, z) SELECT key, value, z FROM read_intermediate_result('insert_select_XXX_90630522'::text, 'text'::citus_copy_format) intermediate_result(key integer, value single_node.new_type, z integer) ON CONFLICT(key) DO UPDATE SET value = ROW(0, 'citus0'::text)::single_node.new_type RETURNING citus_table_alias.z -NOTICE: executing the command locally: SELECT bool_and((z IS NULL)) AS bool_and FROM (SELECT intermediate_result.z FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(z integer)) cte_1 - bool_and ---------------------------------------------------------------------- - t -(1 row) - -RESET citus.local_copy_flush_threshold; -RESET citus.local_copy_flush_threshold; -CREATE OR REPLACE FUNCTION coordinated_transaction_should_use_2PC() -RETURNS BOOL LANGUAGE C STRICT VOLATILE AS 'citus', -$$coordinated_transaction_should_use_2PC$$; --- a multi-shard/single-shard select that is failed over to local --- execution doesn't start a 2PC -BEGIN; - SELECT count(*) FROM another_schema_table; -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630515 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630516 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630517 another_schema_table WHERE true -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630518 another_schema_table WHERE true - count ---------------------------------------------------------------------- - 10001 -(1 row) - - SELECT count(*) FROM another_schema_table WHERE a = 1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM single_node.another_schema_table_90630515 another_schema_table WHERE (a OPERATOR(pg_catalog.=) 1) - count ---------------------------------------------------------------------- - 1 -(1 row) - - WITH cte_1 as (SELECT * FROM another_schema_table LIMIT 10) - SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630515 another_schema_table WHERE true LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630516 another_schema_table WHERE true LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630517 another_schema_table WHERE true LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT a, b FROM single_node.another_schema_table_90630518 another_schema_table WHERE true LIMIT '10'::bigint -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte_1 - count ---------------------------------------------------------------------- - 10 -(1 row) - - WITH cte_1 as (SELECT * FROM another_schema_table WHERE a = 1 LIMIT 10) - SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT another_schema_table.a, another_schema_table.b FROM single_node.another_schema_table_90630515 another_schema_table WHERE (another_schema_table.a OPERATOR(pg_catalog.=) 1) LIMIT 10) cte_1 - count ---------------------------------------------------------------------- - 1 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - f -(1 row) - -ROLLBACK; --- same without a transaction block -WITH cte_1 AS (SELECT count(*) as cnt FROM another_schema_table LIMIT 1000), - cte_2 AS (SELECT coordinated_transaction_should_use_2PC() as enabled_2pc) -SELECT cnt, enabled_2pc FROM cte_1, cte_2; -NOTICE: executing the command locally: SELECT count(*) AS cnt FROM single_node.another_schema_table_90630515 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT count(*) AS cnt FROM single_node.another_schema_table_90630516 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT count(*) AS cnt FROM single_node.another_schema_table_90630517 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT count(*) AS cnt FROM single_node.another_schema_table_90630518 another_schema_table WHERE true LIMIT '1000'::bigint -NOTICE: executing the command locally: SELECT cte_1.cnt, cte_2.enabled_2pc FROM (SELECT intermediate_result.cnt FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint)) cte_1, (SELECT intermediate_result.enabled_2pc FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(enabled_2pc boolean)) cte_2 - cnt | enabled_2pc ---------------------------------------------------------------------- - 10001 | f -(1 row) - --- a multi-shard modification that is failed over to local --- execution starts a 2PC -BEGIN; - UPDATE another_schema_table SET b = b + 1; -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630515 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630516 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630517 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630518 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -ROLLBACK; --- a multi-shard modification that is failed over to local --- execution starts a 2PC -BEGIN; - WITH cte_1 AS (UPDATE another_schema_table SET b = b + 1 RETURNING *) - SELECT count(*) FROM cte_1; -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630515 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630516 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630517 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630518 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: SELECT count(*) AS count FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) cte_1 - count ---------------------------------------------------------------------- - 10001 -(1 row) - - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -ROLLBACK; --- same without transaction block -WITH cte_1 AS (UPDATE another_schema_table SET b = b + 1 RETURNING *) -SELECT coordinated_transaction_should_use_2PC(); -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630515 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630516 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630517 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630518 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) RETURNING a, b -NOTICE: executing the command locally: SELECT single_node.coordinated_transaction_should_use_2pc() AS coordinated_transaction_should_use_2pc - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - --- a single-shard modification that is failed over to local --- starts 2PC execution -BEGIN; - UPDATE another_schema_table SET b = b + 1 WHERE a = 1; -NOTICE: executing the command locally: UPDATE single_node.another_schema_table_90630515 another_schema_table SET b = (b OPERATOR(pg_catalog.+) 1) WHERE (a OPERATOR(pg_catalog.=) 1) - SELECT coordinated_transaction_should_use_2PC(); - coordinated_transaction_should_use_2pc ---------------------------------------------------------------------- - t -(1 row) - -ROLLBACK; --- if the local execution is disabled, we cannot failover to --- local execution and the queries would fail -SET citus.enable_local_execution TO false; -SELECT count(*) from another_schema_table; -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; -UPDATE another_schema_table SET b = b; -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; -INSERT INTO another_schema_table SELECT * FROM another_schema_table; -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; -INSERT INTO another_schema_table SELECT b::int, a::int FROM another_schema_table; -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; -WITH cte_1 AS (SELECT * FROM another_schema_table LIMIT 1000) - SELECT count(*) FROM cte_1; -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; -INSERT INTO another_schema_table VALUES (1,1), (2,2), (3,3), (4,4), (5,5),(6,6),(7,7); -ERROR: the total number of connections on the server is more than max_connections(100) -HINT: This command supports local execution. Consider enabling local execution using SET citus.enable_local_execution TO true; --- copy fails if local execution is disabled and there is no connection slot -COPY another_schema_table(a) FROM PROGRAM 'seq 32'; -ERROR: could not find an available connection -HINT: Set citus.max_shared_pool_size TO -1 to let COPY command finish -CONTEXT: COPY another_schema_table, line 1: "1" --- set the values to originals back -ALTER SYSTEM RESET citus.max_cached_conns_per_worker; -ALTER SYSTEM RESET citus.distributed_deadlock_detection_factor; -ALTER SYSTEM RESET citus.recover_2pc_interval; -ALTER SYSTEM RESET citus.distributed_deadlock_detection_factor; -ALTER SYSTEM RESET citus.local_shared_pool_size; -SELECT pg_reload_conf(); - pg_reload_conf ---------------------------------------------------------------------- - t -(1 row) - --- suppress notices -SET client_min_messages TO error; --- cannot remove coordinator since a reference table exists on coordinator and no other worker nodes are added -SELECT 1 FROM master_remove_node('localhost', :master_port); -ERROR: cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx -DETAIL: One of the table(s) that prevents the operation complete successfully is single_node.ref -HINT: To proceed, either drop the tables or use undistribute_table() function to convert them to local tables --- Cleanup -DROP SCHEMA single_node CASCADE; --- Remove the coordinator again -SELECT 1 FROM master_remove_node('localhost', :master_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - --- restart nodeid sequence so that multi_cluster_management still has the same --- nodeids -ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 1; diff --git a/src/test/regress/expected/upgrade_citus_finish_citus_upgrade_1.out b/src/test/regress/expected/upgrade_citus_finish_citus_upgrade_1.out new file mode 100644 index 000000000..99538b839 --- /dev/null +++ b/src/test/regress/expected/upgrade_citus_finish_citus_upgrade_1.out @@ -0,0 +1,41 @@ +-- Citus upgrades are finished by calling a procedure +-- Note that pg_catalog.citus_finish_citus_upgrade() behaves differently +-- when last upgrade citus version is less than 11 +-- so we have two alternative outputs for this test +\set upgrade_test_old_citus_version `echo "$CITUS_OLD_VERSION"` +SELECT substring(:'upgrade_test_old_citus_version', 'v(\d+)\.\d+\.\d+')::int < 11 +AS upgrade_test_old_citus_version_lt_11_0; + upgrade_test_old_citus_version_lt_11_0 +--------------------------------------------------------------------- + f +(1 row) + +-- this is a transactional procedure, so rollback should be fine +BEGIN; + CALL citus_finish_citus_upgrade(); +NOTICE: already at the latest distributed schema version (11.1-1) +ROLLBACK; +-- do the actual job +CALL citus_finish_citus_upgrade(); +NOTICE: already at the latest distributed schema version (11.1-1) +-- show that the upgrade is successfull +SELECT metadata->>'last_upgrade_version' = extversion +FROM pg_dist_node_metadata, pg_extension WHERE extname = 'citus'; + ?column? +--------------------------------------------------------------------- + f +(1 row) + +-- idempotent, should be called multiple times +-- still, do not NOTICE the version as it changes per release +SET client_min_messages TO WARNING; +CALL citus_finish_citus_upgrade(); +-- we should be able to sync metadata in nontransactional way as well +SET citus.metadata_sync_mode TO 'nontransactional'; +SELECT start_metadata_sync_to_all_nodes(); + start_metadata_sync_to_all_nodes +--------------------------------------------------------------------- + t +(1 row) + +RESET citus.metadata_sync_mode; diff --git a/src/test/regress/sql/citus_local_tables_queries.sql b/src/test/regress/sql/citus_local_tables_queries.sql index f80de6c57..7dbb09c6e 100644 --- a/src/test/regress/sql/citus_local_tables_queries.sql +++ b/src/test/regress/sql/citus_local_tables_queries.sql @@ -1,12 +1,6 @@ -- -- CITUS_LOCAL_TABLES_QUERIES -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; \set VERBOSITY terse diff --git a/src/test/regress/sql/columnar_pg15.sql b/src/test/regress/sql/columnar_pg15.sql index 1a0f6afdd..80dff78e8 100644 --- a/src/test/regress/sql/columnar_pg15.sql +++ b/src/test/regress/sql/columnar_pg15.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - CREATE TABLE alter_am(i int); INSERT INTO alter_am SELECT generate_series(1,1000000); diff --git a/src/test/regress/sql/coordinator_shouldhaveshards.sql b/src/test/regress/sql/coordinator_shouldhaveshards.sql index 0365f07c2..6194d3a59 100644 --- a/src/test/regress/sql/coordinator_shouldhaveshards.sql +++ b/src/test/regress/sql/coordinator_shouldhaveshards.sql @@ -3,12 +3,6 @@ -- -- Test queries on a distributed table with shards on the coordinator -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA coordinator_shouldhaveshards; SET search_path TO coordinator_shouldhaveshards; diff --git a/src/test/regress/sql/cte_inline.sql b/src/test/regress/sql/cte_inline.sql index 3f3e14c88..79cbad48b 100644 --- a/src/test/regress/sql/cte_inline.sql +++ b/src/test/regress/sql/cte_inline.sql @@ -1,12 +1,6 @@ -- -- CTE_INLINE -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA cte_inline; SET search_path TO cte_inline; diff --git a/src/test/regress/sql/detect_conn_close.sql b/src/test/regress/sql/detect_conn_close.sql index 56ec9bd1d..b5f91da3b 100644 --- a/src/test/regress/sql/detect_conn_close.sql +++ b/src/test/regress/sql/detect_conn_close.sql @@ -1,13 +1,6 @@ -- -- PG15+ test as WL_SOCKET_CLOSED exposed for PG15+ -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE SCHEMA socket_close; SET search_path TO socket_close; diff --git a/src/test/regress/sql/grant_on_schema_propagation.sql b/src/test/regress/sql/grant_on_schema_propagation.sql index f0bd233a2..ba75834c9 100644 --- a/src/test/regress/sql/grant_on_schema_propagation.sql +++ b/src/test/regress/sql/grant_on_schema_propagation.sql @@ -1,11 +1,6 @@ -- -- GRANT_ON_SCHEMA_PROPAGATION -- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; -- test grants are propagated when the schema is CREATE SCHEMA dist_schema; diff --git a/src/test/regress/sql/insert_select_repartition.sql b/src/test/regress/sql/insert_select_repartition.sql index 30d77f5b8..940d438e8 100644 --- a/src/test/regress/sql/insert_select_repartition.sql +++ b/src/test/regress/sql/insert_select_repartition.sql @@ -1,12 +1,6 @@ -- -- INSERT_SELECT_REPARTITION -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; -- tests behaviour of INSERT INTO ... SELECT with repartitioning CREATE SCHEMA insert_select_repartition; diff --git a/src/test/regress/sql/intermediate_result_pruning.sql b/src/test/regress/sql/intermediate_result_pruning.sql index 0ebe5825c..bcd60c00b 100644 --- a/src/test/regress/sql/intermediate_result_pruning.sql +++ b/src/test/regress/sql/intermediate_result_pruning.sql @@ -1,12 +1,6 @@ -- -- INTERMEDIATE_RESULT_PRUNING -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA intermediate_result_pruning; SET search_path TO intermediate_result_pruning; diff --git a/src/test/regress/sql/issue_5248.sql b/src/test/regress/sql/issue_5248.sql index 2248f1493..f58e5b1a8 100644 --- a/src/test/regress/sql/issue_5248.sql +++ b/src/test/regress/sql/issue_5248.sql @@ -1,11 +1,6 @@ -- -- ISSUE_5248 -- --- This test file has an alternative output because of the change in the --- backup modes of Postgres. Specifically, there is a renaming --- issue: pg_stop_backup PRE PG15 vs pg_backup_stop PG15+ --- The alternative output can be deleted when we drop support for PG14 --- CREATE SCHEMA issue_5248; SET search_path TO issue_5248; @@ -13,10 +8,6 @@ SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; SET citus.next_shard_id TO 3013000; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset - create table countries( id serial primary key , name text @@ -202,11 +193,7 @@ FROM ( ( SELECT utc_offset FROM pg_catalog.pg_timezone_names limit 1 offset 4) limit 91) AS subq_3 -\if :server_version_ge_15 WHERE pg_catalog.pg_backup_stop() > cast(NULL AS record) limit 100; -\else -WHERE pg_catalog.pg_stop_backup() > cast(NULL AS pg_lsn) limit 100; -\endif SET client_min_messages TO WARNING; DROP SCHEMA issue_5248 CASCADE; diff --git a/src/test/regress/sql/local_shard_execution.sql b/src/test/regress/sql/local_shard_execution.sql index 8acbc2978..2845693c9 100644 --- a/src/test/regress/sql/local_shard_execution.sql +++ b/src/test/regress/sql/local_shard_execution.sql @@ -1,12 +1,6 @@ -- -- LOCAL_SHARD_EXECUTION -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA local_shard_execution; SET search_path TO local_shard_execution; diff --git a/src/test/regress/sql/local_shard_execution_replicated.sql b/src/test/regress/sql/local_shard_execution_replicated.sql index d7e4cc064..1c3d264e0 100644 --- a/src/test/regress/sql/local_shard_execution_replicated.sql +++ b/src/test/regress/sql/local_shard_execution_replicated.sql @@ -1,12 +1,6 @@ -- -- LOCAL_SHARD_EXECUTION_REPLICATED -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA local_shard_execution_replicated; SET search_path TO local_shard_execution_replicated; diff --git a/src/test/regress/sql/merge.sql b/src/test/regress/sql/merge.sql index 5316b5233..14dd04e32 100644 --- a/src/test/regress/sql/merge.sql +++ b/src/test/regress/sql/merge.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS merge_schema CASCADE; --MERGE INTO target diff --git a/src/test/regress/sql/merge_arbitrary.sql b/src/test/regress/sql/merge_arbitrary.sql index 6c0a931dc..2d1cab39b 100644 --- a/src/test/regress/sql/merge_arbitrary.sql +++ b/src/test/regress/sql/merge_arbitrary.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - SET search_path TO merge_arbitrary_schema; INSERT INTO target_cj VALUES (1, 'target', 0); INSERT INTO target_cj VALUES (2, 'target', 0); diff --git a/src/test/regress/sql/merge_arbitrary_create.sql b/src/test/regress/sql/merge_arbitrary_create.sql index efa3185da..4255ca307 100644 --- a/src/test/regress/sql/merge_arbitrary_create.sql +++ b/src/test/regress/sql/merge_arbitrary_create.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - DROP SCHEMA IF EXISTS merge_arbitrary_schema CASCADE; CREATE SCHEMA merge_arbitrary_schema; SET search_path TO merge_arbitrary_schema; diff --git a/src/test/regress/sql/merge_partition_tables.sql b/src/test/regress/sql/merge_partition_tables.sql index ab40fd23e..64fb0799e 100644 --- a/src/test/regress/sql/merge_partition_tables.sql +++ b/src/test/regress/sql/merge_partition_tables.sql @@ -1,12 +1,3 @@ - -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/sql/merge_repartition1.sql b/src/test/regress/sql/merge_repartition1.sql index 858f4710c..d0f4b6e56 100644 --- a/src/test/regress/sql/merge_repartition1.sql +++ b/src/test/regress/sql/merge_repartition1.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/sql/merge_repartition2.sql b/src/test/regress/sql/merge_repartition2.sql index 7a4812274..354f0605b 100644 --- a/src/test/regress/sql/merge_repartition2.sql +++ b/src/test/regress/sql/merge_repartition2.sql @@ -1,12 +1,3 @@ - -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- We create two sets of source and target tables, one set in Postgres and -- the other in Citus distributed. We run the _exact_ MERGE SQL on both sets -- and compare the final results of the target tables in Postgres and Citus. diff --git a/src/test/regress/sql/merge_schema_sharding.sql b/src/test/regress/sql/merge_schema_sharding.sql index 8ea947c1c..d7fc0007f 100644 --- a/src/test/regress/sql/merge_schema_sharding.sql +++ b/src/test/regress/sql/merge_schema_sharding.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS schema_shard_table1 CASCADE; DROP SCHEMA IF EXISTS schema_shard_table2 CASCADE; diff --git a/src/test/regress/sql/merge_vcore.sql b/src/test/regress/sql/merge_vcore.sql index 2ab95e874..e34f998c0 100644 --- a/src/test/regress/sql/merge_vcore.sql +++ b/src/test/regress/sql/merge_vcore.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- MERGE command performs a join from data_source to target_table_name DROP SCHEMA IF EXISTS merge_vcore_schema CASCADE; --MERGE INTO target diff --git a/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql b/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql index f5fd653f5..206decaa7 100644 --- a/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql +++ b/src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql @@ -158,14 +158,8 @@ ALTER TABLE AT_AddConstNoName.products DROP CONSTRAINT products_product_no_key; -- Check "ADD UNIQUE NULLS NOT DISTICT" -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 - ALTER TABLE AT_AddConstNoName.products ADD UNIQUE NULLS NOT DISTINCT (product_no, price); ALTER TABLE AT_AddConstNoName.products DROP CONSTRAINT products_product_no_price_key; -\endif -- Check "ADD UNIQUE ... DEFERRABLE" ALTER TABLE AT_AddConstNoName.products ADD UNIQUE(product_no) INCLUDE(price) DEFERRABLE; diff --git a/src/test/regress/sql/multi_deparse_shard_query.sql b/src/test/regress/sql/multi_deparse_shard_query.sql index faffdf862..2bd11c811 100644 --- a/src/test/regress/sql/multi_deparse_shard_query.sql +++ b/src/test/regress/sql/multi_deparse_shard_query.sql @@ -1,12 +1,6 @@ -- -- MULTI_DEPARSE_SHARD_QUERY -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA multi_deparse_shard_query; SET search_path TO multi_deparse_shard_query; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index e0c70fe28..b5b61e329 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -320,14 +320,8 @@ SELECT * FROM multi_extension.print_extension_changes(); -- recreate public schema, and recreate citus_tables in the public schema by default CREATE SCHEMA public; --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 +-- public schema is owned by pg_database_owner role ALTER SCHEMA public OWNER TO pg_database_owner; -\endif GRANT ALL ON SCHEMA public TO public; ALTER EXTENSION citus UPDATE TO '9.5-1'; ALTER EXTENSION citus UPDATE TO '10.0-4'; diff --git a/src/test/regress/sql/multi_insert_select.sql b/src/test/regress/sql/multi_insert_select.sql index b10be8424..b773ce906 100644 --- a/src/test/regress/sql/multi_insert_select.sql +++ b/src/test/regress/sql/multi_insert_select.sql @@ -1,14 +1,9 @@ -- -- MULTI_INSERT_SELECT -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- + CREATE SCHEMA multi_insert_select; SET search_path = multi_insert_select,public; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; SET citus.next_shard_id TO 13300000; SET citus.next_placement_id TO 13300000; diff --git a/src/test/regress/sql/multi_insert_select_conflict.sql b/src/test/regress/sql/multi_insert_select_conflict.sql index cb0ac01f5..06822933c 100644 --- a/src/test/regress/sql/multi_insert_select_conflict.sql +++ b/src/test/regress/sql/multi_insert_select_conflict.sql @@ -1,12 +1,6 @@ -- -- MULTI_INSERT_SELECT_CONFLICT -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA on_conflict; SET search_path TO on_conflict, public; diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql index 1b8043cdd..9c584c1ac 100644 --- a/src/test/regress/sql/multi_metadata_sync.sql +++ b/src/test/regress/sql/multi_metadata_sync.sql @@ -1,11 +1,6 @@ -- -- MULTI_METADATA_SYNC -- --- this test has different output for PG14 compared to PG15 --- In PG15, public schema is owned by pg_database_owner role --- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; -- Tests for metadata snapshot functions, metadata syncing functions and propagation of -- metadata changes to MX tables. diff --git a/src/test/regress/sql/multi_mx_insert_select_repartition.sql b/src/test/regress/sql/multi_mx_insert_select_repartition.sql index 4a9c8c96f..f092a96ba 100644 --- a/src/test/regress/sql/multi_mx_insert_select_repartition.sql +++ b/src/test/regress/sql/multi_mx_insert_select_repartition.sql @@ -3,12 +3,6 @@ -- -- Test behaviour of repartitioned INSERT ... SELECT in MX setup -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA multi_mx_insert_select_repartition; SET search_path TO multi_mx_insert_select_repartition; diff --git a/src/test/regress/sql/mx_coordinator_shouldhaveshards.sql b/src/test/regress/sql/mx_coordinator_shouldhaveshards.sql index 9a892a457..d8d304351 100644 --- a/src/test/regress/sql/mx_coordinator_shouldhaveshards.sql +++ b/src/test/regress/sql/mx_coordinator_shouldhaveshards.sql @@ -1,12 +1,6 @@ -- -- MX_COORDINATOR_SHOULDHAVESHARDS -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA mx_coordinator_shouldhaveshards; SET search_path TO mx_coordinator_shouldhaveshards; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 276092dcd..626acb79d 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1,13 +1,6 @@ -- -- PG15 -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif CREATE SCHEMA pg15; SET search_path TO pg15; diff --git a/src/test/regress/sql/pg15_jsonpath.sql b/src/test/regress/sql/pg15_jsonpath.sql index 1f4077c11..f5c5916a0 100644 --- a/src/test/regress/sql/pg15_jsonpath.sql +++ b/src/test/regress/sql/pg15_jsonpath.sql @@ -2,14 +2,6 @@ -- PG15 jsonpath tests -- Relevant pg commit: e26114c817b610424010cfbe91a743f591246ff1 -- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - CREATE SCHEMA jsonpath; SET search_path TO jsonpath; diff --git a/src/test/regress/sql/pgmerge.sql b/src/test/regress/sql/pgmerge.sql index 69a0210bc..eeeb881d3 100644 --- a/src/test/regress/sql/pgmerge.sql +++ b/src/test/regress/sql/pgmerge.sql @@ -1,11 +1,3 @@ -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -\q -\endif - -- -- MERGE test from PG community (adapted to Citus by converting all tables to Citus local) -- diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 06bdc39fe..06234994f 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -186,18 +186,6 @@ DROP PUBLICATION "pub-all"; DROP PUBLICATION pubpartitioned; DROP PUBLICATION pubnotdistributed; -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 -\gset -\if :server_version_ge_15 -\else -SET client_min_messages TO ERROR; -DROP SCHEMA publication CASCADE; -DROP SCHEMA "publication-1" CASCADE; -DROP SCHEMA citus_schema_1 CASCADE; -\q -\endif - -- recreate a mixed publication CREATE PUBLICATION pubtables FOR TABLE test, "publication-1"."test-pubs", citus_schema_1.test; diff --git a/src/test/regress/sql/single_node.sql b/src/test/regress/sql/single_node.sql index 2bb7c58a3..962f59f79 100644 --- a/src/test/regress/sql/single_node.sql +++ b/src/test/regress/sql/single_node.sql @@ -1,12 +1,6 @@ -- -- SINGLE_NODE -- --- This test file has an alternative output because of the change in the --- display of SQL-standard function's arguments in INSERT/SELECT in PG15. --- The alternative output can be deleted when we drop support for PG14 --- -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15; CREATE SCHEMA single_node; SET search_path TO single_node; From ee76c4423eecfb1e367b965e324f5a147d17f187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 31 May 2024 20:52:17 +0300 Subject: [PATCH 145/155] Updates github checkout actions to v4 (#7611) (cherry picked from commit 3fe22406e62fb40da12a0d91f3ecc0cba81cdb24) --- .github/workflows/build_and_test.yml | 24 +++++++++---------- .github/workflows/codeql.yml | 2 +- .github/workflows/flaky_test_debugging.yml | 6 ++--- .../workflows/packaging-test-pipelines.yml | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 36ba97039..a6edefd72 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -44,7 +44,7 @@ jobs: image: ${{ needs.params.outputs.build_image_name }}:${{ needs.params.outputs.sql_snapshot_pg_version }}${{ needs.params.outputs.image_suffix }} options: --user root steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - name: Check Snapshots run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} @@ -58,7 +58,7 @@ jobs: - name: Check Snapshots run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check C Style @@ -114,7 +114,7 @@ jobs: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" options: --user root steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - name: Expose $PG_MAJOR to Github Env run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV shell: bash @@ -228,7 +228,7 @@ jobs: - params - build steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Run Test run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }} @@ -262,7 +262,7 @@ jobs: - ${{ needs.params.outputs.pg17_version }} parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Test arbitrary configs run: |- @@ -314,7 +314,7 @@ jobs: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" with: pg_major: "${{ env.old_pg_major }}" @@ -354,7 +354,7 @@ jobs: - params - build steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" with: skip_installation: true @@ -421,7 +421,7 @@ jobs: needs: - build steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} @@ -439,7 +439,7 @@ jobs: needs: - build steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} @@ -458,7 +458,7 @@ jobs: outputs: json: ${{ steps.parallelization.outputs.json }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/parallelization" id: parallelization with: @@ -471,7 +471,7 @@ jobs: outputs: tests: ${{ steps.detect-regression-tests.outputs.tests }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Detect regression tests need to be ran @@ -506,7 +506,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4.1.8 - uses: "./.github/actions/setup_extension" - name: Run minimal tests diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6478abf4b..027f5a048 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml index 574cb7813..04ec1837e 100644 --- a/.github/workflows/flaky_test_debugging.yml +++ b/.github/workflows/flaky_test_debugging.yml @@ -28,7 +28,7 @@ jobs: image: ${{ vars.build_image_name }}:${{ vars.pg15_version }}${{ vars.image_suffix }} options: --user root steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - name: Configure, Build, and Install run: | echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV @@ -46,7 +46,7 @@ jobs: outputs: json: ${{ steps.parallelization.outputs.json }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/parallelization" id: parallelization with: @@ -67,7 +67,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Run minimal tests run: |- diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 34af216c0..dfef4aadf 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -128,7 +128,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set pg_config path and python parameters for deb based distros run: | From 0b4896f7b47c74ea1cf2c38e36f3b7f9c3f43b75 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Tue, 4 Feb 2025 11:10:37 +0300 Subject: [PATCH 146/155] Revert "Release RowExclusiveLock on pg_dist_transaction as soon as remote xacts are recovered" This reverts commit 684b4c6b9666954bdf13c9eb597390eb71a66d48. --- .../transaction/transaction_recovery.c | 18 +----------------- .../isolation_create_restore_point.out | 7 ++++--- .../spec/isolation_create_restore_point.spec | 5 +---- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index c114a2ed3..f25823b30 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -347,23 +347,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode) } systable_endscan(scanDescriptor); - - /* - * Here we release the lock on pg_dist_transaction while closing it to avoid - * deadlocks that might occur because of trying to acquire a lock on - * pg_dist_authinfo while holding a lock on pg_dist_transaction. Such a scenario - * can only cause a deadlock if another transaction is trying to acquire a strong - * lock on pg_dist_transaction while holding a lock on pg_dist_authinfo. As of - * today, we (implicitly) acquire a strong lock on pg_dist_transaction only when - * upgrading Citus to 11.3-1 and this happens when creating a REPLICA IDENTITY on - * pg_dist_transaction. - * - * And reglardless of the code-path we are in, it should be okay to release the - * lock now because all we do after this point is to abort the prepared - * transactions that are not part of an in-progress distributed transaction and - * releasing the lock before doing so should be just fine. - */ - table_close(pgDistTransaction, RowExclusiveLock); + table_close(pgDistTransaction, NoLock); if (!recoveryFailed) { diff --git a/src/test/regress/expected/isolation_create_restore_point.out b/src/test/regress/expected/isolation_create_restore_point.out index dce15a35d..3b1bdf9eb 100644 --- a/src/test/regress/expected/isolation_create_restore_point.out +++ b/src/test/regress/expected/isolation_create_restore_point.out @@ -147,15 +147,16 @@ recover_prepared_transactions step s2-create-restore: SELECT 1 FROM citus_create_restore_point('citus-test'); + +step s1-commit: + COMMIT; +step s2-create-restore: <... completed> ?column? --------------------------------------------------------------------- 1 (1 row) -step s1-commit: - COMMIT; - starting permutation: s1-begin s1-drop s2-create-restore s1-commit create_reference_table diff --git a/src/test/regress/spec/isolation_create_restore_point.spec b/src/test/regress/spec/isolation_create_restore_point.spec index c62a64a44..2cdc66f85 100644 --- a/src/test/regress/spec/isolation_create_restore_point.spec +++ b/src/test/regress/spec/isolation_create_restore_point.spec @@ -154,10 +154,7 @@ permutation "s1-begin" "s1-ddl" "s2-create-restore" "s1-commit" // verify that citus_create_restore_point is not blocked by concurrent COPY (only commit) permutation "s1-begin" "s1-copy" "s2-create-restore" "s1-commit" -// verify that citus_create_restore_point is partially blocked by concurrent recover_prepared_transactions. -// In the test output, we won't be able to explicitly observe this since -// recover_prepared_transactions unblocks citus_create_restore_point after in-progress prepared transactions -// are recovered. +// verify that citus_create_restore_point is blocked by concurrent recover_prepared_transactions permutation "s1-begin" "s1-recover" "s2-create-restore" "s1-commit" // verify that citus_create_restore_point is blocked by concurrent DROP TABLE From b6b73e2f4c66211c23e167e094256264992c8bd4 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Tue, 4 Feb 2025 11:45:24 +0300 Subject: [PATCH 147/155] Disable 2PC recovery while executing ALTER EXTENSION cmd during Citus upgrade tests --- .../citus_tests/upgrade/citus_upgrade_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py index 1ab448031..c25a34482 100755 --- a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py +++ b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py @@ -62,10 +62,16 @@ def run_citus_upgrade_tests(config, before_upgrade_schedule, after_upgrade_sched install_citus(config.post_tar_path) + # disable 2pc recovery for all nodes to work around https://github.com/citusdata/citus/issues/7875 + disable_2pc_recovery_for_all_nodes(config.bindir, config) + restart_databases(config.bindir, config.datadir, config.mixed_mode, config) run_alter_citus(config.bindir, config.mixed_mode, config) verify_upgrade(config, config.mixed_mode, config.node_name_to_ports.values()) + # re-enable 2pc recovery for all nodes + enable_2pc_recovery_for_all_nodes(config.bindir, config) + run_test_on_coordinator(config, after_upgrade_schedule) remove_citus(config.post_tar_path) @@ -146,6 +152,18 @@ def restart_database(pg_path, abs_data_path, node_name, node_ports, logfile_pref subprocess.run(command, check=True) +def disable_2pc_recovery_for_all_nodes(pg_path, config): + for port in config.node_name_to_ports.values(): + utils.psql(pg_path, port, "ALTER SYSTEM SET citus.recover_2pc_interval TO -1;") + utils.psql(pg_path, port, "SELECT pg_reload_conf();") + + +def enable_2pc_recovery_for_all_nodes(pg_path, config): + for port in config.node_name_to_ports.values(): + utils.psql(pg_path, port, "ALTER SYSTEM RESET citus.recover_2pc_interval;") + utils.psql(pg_path, port, "SELECT pg_reload_conf();") + + def run_alter_citus(pg_path, mixed_mode, config): for port in config.node_name_to_ports.values(): if mixed_mode and port in ( From e5a1c171341a4b3c3d36451696d984631acac683 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:15:05 +0300 Subject: [PATCH 148/155] Bump Citus version to 13.0.1 (#7872) --- configure | 18 +++++++++--------- configure.ac | 2 +- src/test/regress/citus_tests/common.py | 2 +- src/test/regress/expected/multi_extension.out | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/configure b/configure index d2f4060a7..07abd5c52 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 13.0.0. +# Generated by GNU Autoconf 2.69 for Citus 13.0.1. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='13.0.0' -PACKAGE_STRING='Citus 13.0.0' +PACKAGE_VERSION='13.0.1' +PACKAGE_STRING='Citus 13.0.1' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 13.0.0 to adapt to many kinds of systems. +\`configure' configures Citus 13.0.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 13.0.0:";; + short | recursive ) echo "Configuration of Citus 13.0.1:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 13.0.0 +Citus configure 13.0.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 13.0.0, which was +It was created by Citus $as_me 13.0.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 13.0.0, which was +This file was extended by Citus $as_me 13.0.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 13.0.0 +Citus config.status 13.0.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 0d79adce1..23448251b 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [13.0.0]) +AC_INIT([Citus], [13.0.1]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 3460afc1b..b06d56382 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -93,7 +93,7 @@ OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 14: "10.2.0", 15: "11.1.5", 16: "12.1.5", - 17: "13.0.0", + 17: "13.0.1", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION] diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 128872df5..f05f56d67 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1445,7 +1445,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; SHOW citus.version; citus.version --------------------------------------------------------------------- - 13.0.0 + 13.0.1 (1 row) -- ensure no unexpected objects were created outside pg_catalog From d28a5eae6c78935313824d319480632783d48d10 Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:55:35 +0300 Subject: [PATCH 149/155] Changelog entries for v13.0.1 (#7873) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4410d439..f6c45a042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### citus v13.0.1 (February 4th, 2025) ### + +* Drops support for PostgreSQL 14 (#7753) + ### citus v13.0.0 (January 22, 2025) ### * Adds support for PostgreSQL 17 (#7699, #7661) From 8f3d9deffebfdfdc11371244a206767fe5476484 Mon Sep 17 00:00:00 2001 From: Colm Date: Tue, 18 Feb 2025 12:41:34 +0000 Subject: [PATCH 150/155] [Bug Fix] SEGV on query with Left Outer Join (#7787) (#7901) DESCRIPTION: Fixes a crash in left outer joins that can happen when there is an an aggregate on a column from the inner side of the join. Fix the SEGV seen in #7787 and #7899; it occurs because a column in the targetlist of a worker subquery can contain a non-empty varnullingrels field if the column is from the inner side of a left outer join. The issue can also occur with the columns in the HAVING clause, and this is also tested in the fix. The issue was triggered by the introduction of the varnullingrels to Vars in Postgres 16 (2489d76c) There is a related issue, #7705, where a non-empty varnullingrels was incorrectly copied into the query tree for the combine query. Here, a non-empty varnullingrels field of a var is incorrectly copied into the query tree for a worker subquery. The regress file from #7705 is used (and renamed) to also test this (#7787). An alternative test output file is required for Postgres 15 because of an optimization to DISTINCT in Postgres 16 (1349d2790bf). --- .../planner/query_pushdown_planning.c | 10 + .../expected/multi_outer_join_columns.out | 426 ++++++++++++++++++ ...705.out => multi_outer_join_columns_1.out} | 220 +++++++-- src/test/regress/multi_schedule | 4 +- ..._7705.sql => multi_outer_join_columns.sql} | 51 ++- 5 files changed, 677 insertions(+), 34 deletions(-) create mode 100644 src/test/regress/expected/multi_outer_join_columns.out rename src/test/regress/expected/{issue_7705.out => multi_outer_join_columns_1.out} (51%) rename src/test/regress/sql/{issue_7705.sql => multi_outer_join_columns.sql} (62%) diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 20175eac3..5317e578d 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -2097,6 +2097,16 @@ CreateSubqueryTargetListAndAdjustVars(List *columnList) */ column->varno = 1; column->varattno = resNo; + + /* + * 1 subquery means there is one range table entry so with Postgres 16+ we need + * to ensure that column's varnullingrels - the set of join rels that can null + * the var - is empty. Otherwise, when given the query, the Postgres planner + * may attempt to access a non-existent range table and segfault, as in #7787. + */ +#if PG_VERSION_NUM >= PG_VERSION_16 + column->varnullingrels = NULL; +#endif } return subqueryTargetEntryList; diff --git a/src/test/regress/expected/multi_outer_join_columns.out b/src/test/regress/expected/multi_outer_join_columns.out new file mode 100644 index 000000000..79527f7c6 --- /dev/null +++ b/src/test/regress/expected/multi_outer_join_columns.out @@ -0,0 +1,426 @@ +--- Test for verifying that column references (var nodes) in targets that cannot be pushed down +--- do not cause issues for the postgres planner, in particular postgres versions 16+, where the +--- varnullingrels field of a VAR node may contain relids of join relations that can make the var +--- NULL; in a rewritten distributed query without a join such relids do not have a meaning. +-- This test has an alternative goldfile because of the following feature in Postgres 16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + t +(1 row) + +CREATE SCHEMA outer_join_columns_testing; +SET search_path to 'outer_join_columns_testing'; +SET citus.next_shard_id TO 30070000; +SET citus.shard_replication_factor TO 1; +SET citus.enable_local_execution TO ON; +CREATE TABLE t1 (id INT PRIMARY KEY); +INSERT INTO t1 VALUES (1), (2); +CREATE TABLE t2 (id INT, account_id INT, a2 INT, PRIMARY KEY(id, account_id)); +INSERT INTO t2 VALUES (3, 1, 10), (4, 2, 20), (5, 1, NULL); +SELECT create_distributed_table('t1', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$outer_join_columns_testing.t1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('t2', 'account_id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$outer_join_columns_testing.t2$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- Test the issue seen in #7705; a target expression with +-- a window function that cannot be pushed down because the +-- partion by is not on the distribution column also includes +-- a column from the inner side of a left outer join, which +-- produces a non-empty varnullingrels set in PG 16 (and higher) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | 10 + 2 | 20 + 1 | +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | 10 + 2 | 20 + 1 | +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t2_30070004 t2 RIGHT JOIN outer_join_columns_testing.t1_30070000 t1 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + id | max +--------------------------------------------------------------------- + 1 | + 1 | 10 + 2 | 20 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate + Output: remote_scan.id, (max(remote_scan.max) OVER (?)), remote_scan.worker_column_3 + Group Key: remote_scan.id, max(remote_scan.max) OVER (?) + -> WindowAgg + Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 + -> Sort + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Sort Key: remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Output: remote_scan.worker_column_3, remote_scan.id, remote_scan.max + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(25 rows) + +CREATE SEQUENCE test_seq START 101; +CREATE OR REPLACE FUNCTION TEST_F(int) returns INT language sql stable as $$ select $1 + 42; $$ ; +-- Issue #7705 also occurs if a target expression includes a column +-- of a distributed table that is on the inner side of a left outer +-- join and a call to nextval(), because nextval() cannot be pushed +-- down, and must be run on the coordinator +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + id | test_f +--------------------------------------------------------------------- + 1 | 153 + 1 | + 2 | 165 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Result + Output: remote_scan.id, ((remote_scan.test_f + (nextval('test_seq'::regclass))::integer) + 42) + -> Sort + Output: remote_scan.id, remote_scan.test_f + Sort Key: remote_scan.id + -> Custom Scan (Citus Adaptive) + Output: remote_scan.id, remote_scan.test_f + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS test_f FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +SELECT t1.id, CASE nextval('test_seq') % 2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + id | case +--------------------------------------------------------------------- + 1 | 10 + 1 | 1 + 2 | 20 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT t1.id, CASE nextval('test_seq') %2 = 0 WHEN true THEN t2.a2 ELSE 1 END +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +ORDER BY t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Result + Output: remote_scan.id, CASE ((nextval('test_seq'::regclass) % '2'::bigint) = 0) WHEN CASE_TEST_EXPR THEN remote_scan."case" ELSE 1 END + -> Sort + Output: remote_scan.id, remote_scan."case" + Sort Key: remote_scan.id + -> Custom Scan (Citus Adaptive) + Output: remote_scan.id, remote_scan."case" + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS id, worker_column_2 AS "case" FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Node: host=localhost port=xxxxx dbname=regression + -> Hash Right Join + Output: t1.id, t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +-- Issue #7787: count distinct of a column from the inner side of a +-- left outer join will have a non-empty varnullingrels in the query +-- tree returned by Postgres 16+, so ensure this is not reflected in +-- the worker subquery constructed by Citus; it has just one relation, +-- for the pushed down subquery. +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + -> Sort + Output: remote_scan.count + Sort Key: remote_scan.count + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(25 rows) + +-- Issue #7787 also occurs with a HAVING clause +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: remote_scan."?column?" + Filter: (count(DISTINCT remote_scan.worker_column_2) > 1) + -> Sort + Output: remote_scan."?column?", remote_scan.worker_column_2 + Sort Key: remote_scan.worker_column_2 + -> Custom Scan (Citus Adaptive) + Output: remote_scan."?column?", remote_scan.worker_column_2 + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT 1, worker_column_1 AS worker_column_2 FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: 1, t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(26 rows) + +-- Check right outer join +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + -> Sort + Output: remote_scan.count + Sort Key: remote_scan.count + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t2_30070004 t2 RIGHT JOIN outer_join_columns_testing.t1_30070000 t1 ON ((t2.account_id OPERATOR(pg_catalog.=) t1.id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(25 rows) + +-- Check both count distinct and having clause +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + Filter: (count(DISTINCT remote_scan.worker_column_2) > 1) + -> Sort + Output: remote_scan.count, remote_scan.worker_column_2 + Sort Key: remote_scan.count + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count, remote_scan.worker_column_2 + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count, worker_column_2 FROM (SELECT t2.a2 AS worker_column_1, t2.id AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1, worker_column_2 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2, t2.id + Group Key: t2.a2, t2.id + -> Hash Right Join + Output: t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(26 rows) + +--- cleanup +\set VERBOSITY TERSE +DROP SCHEMA outer_join_columns_testing CASCADE; +NOTICE: drop cascades to 4 other objects +RESET all; diff --git a/src/test/regress/expected/issue_7705.out b/src/test/regress/expected/multi_outer_join_columns_1.out similarity index 51% rename from src/test/regress/expected/issue_7705.out rename to src/test/regress/expected/multi_outer_join_columns_1.out index 20b078226..10dc6773d 100644 --- a/src/test/regress/expected/issue_7705.out +++ b/src/test/regress/expected/multi_outer_join_columns_1.out @@ -2,10 +2,18 @@ --- do not cause issues for the postgres planner, in particular postgres versions 16+, where the --- varnullingrels field of a VAR node may contain relids of join relations that can make the var --- NULL; in a rewritten distributed query without a join such relids do not have a meaning. ---- Issue #7705: [SEGFAULT] Querying distributed tables with window partition causes segmentation fault ---- https://github.com/citusdata/citus/issues/7705 -CREATE SCHEMA issue_7705; -SET search_path to 'issue_7705'; +-- This test has an alternative goldfile because of the following feature in Postgres 16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + f +(1 row) + +CREATE SCHEMA outer_join_columns_testing; +SET search_path to 'outer_join_columns_testing'; SET citus.next_shard_id TO 30070000; SET citus.shard_replication_factor TO 1; SET citus.enable_local_execution TO ON; @@ -17,7 +25,7 @@ SELECT create_distributed_table('t1', 'id'); NOTICE: Copying data from local table... NOTICE: copying the data has completed DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$issue_7705.t1$$) +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$outer_join_columns_testing.t1$$) create_distributed_table --------------------------------------------------------------------- @@ -27,7 +35,7 @@ SELECT create_distributed_table('t2', 'account_id'); NOTICE: Copying data from local table... NOTICE: copying the data has completed DETAIL: The local data in the table is no longer visible, but is still on disk. -HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$issue_7705.t2$$) +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$outer_join_columns_testing.t2$$) create_distributed_table --------------------------------------------------------------------- @@ -50,7 +58,7 @@ FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- WindowAgg Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 @@ -62,17 +70,17 @@ FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery Node: host=localhost port=xxxxx dbname=regression -> Hash Right Join Output: t1.id, t2.a2, t2.id Inner Unique: true Hash Cond: (t2.account_id = t1.id) - -> Seq Scan on issue_7705.t2_30070004 t2 + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 Output: t2.id, t2.account_id, t2.a2 -> Hash Output: t1.id - -> Seq Scan on issue_7705.t1_30070000 t1 + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 Output: t1.id (22 rows) @@ -88,7 +96,7 @@ FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) SELECT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- WindowAgg Output: remote_scan.id, max(remote_scan.max) OVER (?), remote_scan.worker_column_3 @@ -100,17 +108,17 @@ FROM t2 RIGHT OUTER JOIN t1 ON t1.id = t2.account_id; Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t2_30070004 t2 RIGHT JOIN issue_7705.t1_30070000 t1 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t2_30070004 t2 RIGHT JOIN outer_join_columns_testing.t1_30070000 t1 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery Node: host=localhost port=xxxxx dbname=regression -> Hash Right Join Output: t1.id, t2.a2, t2.id Inner Unique: true Hash Cond: (t2.account_id = t1.id) - -> Seq Scan on issue_7705.t2_30070004 t2 + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 Output: t2.id, t2.account_id, t2.a2 -> Hash Output: t1.id - -> Seq Scan on issue_7705.t1_30070000 t1 + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 Output: t1.id (22 rows) @@ -126,7 +134,7 @@ FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) SELECT DISTINCT t1.id, MAX(t2.a2) OVER (PARTITION BY t2.id) FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- HashAggregate Output: remote_scan.id, (max(remote_scan.max) OVER (?)), remote_scan.worker_column_3 @@ -141,17 +149,17 @@ FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Query: SELECT worker_column_1 AS id, worker_column_2 AS max, worker_column_3 FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2, t2.id AS worker_column_3 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery Node: host=localhost port=xxxxx dbname=regression -> Hash Right Join Output: t1.id, t2.a2, t2.id Inner Unique: true Hash Cond: (t2.account_id = t1.id) - -> Seq Scan on issue_7705.t2_30070004 t2 + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 Output: t2.id, t2.account_id, t2.a2 -> Hash Output: t1.id - -> Seq Scan on issue_7705.t1_30070000 t1 + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 Output: t1.id (25 rows) @@ -175,7 +183,7 @@ EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) SELECT t1.id, TEST_F(t2.a2 + nextval('test_seq') :: int) FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id ORDER BY t1.id; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Result Output: remote_scan.id, ((remote_scan.test_f + (nextval('test_seq'::regclass))::integer) + 42) @@ -187,17 +195,17 @@ ORDER BY t1.id; Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS id, worker_column_2 AS test_f FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Query: SELECT worker_column_1 AS id, worker_column_2 AS test_f FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery Node: host=localhost port=xxxxx dbname=regression -> Hash Right Join Output: t1.id, t2.a2 Inner Unique: true Hash Cond: (t2.account_id = t1.id) - -> Seq Scan on issue_7705.t2_30070004 t2 + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 Output: t2.id, t2.account_id, t2.a2 -> Hash Output: t1.id - -> Seq Scan on issue_7705.t1_30070000 t1 + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 Output: t1.id (22 rows) @@ -215,7 +223,7 @@ EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) SELECT t1.id, CASE nextval('test_seq') %2 = 0 WHEN true THEN t2.a2 ELSE 1 END FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id ORDER BY t1.id; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Result Output: remote_scan.id, CASE ((nextval('test_seq'::regclass) % '2'::bigint) = 0) WHEN CASE_TEST_EXPR THEN remote_scan."case" ELSE 1 END @@ -227,22 +235,180 @@ ORDER BY t1.id; Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS id, worker_column_2 AS "case" FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (issue_7705.t1_30070000 t1 LEFT JOIN issue_7705.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery + Query: SELECT worker_column_1 AS id, worker_column_2 AS "case" FROM (SELECT t1.id AS worker_column_1, t2.a2 AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery Node: host=localhost port=xxxxx dbname=regression -> Hash Right Join Output: t1.id, t2.a2 Inner Unique: true Hash Cond: (t2.account_id = t1.id) - -> Seq Scan on issue_7705.t2_30070004 t2 + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 Output: t2.id, t2.account_id, t2.a2 -> Hash Output: t1.id - -> Seq Scan on issue_7705.t1_30070000 t1 + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 Output: t1.id (22 rows) +-- Issue #7787: count distinct of a column from the inner side of a +-- left outer join will have a non-empty varnullingrels in the query +-- tree returned by Postgres 16+, so ensure this is not reflected in +-- the worker subquery constructed by Citus; it has just one relation, +-- for the pushed down subquery. +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +-- Issue #7787 also occurs with a HAVING clause +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: remote_scan."?column?" + Filter: (count(DISTINCT remote_scan.worker_column_2) > 1) + -> Custom Scan (Citus Adaptive) + Output: remote_scan."?column?", remote_scan.worker_column_2 + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT 1, worker_column_1 AS worker_column_2 FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: 1, t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(23 rows) + +-- Check right outer join +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count FROM (SELECT t2.a2 AS worker_column_1 FROM (outer_join_columns_testing.t2_30070004 t2 RIGHT JOIN outer_join_columns_testing.t1_30070000 t1 ON ((t2.account_id OPERATOR(pg_catalog.=) t1.id)))) worker_subquery GROUP BY worker_column_1 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2 + Group Key: t2.a2 + -> Hash Right Join + Output: t2.a2 + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(22 rows) + +-- Check both count distinct and having clause +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; + count +--------------------------------------------------------------------- + 2 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count) + Filter: (count(DISTINCT remote_scan.worker_column_2) > 1) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count, remote_scan.worker_column_2 + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT worker_column_1 AS count, worker_column_2 FROM (SELECT t2.a2 AS worker_column_1, t2.id AS worker_column_2 FROM (outer_join_columns_testing.t1_30070000 t1 LEFT JOIN outer_join_columns_testing.t2_30070004 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.account_id)))) worker_subquery GROUP BY worker_column_1, worker_column_2 + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: t2.a2, t2.id + Group Key: t2.a2, t2.id + -> Hash Right Join + Output: t2.a2, t2.id + Inner Unique: true + Hash Cond: (t2.account_id = t1.id) + -> Seq Scan on outer_join_columns_testing.t2_30070004 t2 + Output: t2.id, t2.account_id, t2.a2 + -> Hash + Output: t1.id + -> Seq Scan on outer_join_columns_testing.t1_30070000 t1 + Output: t1.id +(23 rows) + --- cleanup \set VERBOSITY TERSE -DROP SCHEMA issue_7705 CASCADE; +DROP SCHEMA outer_join_columns_testing CASCADE; NOTICE: drop cascades to 4 other objects RESET all; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index ae9d9ac6c..db2794171 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -84,7 +84,7 @@ test: forcedelegation_functions system_queries # this should be run alone as it gets too many clients test: join_pushdown test: multi_subquery_union multi_subquery_in_where_clause multi_subquery_misc statement_cancel_error_message -test: multi_agg_distinct multi_limit_clause_approximate multi_outer_join_reference multi_single_relation_subquery multi_prepare_plsql set_role_in_transaction +test: multi_agg_distinct multi_limit_clause_approximate multi_outer_join_reference multi_outer_join_columns multi_single_relation_subquery multi_prepare_plsql set_role_in_transaction test: multi_reference_table multi_select_for_update relation_access_tracking pg13_with_ties test: custom_aggregate_support aggregate_support tdigest_aggregate_support test: multi_average_expression multi_working_columns multi_having_pushdown having_subquery @@ -103,7 +103,7 @@ test: multi_dropped_column_aliases foreign_key_restriction_enforcement test: binary_protocol test: alter_table_set_access_method test: alter_distributed_table -test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 issue_7705 +test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 test: object_propagation_debug test: undistribute_table test: run_command_on_all_nodes diff --git a/src/test/regress/sql/issue_7705.sql b/src/test/regress/sql/multi_outer_join_columns.sql similarity index 62% rename from src/test/regress/sql/issue_7705.sql rename to src/test/regress/sql/multi_outer_join_columns.sql index 950933017..eec111cb5 100644 --- a/src/test/regress/sql/issue_7705.sql +++ b/src/test/regress/sql/multi_outer_join_columns.sql @@ -2,11 +2,16 @@ --- do not cause issues for the postgres planner, in particular postgres versions 16+, where the --- varnullingrels field of a VAR node may contain relids of join relations that can make the var --- NULL; in a rewritten distributed query without a join such relids do not have a meaning. ---- Issue #7705: [SEGFAULT] Querying distributed tables with window partition causes segmentation fault ---- https://github.com/citusdata/citus/issues/7705 -CREATE SCHEMA issue_7705; -SET search_path to 'issue_7705'; +-- This test has an alternative goldfile because of the following feature in Postgres 16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- + +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + +CREATE SCHEMA outer_join_columns_testing; +SET search_path to 'outer_join_columns_testing'; SET citus.next_shard_id TO 30070000; SET citus.shard_replication_factor TO 1; SET citus.enable_local_execution TO ON; @@ -66,7 +71,43 @@ SELECT t1.id, CASE nextval('test_seq') %2 = 0 WHEN true THEN t2.a2 ELSE 1 END FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id ORDER BY t1.id; +-- Issue #7787: count distinct of a column from the inner side of a +-- left outer join will have a non-empty varnullingrels in the query +-- tree returned by Postgres 16+, so ensure this is not reflected in +-- the worker subquery constructed by Citus; it has just one relation, +-- for the pushed down subquery. +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id; + +-- Issue #7787 also occurs with a HAVING clause +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT 1 +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT a2) > 1; + +-- Check right outer join +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t2 RIGHT OUTER JOIN t1 ON t2.account_id = t1.id; + +-- Check both count distinct and having clause +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; +EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF) +SELECT COUNT(DISTINCT a2) +FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.account_id +HAVING COUNT(DISTINCT t2.id) > 1; + --- cleanup \set VERBOSITY TERSE -DROP SCHEMA issue_7705 CASCADE; +DROP SCHEMA outer_join_columns_testing CASCADE; RESET all; From 459c283e7d03dad12eb9bc5def710a58af0657c3 Mon Sep 17 00:00:00 2001 From: OlgaSergeyevaB Date: Tue, 18 Feb 2025 23:58:02 +0300 Subject: [PATCH 151/155] Custom Scan (ColumnarScan): exclude outer_join_rels from CandidateRelids (#7703) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESCRIPTION: Fixes a crash in columnar custom scan that happens when a columnar table is used in a join. Fixes issue #7647. Co-authored-by: Ольга Сергеева --- src/backend/columnar/columnar_customscan.c | 9 + src/test/regress/expected/columnar_join.out | 330 +++++++++++++++++++- src/test/regress/sql/columnar_join.sql | 149 +++++++++ 3 files changed, 487 insertions(+), 1 deletion(-) diff --git a/src/backend/columnar/columnar_customscan.c b/src/backend/columnar/columnar_customscan.c index 6b4e338c3..8d0adb39c 100644 --- a/src/backend/columnar/columnar_customscan.c +++ b/src/backend/columnar/columnar_customscan.c @@ -1051,6 +1051,15 @@ FindCandidateRelids(PlannerInfo *root, RelOptInfo *rel, List *joinClauses) candidateRelids = bms_del_members(candidateRelids, rel->relids); candidateRelids = bms_del_members(candidateRelids, rel->lateral_relids); + + /* + * For the relevant PG16 commit requiring this addition: + * postgres/postgres@2489d76 + */ +#if PG_VERSION_NUM >= PG_VERSION_16 + candidateRelids = bms_del_members(candidateRelids, root->outer_join_rels); +#endif + return candidateRelids; } diff --git a/src/test/regress/expected/columnar_join.out b/src/test/regress/expected/columnar_join.out index 04ffae31b..a2c0f1467 100644 --- a/src/test/regress/expected/columnar_join.out +++ b/src/test/regress/expected/columnar_join.out @@ -56,5 +56,333 @@ GROUP BY u1.id, u2.id; Columnar Projected Columns: id (10 rows) +-- ================================ +-- join COLUMNAR with HEAP +-- ================================ +-- Left Join with Mixed Table Types +CREATE TABLE tbl_left_heap1 (id integer); +CREATE TABLE tbl_left_heap2 (id integer); +CREATE TABLE tbl_left_columnar (id integer) USING columnar; +INSERT INTO tbl_left_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_left_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_left_columnar VALUES (3), (5), (7); +SELECT * +FROM tbl_left_heap1 h1 +LEFT JOIN tbl_left_heap2 h2 ON h1.id = h2.id +LEFT JOIN tbl_left_columnar c ON h2.id = c.id +ORDER BY 1; + id | id | id +--------------------------------------------------------------------- + 1 | | + 2 | 2 | + 3 | 3 | 3 + 4 | | +(4 rows) + +-- Left Join with Filter +CREATE TABLE tbl_left_filter_heap1 (id integer); +CREATE TABLE tbl_left_filter_heap2 (id integer); +CREATE TABLE tbl_left_filter_columnar (id integer) USING columnar; +INSERT INTO tbl_left_filter_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_left_filter_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_left_filter_columnar VALUES (3), (5), (7); +SELECT * +FROM tbl_left_filter_heap1 h1 +LEFT JOIN tbl_left_filter_heap2 h2 ON h1.id = h2.id +LEFT JOIN tbl_left_filter_columnar c ON h2.id = c.id +WHERE h1.id > 2 +ORDER BY 1; + id | id | id +--------------------------------------------------------------------- + 3 | 3 | 3 + 4 | | +(2 rows) + +-- Right Join with Mixed Table Types +CREATE TABLE tbl_right_heap1 (id integer); +CREATE TABLE tbl_right_heap2 (id integer); +CREATE TABLE tbl_right_columnar (id integer) USING columnar; +INSERT INTO tbl_right_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_right_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_right_columnar VALUES (3), (5), (7); +SELECT * +FROM tbl_right_heap1 h1 +RIGHT JOIN tbl_right_heap2 h2 ON h1.id = h2.id +RIGHT JOIN tbl_right_columnar c ON h2.id = c.id +ORDER BY 3; + id | id | id +--------------------------------------------------------------------- + 3 | 3 | 3 + | 5 | 5 + | | 7 +(3 rows) + +-- Right Join with Filters +CREATE TABLE tbl_right_filter_heap1 (id integer); +CREATE TABLE tbl_right_filter_heap2 (id integer); +CREATE TABLE tbl_right_filter_columnar (id integer) USING columnar; +INSERT INTO tbl_right_filter_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_right_filter_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_right_filter_columnar VALUES (3), (5), (7); +SELECT * +FROM tbl_right_filter_heap1 h1 +RIGHT JOIN tbl_right_filter_heap2 h2 ON h1.id = h2.id +RIGHT JOIN tbl_right_filter_columnar c ON h2.id = c.id +WHERE c.id < 6 +ORDER BY 3; + id | id | id +--------------------------------------------------------------------- + 3 | 3 | 3 + | 5 | 5 +(2 rows) + +-- Inner Join with Mixed Table Types +CREATE TABLE tbl_heap1 (id serial primary key, val integer); +CREATE TABLE tbl_heap2 (id serial primary key, val integer); +CREATE TABLE tbl_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_heap1 (val) SELECT generate_series(1, 100); +INSERT INTO tbl_heap2 (val) SELECT generate_series(50, 150); +INSERT INTO tbl_columnar SELECT generate_series(75, 125), generate_series(200, 250); +SELECT h1.id, h1.val, h2.val, c.val +FROM tbl_heap1 h1 +JOIN tbl_heap2 h2 ON h1.val = h2.val +JOIN tbl_columnar c ON h1.val = c.id +ORDER BY 1; + id | val | val | val +--------------------------------------------------------------------- + 75 | 75 | 75 | 200 + 76 | 76 | 76 | 201 + 77 | 77 | 77 | 202 + 78 | 78 | 78 | 203 + 79 | 79 | 79 | 204 + 80 | 80 | 80 | 205 + 81 | 81 | 81 | 206 + 82 | 82 | 82 | 207 + 83 | 83 | 83 | 208 + 84 | 84 | 84 | 209 + 85 | 85 | 85 | 210 + 86 | 86 | 86 | 211 + 87 | 87 | 87 | 212 + 88 | 88 | 88 | 213 + 89 | 89 | 89 | 214 + 90 | 90 | 90 | 215 + 91 | 91 | 91 | 216 + 92 | 92 | 92 | 217 + 93 | 93 | 93 | 218 + 94 | 94 | 94 | 219 + 95 | 95 | 95 | 220 + 96 | 96 | 96 | 221 + 97 | 97 | 97 | 222 + 98 | 98 | 98 | 223 + 99 | 99 | 99 | 224 + 100 | 100 | 100 | 225 +(26 rows) + +-- Outer Join with NULLs +CREATE TABLE tbl_null_heap (id integer, val integer); +CREATE TABLE tbl_null_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_null_heap VALUES (1, NULL), (2, 20), (3, 30); +INSERT INTO tbl_null_columnar VALUES (1, 100), (NULL, 200), (3, 300); +SELECT nh.id, nh.val, nc.val +FROM tbl_null_heap nh +FULL OUTER JOIN tbl_null_columnar nc ON nh.id = nc.id +ORDER BY 1; + id | val | val +--------------------------------------------------------------------- + 1 | | 100 + 2 | 20 | + 3 | 30 | 300 + | | 200 +(4 rows) + +-- Join with Aggregates +CREATE TABLE tbl_agg_heap (id serial primary key, val integer); +CREATE TABLE tbl_agg_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_agg_heap (val) SELECT generate_series(1, 100); +INSERT INTO tbl_agg_columnar SELECT generate_series(50, 150), generate_series(200, 300); +SELECT ah.val AS heap_val, COUNT(ac.val) AS columnar_count +FROM tbl_agg_heap ah +LEFT JOIN tbl_agg_columnar ac ON ah.val = ac.id +GROUP BY ah.val +ORDER BY ah.val; + heap_val | columnar_count +--------------------------------------------------------------------- + 1 | 0 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 0 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 0 + 12 | 0 + 13 | 0 + 14 | 0 + 15 | 0 + 16 | 0 + 17 | 0 + 18 | 0 + 19 | 0 + 20 | 0 + 21 | 0 + 22 | 0 + 23 | 0 + 24 | 0 + 25 | 0 + 26 | 0 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 0 + 31 | 0 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 0 + 36 | 0 + 37 | 0 + 38 | 0 + 39 | 0 + 40 | 0 + 41 | 0 + 42 | 0 + 43 | 0 + 44 | 0 + 45 | 0 + 46 | 0 + 47 | 0 + 48 | 0 + 49 | 0 + 50 | 1 + 51 | 1 + 52 | 1 + 53 | 1 + 54 | 1 + 55 | 1 + 56 | 1 + 57 | 1 + 58 | 1 + 59 | 1 + 60 | 1 + 61 | 1 + 62 | 1 + 63 | 1 + 64 | 1 + 65 | 1 + 66 | 1 + 67 | 1 + 68 | 1 + 69 | 1 + 70 | 1 + 71 | 1 + 72 | 1 + 73 | 1 + 74 | 1 + 75 | 1 + 76 | 1 + 77 | 1 + 78 | 1 + 79 | 1 + 80 | 1 + 81 | 1 + 82 | 1 + 83 | 1 + 84 | 1 + 85 | 1 + 86 | 1 + 87 | 1 + 88 | 1 + 89 | 1 + 90 | 1 + 91 | 1 + 92 | 1 + 93 | 1 + 94 | 1 + 95 | 1 + 96 | 1 + 97 | 1 + 98 | 1 + 99 | 1 + 100 | 1 +(100 rows) + +-- Join with Filters +CREATE TABLE tbl_filter_heap (id integer, val integer); +CREATE TABLE tbl_filter_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_filter_heap SELECT generate_series(1, 100), generate_series(1001, 1100); +INSERT INTO tbl_filter_columnar SELECT generate_series(90, 120), generate_series(2001, 2031); +SELECT fh.id, fh.val, fc.val +FROM tbl_filter_heap fh +INNER JOIN tbl_filter_columnar fc ON fh.id = fc.id +WHERE fh.val > 1050 AND fc.val < 2025 +ORDER BY 1; + id | val | val +--------------------------------------------------------------------- + 90 | 1090 | 2001 + 91 | 1091 | 2002 + 92 | 1092 | 2003 + 93 | 1093 | 2004 + 94 | 1094 | 2005 + 95 | 1095 | 2006 + 96 | 1096 | 2007 + 97 | 1097 | 2008 + 98 | 1098 | 2009 + 99 | 1099 | 2010 + 100 | 1100 | 2011 +(11 rows) + +-- Cross Join +CREATE TABLE tbl_cross_heap (id integer, val integer); +CREATE TABLE tbl_cross_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_cross_heap VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO tbl_cross_columnar VALUES (4, 40), (5, 50), (6, 60); +SELECT h.id AS heap_id, h.val AS heap_val, c.id AS columnar_id, c.val AS columnar_val +FROM tbl_cross_heap h +CROSS JOIN tbl_cross_columnar c +ORDER BY 3,4,1,2; + heap_id | heap_val | columnar_id | columnar_val +--------------------------------------------------------------------- + 1 | 10 | 4 | 40 + 2 | 20 | 4 | 40 + 3 | 30 | 4 | 40 + 1 | 10 | 5 | 50 + 2 | 20 | 5 | 50 + 3 | 30 | 5 | 50 + 1 | 10 | 6 | 60 + 2 | 20 | 6 | 60 + 3 | 30 | 6 | 60 +(9 rows) + +-- Left Join with Mixed Table Types and columnar in the middle +CREATE TABLE tbl_middle_left_heap1 (id integer); +CREATE TABLE tbl_middle_left_heap2 (id integer); +CREATE TABLE tbl_middle_left_columnar (id integer) USING columnar; +INSERT INTO tbl_middle_left_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_middle_left_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_middle_left_columnar VALUES (3), (5), (7); +EXPLAIN (COSTS OFF) +SELECT h1.*, h2.*, c.* +FROM tbl_middle_left_heap1 h1 +LEFT JOIN tbl_middle_left_columnar c ON h1.id = c.id +LEFT JOIN tbl_middle_left_heap2 h2 ON c.id = h2.id +ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- +Sort + Sort Key: h1.id + -> Nested Loop Left Join + Join Filter: (c.id = h2.id) + -> Nested Loop Left Join + Join Filter: (h1.id = c.id) + -> Seq Scan on tbl_middle_left_heap1 h1 + -> Custom Scan (ColumnarScan) on tbl_middle_left_columnar c + Columnar Projected Columns: id + -> Seq Scan on tbl_middle_left_heap2 h2 +(10 rows) + +-- End test case SET client_min_messages TO warning; -DROP SCHEMA am_columnar_join CASCADE; +DROP SCHEMA am_columnar_join CASCADE; \ No newline at end of file diff --git a/src/test/regress/sql/columnar_join.sql b/src/test/regress/sql/columnar_join.sql index bbeab54b5..9cbfd6177 100644 --- a/src/test/regress/sql/columnar_join.sql +++ b/src/test/regress/sql/columnar_join.sql @@ -31,5 +31,154 @@ JOIN users u2 ON (u1.id::text = u2.name) WHERE u2.id > 299990 GROUP BY u1.id, u2.id; +-- ================================ +-- join COLUMNAR with HEAP +-- ================================ + +-- Left Join with Mixed Table Types +CREATE TABLE tbl_left_heap1 (id integer); +CREATE TABLE tbl_left_heap2 (id integer); +CREATE TABLE tbl_left_columnar (id integer) USING columnar; + +INSERT INTO tbl_left_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_left_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_left_columnar VALUES (3), (5), (7); + +SELECT * +FROM tbl_left_heap1 h1 +LEFT JOIN tbl_left_heap2 h2 ON h1.id = h2.id +LEFT JOIN tbl_left_columnar c ON h2.id = c.id +ORDER BY 1; + +-- Left Join with Filter +CREATE TABLE tbl_left_filter_heap1 (id integer); +CREATE TABLE tbl_left_filter_heap2 (id integer); +CREATE TABLE tbl_left_filter_columnar (id integer) USING columnar; + +INSERT INTO tbl_left_filter_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_left_filter_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_left_filter_columnar VALUES (3), (5), (7); + +SELECT * +FROM tbl_left_filter_heap1 h1 +LEFT JOIN tbl_left_filter_heap2 h2 ON h1.id = h2.id +LEFT JOIN tbl_left_filter_columnar c ON h2.id = c.id +WHERE h1.id > 2 +ORDER BY 1; + + +-- Right Join with Mixed Table Types +CREATE TABLE tbl_right_heap1 (id integer); +CREATE TABLE tbl_right_heap2 (id integer); +CREATE TABLE tbl_right_columnar (id integer) USING columnar; + +INSERT INTO tbl_right_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_right_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_right_columnar VALUES (3), (5), (7); + +SELECT * +FROM tbl_right_heap1 h1 +RIGHT JOIN tbl_right_heap2 h2 ON h1.id = h2.id +RIGHT JOIN tbl_right_columnar c ON h2.id = c.id +ORDER BY 3; + +-- Right Join with Filters +CREATE TABLE tbl_right_filter_heap1 (id integer); +CREATE TABLE tbl_right_filter_heap2 (id integer); +CREATE TABLE tbl_right_filter_columnar (id integer) USING columnar; + +INSERT INTO tbl_right_filter_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_right_filter_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_right_filter_columnar VALUES (3), (5), (7); + +SELECT * +FROM tbl_right_filter_heap1 h1 +RIGHT JOIN tbl_right_filter_heap2 h2 ON h1.id = h2.id +RIGHT JOIN tbl_right_filter_columnar c ON h2.id = c.id +WHERE c.id < 6 +ORDER BY 3; + + +-- Inner Join with Mixed Table Types +CREATE TABLE tbl_heap1 (id serial primary key, val integer); +CREATE TABLE tbl_heap2 (id serial primary key, val integer); +CREATE TABLE tbl_columnar (id integer, val integer) USING columnar; +INSERT INTO tbl_heap1 (val) SELECT generate_series(1, 100); +INSERT INTO tbl_heap2 (val) SELECT generate_series(50, 150); +INSERT INTO tbl_columnar SELECT generate_series(75, 125), generate_series(200, 250); + +SELECT h1.id, h1.val, h2.val, c.val +FROM tbl_heap1 h1 +JOIN tbl_heap2 h2 ON h1.val = h2.val +JOIN tbl_columnar c ON h1.val = c.id +ORDER BY 1; + +-- Outer Join with NULLs +CREATE TABLE tbl_null_heap (id integer, val integer); +CREATE TABLE tbl_null_columnar (id integer, val integer) USING columnar; + +INSERT INTO tbl_null_heap VALUES (1, NULL), (2, 20), (3, 30); +INSERT INTO tbl_null_columnar VALUES (1, 100), (NULL, 200), (3, 300); + +SELECT nh.id, nh.val, nc.val +FROM tbl_null_heap nh +FULL OUTER JOIN tbl_null_columnar nc ON nh.id = nc.id +ORDER BY 1; + +-- Join with Aggregates +CREATE TABLE tbl_agg_heap (id serial primary key, val integer); +CREATE TABLE tbl_agg_columnar (id integer, val integer) USING columnar; + +INSERT INTO tbl_agg_heap (val) SELECT generate_series(1, 100); +INSERT INTO tbl_agg_columnar SELECT generate_series(50, 150), generate_series(200, 300); + +SELECT ah.val AS heap_val, COUNT(ac.val) AS columnar_count +FROM tbl_agg_heap ah +LEFT JOIN tbl_agg_columnar ac ON ah.val = ac.id +GROUP BY ah.val +ORDER BY ah.val; + +-- Join with Filters +CREATE TABLE tbl_filter_heap (id integer, val integer); +CREATE TABLE tbl_filter_columnar (id integer, val integer) USING columnar; + +INSERT INTO tbl_filter_heap SELECT generate_series(1, 100), generate_series(1001, 1100); +INSERT INTO tbl_filter_columnar SELECT generate_series(90, 120), generate_series(2001, 2031); + +SELECT fh.id, fh.val, fc.val +FROM tbl_filter_heap fh +INNER JOIN tbl_filter_columnar fc ON fh.id = fc.id +WHERE fh.val > 1050 AND fc.val < 2025 +ORDER BY 1; + +-- Cross Join +CREATE TABLE tbl_cross_heap (id integer, val integer); +CREATE TABLE tbl_cross_columnar (id integer, val integer) USING columnar; + +INSERT INTO tbl_cross_heap VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO tbl_cross_columnar VALUES (4, 40), (5, 50), (6, 60); + +SELECT h.id AS heap_id, h.val AS heap_val, c.id AS columnar_id, c.val AS columnar_val +FROM tbl_cross_heap h +CROSS JOIN tbl_cross_columnar c +ORDER BY 3,4,1,2; + +-- Left Join with Mixed Table Types and columnar in the middle +CREATE TABLE tbl_middle_left_heap1 (id integer); +CREATE TABLE tbl_middle_left_heap2 (id integer); +CREATE TABLE tbl_middle_left_columnar (id integer) USING columnar; + +INSERT INTO tbl_middle_left_heap1 VALUES (1), (2), (3), (4); +INSERT INTO tbl_middle_left_heap2 VALUES (2), (3), (5), (6); +INSERT INTO tbl_middle_left_columnar VALUES (3), (5), (7); + +EXPLAIN (COSTS OFF) +SELECT h1.*, h2.*, c.* +FROM tbl_middle_left_heap1 h1 +LEFT JOIN tbl_middle_left_columnar c ON h1.id = c.id +LEFT JOIN tbl_middle_left_heap2 h2 ON c.id = h2.id +ORDER BY 1; + +-- End test case SET client_min_messages TO warning; DROP SCHEMA am_columnar_join CASCADE; From c1f5762645f0f85eb5216e5fd675ae0119a61c45 Mon Sep 17 00:00:00 2001 From: Colm Date: Mon, 24 Feb 2025 09:11:19 +0000 Subject: [PATCH 152/155] Enhance MERGE .. WHEN NOT MATCHED BY SOURCE for repartitioned source (#7900) DESCRIPTION: Ensure that a MERGE command on a distributed table with a `WHEN NOT MATCHED BY SOURCE` clause runs against all shards of the distributed table. The Postgres MERGE command updates a table using a table or a query as a data source. It provides three ways to match the target table with the source: `WHEN MATCHED` means that there is a row in both the target and source; `WHEN NOT MATCHED` means that there is a row in the source that has no match (is not present) in the target; and, as of PG17, `WHEN NOT MATCHED BY SOURCE` means that there is a row in the target that has no match in the source. In Citus, when a MERGE command updates a distributed table using a local/reference table or a distributed query as source, that source is repartitioned, and for each repartitioned shard that has data (i.e. 1 or more rows) the MERGE is run against the corresponding distributed table shard. Suppose the distributed table has 32 shards, and the source repartitions into 4 shards that have data, with the remaining 28 shards being empty; then the MERGE command is performed on the 4 corresponding shards of the distributed table. However, the semantics of `WHEN NOT MATCHED BY SOURCE` are that the specified action must be performed on the target for each row in the target that is not in the source; so if the source is empty, all target rows should be updated. To see this, consider the following MERGE command: ``` MERGE INTO target AS t USING source AS s ON t.id = s.id WHEN NOT MATCHED BY SOURCE THEN UPDATE t SET t.col1 = 100 ``` If the source has zero rows then every row in the target is updated s.t. its col1 value is 100. Currently in Citus a MERGE on a distributed table with a local/reference table or a distributed query as source ignores shards of the distributed table when the corresponding shard of the repartitioned source has zero rows. However, if the MERGE command specifies a `WHEN NOT MATCHED BY SOURCE` clause, then the MERGE should be performed on all shards of the distributed table, to ensure that the specified action is performed on the target for each row in the target that is not in the source. This PR enhances Citus MERGE execution so that when a repartitioned source shard has zero rows, and the MERGE command specifies a `WHEN NOT MATCHED BY SOURCE` clause, the MERGE is performed against the corresponding shard of the distributed table using an empty (zero row) relation as source, by generating a query of the form: ``` MERGE INTO target_shard_0002 AS t USING (SELECT id FROM (VALUES (NULL) ) source_0002(id) WHERE FALSE) AS s ON t.id = s.id WHEN NOT MATCHED BY SOURCE THEN UPDATE t set t.col1 = 100 ``` This works because each row in the target shard will be updated, and `WHEN MATCHED` and `WHEN NOT MATCHED`, if specified, will be no-ops because the source has zero rows. To implement this when the source is a local or reference table involves teaching function `ExcuteSourceAtCoordAndRedistribution()` in `merge_executor.c` to not prune tasks when the query has `WHEN NOT MATCHED BY SOURCE` but to instead replace the task's query to one that uses an empty relation as source. And when the source is a distributed query, function `ExecuteMergeSourcePlanIntoColocatedIntermediateResults()` (also in `merge_executor.c`) instead of skipping empty tasks now generates a query that uses an empty relation as source for the corresponding target shard of the distributed table, but again only when the query has `WHEN NOT MATCHED BY SOURCE`. A new function `BuildEmptyResultQuery()` is added to `recursive_planning.c` and it is used by both the aforementioned functions in `merge_executor.c` to build an empty relation to use as the source. It applies the appropriate type to each column of the empty relation so the join with the target makes sense to the query compiler. --- .../distributed/executor/merge_executor.c | 19 +- .../executor/repartition_executor.c | 121 +++++++- .../distributed/planner/recursive_planning.c | 123 ++++++++ src/include/distributed/recursive_planning.h | 1 + .../distributed/repartition_executor.h | 5 + src/test/regress/expected/pg17.out | 285 +++++++++++++++++- src/test/regress/sql/pg17.sql | 171 +++++++++++ 7 files changed, 713 insertions(+), 12 deletions(-) diff --git a/src/backend/distributed/executor/merge_executor.c b/src/backend/distributed/executor/merge_executor.c index ce1eb0073..7af37d950 100644 --- a/src/backend/distributed/executor/merge_executor.c +++ b/src/backend/distributed/executor/merge_executor.c @@ -219,6 +219,7 @@ ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) copyObject(distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition); char *intermediateResultIdPrefix = distributedPlan->intermediateResultIdPrefix; bool hasReturning = distributedPlan->expectResults; + bool hasNotMatchedBySource = HasMergeNotMatchedBySource(mergeQuery); int partitionColumnIndex = distributedPlan->sourceResultRepartitionColumnIndex; /* @@ -233,7 +234,7 @@ ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) ereport(DEBUG1, (errmsg("Collect source query results on coordinator"))); - List *prunedTaskList = NIL; + List *prunedTaskList = NIL, *emptySourceTaskList = NIL; HTAB *shardStateHash = ExecuteMergeSourcePlanIntoColocatedIntermediateResults( targetRelationId, @@ -255,7 +256,8 @@ ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) * We cannot actually execute MERGE INTO ... tasks that read from * intermediate results that weren't created because no rows were * written to them. Prune those tasks out by only including tasks - * on shards with connections. + * on shards with connections; however, if the MERGE INTO includes + * a NOT MATCHED BY SOURCE clause we need to include the task. */ Task *task = NULL; foreach_declared_ptr(task, taskList) @@ -268,6 +270,19 @@ ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) { prunedTaskList = lappend(prunedTaskList, task); } + else if (hasNotMatchedBySource) + { + emptySourceTaskList = lappend(emptySourceTaskList, task); + } + } + + if (emptySourceTaskList != NIL) + { + ereport(DEBUG1, (errmsg("MERGE has NOT MATCHED BY SOURCE clause, " + "execute MERGE on all shards"))); + AdjustTaskQueryForEmptySource(targetRelationId, mergeQuery, emptySourceTaskList, + intermediateResultIdPrefix); + prunedTaskList = list_concat(prunedTaskList, emptySourceTaskList); } if (prunedTaskList == NIL) diff --git a/src/backend/distributed/executor/repartition_executor.c b/src/backend/distributed/executor/repartition_executor.c index 6e4dd3df4..4e83be889 100644 --- a/src/backend/distributed/executor/repartition_executor.c +++ b/src/backend/distributed/executor/repartition_executor.c @@ -17,6 +17,7 @@ #include "nodes/parsenodes.h" #include "distributed/citus_custom_scan.h" +#include "distributed/deparse_shard_query.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/multi_physical_planner.h" @@ -101,6 +102,40 @@ IsRedistributablePlan(Plan *selectPlan) } +/* + * HasMergeNotMatchedBySource returns true if the MERGE query has a + * WHEN NOT MATCHED BY SOURCE clause. If it does, we need to execute + * the MERGE query on all shards of the target table, regardless of + * whether or not the source shard has any rows. + */ +bool +HasMergeNotMatchedBySource(Query *query) +{ + if (!IsMergeQuery(query)) + { + return false; + } + + bool haveNotMatchedBySource = false; + + #if PG_VERSION_NUM >= PG_VERSION_17 + ListCell *lc; + foreach(lc, query->mergeActionList) + { + MergeAction *action = lfirst_node(MergeAction, lc); + + if (action->matchKind == MERGE_WHEN_NOT_MATCHED_BY_SOURCE) + { + haveNotMatchedBySource = true; + break; + } + } + #endif + + return haveNotMatchedBySource; +} + + /* * GenerateTaskListWithColocatedIntermediateResults generates a list of tasks * for a query that inserts into a target relation and selects from a set of @@ -200,6 +235,61 @@ GenerateTaskListWithColocatedIntermediateResults(Oid targetRelationId, } +/* + * AdjustTaskQueryForEmptySource adjusts the query for tasks that read from an + * intermediate result to instead read from an empty relation. This ensures that + * the MERGE query is executed on all shards of the target table, because it has + * a NOT MATCHED BY SOURCE clause, which will be true for all target shards where + * the source shard has no rows. + */ +void +AdjustTaskQueryForEmptySource(Oid targetRelationId, + Query *mergeQuery, + List *tasks, + char *resultIdPrefix) +{ + Query *mergeQueryCopy = copyObject(mergeQuery); + RangeTblEntry *selectRte = ExtractSourceResultRangeTableEntry(mergeQueryCopy); + RangeTblEntry *mergeRte = ExtractResultRelationRTE(mergeQueryCopy); + List *targetList = selectRte->subquery->targetList; + ListCell *taskCell = NULL; + + foreach(taskCell, tasks) + { + Task *task = lfirst(taskCell); + uint64 shardId = task->anchorShardId; + StringInfo queryString = makeStringInfo(); + StringInfo resultId = makeStringInfo(); + + appendStringInfo(resultId, "%s_" UINT64_FORMAT, resultIdPrefix, shardId); + + /* Generate a query for an empty relation */ + selectRte->subquery = BuildEmptyResultQuery(targetList, resultId->data); + + /* setting an alias simplifies deparsing of RETURNING */ + if (mergeRte->alias == NULL) + { + Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); + mergeRte->alias = alias; + } + + /* + * Generate a query string for the query that merges into a shard and reads + * from an empty relation. + * + * Since CTEs have already been converted to intermediate results, they need + * to removed from the query. Otherwise, worker queries include both + * intermediate results and CTEs in the query. + */ + mergeQueryCopy->cteList = NIL; + deparse_shard_query(mergeQueryCopy, targetRelationId, shardId, queryString); + ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data))); + + SetTaskQueryString(task, queryString->data); + } +} + + /* * GenerateTaskListWithRedistributedResults returns a task list to insert given * redistributedResults into the given target relation. @@ -223,6 +313,7 @@ GenerateTaskListWithRedistributedResults(Query *modifyQueryViaCoordinatorOrRepar Query *modifyResultQuery = copyObject(modifyQueryViaCoordinatorOrRepartition); RangeTblEntry *insertRte = ExtractResultRelationRTE(modifyResultQuery); Oid targetRelationId = targetRelation->relationId; + bool hasNotMatchedBySource = HasMergeNotMatchedBySource(modifyResultQuery); int shardCount = targetRelation->shardIntervalArrayLength; int shardOffset = 0; @@ -242,19 +333,33 @@ GenerateTaskListWithRedistributedResults(Query *modifyQueryViaCoordinatorOrRepar StringInfo queryString = makeStringInfo(); /* skip empty tasks */ - if (resultIdList == NIL) + if (resultIdList == NIL && !hasNotMatchedBySource) { continue; } - /* sort result ids for consistent test output */ - List *sortedResultIds = SortList(resultIdList, pg_qsort_strcmp); + Query *fragmentSetQuery = NULL; - /* generate the query on the intermediate result */ - Query *fragmentSetQuery = BuildReadIntermediateResultsArrayQuery(selectTargetList, - NIL, - sortedResultIds, - useBinaryFormat); + if (resultIdList != NIL) + { + /* sort result ids for consistent test output */ + List *sortedResultIds = SortList(resultIdList, pg_qsort_strcmp); + + /* generate the query on the intermediate result */ + fragmentSetQuery = BuildReadIntermediateResultsArrayQuery(selectTargetList, + NIL, + sortedResultIds, + useBinaryFormat); + } + else + { + /* No source data, but MERGE query has NOT MATCHED BY SOURCE */ + StringInfo emptyFragmentId = makeStringInfo(); + appendStringInfo(emptyFragmentId, "%s_" UINT64_FORMAT, "temp_empty_rel_", + shardId); + fragmentSetQuery = BuildEmptyResultQuery(selectTargetList, + emptyFragmentId->data); + } /* put the intermediate result query in the INSERT..SELECT */ selectRte->subquery = fragmentSetQuery; diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 9335b5ffc..d65a64410 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -2291,6 +2291,129 @@ BuildReadIntermediateResultsArrayQuery(List *targetEntryList, } +/* + * For the given target list, build an empty relation with the same target list. + * For example, if the target list is (a, b, c), and resultId is "empty", then + * it returns a Query object for this SQL: + * SELECT a, b, c FROM (VALUES (NULL, NULL, NULL)) AS empty(a, b, c) WHERE false; + */ +Query * +BuildEmptyResultQuery(List *targetEntryList, char *resultId) +{ + List *targetList = NIL; + ListCell *targetEntryCell = NULL; + + List *colTypes = NIL; + List *colTypMods = NIL; + List *colCollations = NIL; + List *colNames = NIL; + + List *valueConsts = NIL; + List *valueTargetList = NIL; + List *valueColNames = NIL; + + int targetIndex = 1; + + /* build the target list and column lists needed */ + foreach(targetEntryCell, targetEntryList) + { + TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); + Node *targetExpr = (Node *) targetEntry->expr; + char *columnName = targetEntry->resname; + Oid columnType = exprType(targetExpr); + Oid columnTypMod = exprTypmod(targetExpr); + Oid columnCollation = exprCollation(targetExpr); + + if (targetEntry->resjunk) + { + continue; + } + + Var *tgtVar = makeVar(1, targetIndex, columnType, columnTypMod, columnCollation, + 0); + TargetEntry *tgtEntry = makeTargetEntry((Expr *) tgtVar, targetIndex, columnName, + false); + Const *valueConst = makeConst(columnType, columnTypMod, columnCollation, 0, + (Datum) 0, true, false); + + StringInfoData *columnString = makeStringInfo(); + appendStringInfo(columnString, "column%d", targetIndex); + + TargetEntry *valueTgtEntry = makeTargetEntry((Expr *) tgtVar, targetIndex, + columnString->data, false); + + valueConsts = lappend(valueConsts, valueConst); + valueTargetList = lappend(valueTargetList, valueTgtEntry); + valueColNames = lappend(valueColNames, makeString(columnString->data)); + + colNames = lappend(colNames, makeString(columnName)); + colTypes = lappend_oid(colTypes, columnType); + colTypMods = lappend_oid(colTypMods, columnTypMod); + colCollations = lappend_oid(colCollations, columnCollation); + + targetList = lappend(targetList, tgtEntry); + + targetIndex++; + } + + /* Build a RangeTable Entry for the VALUES relation */ + RangeTblEntry *valuesRangeTable = makeNode(RangeTblEntry); + valuesRangeTable->rtekind = RTE_VALUES; + valuesRangeTable->values_lists = list_make1(valueConsts); + valuesRangeTable->colcollations = colCollations; + valuesRangeTable->coltypes = colTypes; + valuesRangeTable->coltypmods = colTypMods; + valuesRangeTable->alias = NULL; + valuesRangeTable->eref = makeAlias("*VALUES*", valueColNames); + valuesRangeTable->inFromCl = true; + + RangeTblRef *valuesRTRef = makeNode(RangeTblRef); + valuesRTRef->rtindex = 1; + + FromExpr *valuesJoinTree = makeNode(FromExpr); + valuesJoinTree->fromlist = list_make1(valuesRTRef); + + /* build the VALUES query */ + Query *valuesQuery = makeNode(Query); + valuesQuery->canSetTag = true; + valuesQuery->commandType = CMD_SELECT; + valuesQuery->rtable = list_make1(valuesRangeTable); + #if PG_VERSION_NUM >= PG_VERSION_16 + valuesQuery->rteperminfos = NIL; + #endif + valuesQuery->jointree = valuesJoinTree; + valuesQuery->targetList = valueTargetList; + + /* build the relation selecting from the VALUES */ + RangeTblEntry *emptyRangeTable = makeNode(RangeTblEntry); + emptyRangeTable->rtekind = RTE_SUBQUERY; + emptyRangeTable->subquery = valuesQuery; + emptyRangeTable->alias = makeAlias(resultId, colNames); + emptyRangeTable->eref = emptyRangeTable->alias; + emptyRangeTable->inFromCl = true; + + /* build the SELECT query */ + Query *resultQuery = makeNode(Query); + resultQuery->commandType = CMD_SELECT; + resultQuery->canSetTag = true; + resultQuery->rtable = list_make1(emptyRangeTable); +#if PG_VERSION_NUM >= PG_VERSION_16 + resultQuery->rteperminfos = NIL; +#endif + RangeTblRef *rangeTableRef = makeNode(RangeTblRef); + rangeTableRef->rtindex = 1; + + /* insert a FALSE qual to ensure 0 rows returned */ + FromExpr *joinTree = makeNode(FromExpr); + joinTree->fromlist = list_make1(rangeTableRef); + joinTree->quals = makeBoolConst(false, false); + resultQuery->jointree = joinTree; + resultQuery->targetList = targetList; + + return resultQuery; +} + + /* * BuildReadIntermediateResultsQuery is the common code for generating * queries to read from result files. It is used by diff --git a/src/include/distributed/recursive_planning.h b/src/include/distributed/recursive_planning.h index c37eba343..b4aaa4785 100644 --- a/src/include/distributed/recursive_planning.h +++ b/src/include/distributed/recursive_planning.h @@ -40,6 +40,7 @@ extern Query * BuildReadIntermediateResultsArrayQuery(List *targetEntryList, List *columnAliasList, List *resultIdList, bool useBinaryCopyFormat); +extern Query * BuildEmptyResultQuery(List *targetEntryList, char *resultId); extern bool GeneratingSubplans(void); extern bool ContainsLocalTableDistributedTableJoin(List *rangeTableList); extern void ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, diff --git a/src/include/distributed/repartition_executor.h b/src/include/distributed/repartition_executor.h index de4ad122a..f636877e7 100644 --- a/src/include/distributed/repartition_executor.h +++ b/src/include/distributed/repartition_executor.h @@ -28,5 +28,10 @@ extern List * GenerateTaskListWithRedistributedResults( bool useBinaryFormat); extern bool IsSupportedRedistributionTarget(Oid targetRelationId); extern bool IsRedistributablePlan(Plan *selectPlan); +extern bool HasMergeNotMatchedBySource(Query *query); +extern void AdjustTaskQueryForEmptySource(Oid targetRelationId, + Query *mergeQuery, + List *emptySourceTaskList, + char *resultIdPrefix); #endif /* REPARTITION_EXECUTOR_H */ diff --git a/src/test/regress/expected/pg17.out b/src/test/regress/expected/pg17.out index c6deb41aa..5c0dc73c6 100644 --- a/src/test/regress/expected/pg17.out +++ b/src/test/regress/expected/pg17.out @@ -2555,6 +2555,285 @@ MERGE INTO citus_reference_target t WHEN NOT MATCHED BY SOURCE THEN UPDATE SET val = val || ' not matched by source'; ERROR: Reference table as target is not allowed in MERGE command +-- Test Distributed-reference and distributed-local when the source table has fewer rows +-- than distributed target; this tests that MERGE with NOT MATCHED BY SOURCE needs to run +-- on all shards of the distributed target, regardless of whether or not the reshuffled +-- source table has data in the corresponding shard. +-- Re-populate the Postgres tables; +DELETE FROM postgres_source; +DELETE FROM postgres_target_1; +DELETE FROM postgres_target_2; +-- This time, the source table has fewer rows +INSERT INTO postgres_target_1 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_target_2 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_source SELECT id, id * 10 FROM generate_series(1,4) AS id; +-- try simple MERGE +MERGE INTO postgres_target_1 t + USING postgres_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_1 ORDER BY tid, val; + tid | balance | val +--------------------------------------------------------------------- + 1 | 110 | initial updated by merge + 2 | 20 | inserted by merge + 3 | 330 | initial updated by merge + 4 | 40 | inserted by merge + 5 | 500 | initial not matched by source + 7 | 700 | initial not matched by source + 9 | 900 | initial not matched by source + 11 | 1100 | initial not matched by source + 13 | 1300 | initial not matched by source + 15 | 1500 | initial not matched by source +(10 rows) + +-- same with a constant qual +MERGE INTO postgres_target_2 t + USING postgres_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_2 ORDER BY tid, val; + tid | balance | val +--------------------------------------------------------------------- + 1 | 110 | initial updated by merge + 2 | 20 | inserted by merge + 3 | 300 | initial not matched by source + 3 | 30 | inserted by merge + 4 | 40 | inserted by merge + 5 | 500 | initial not matched by source + 7 | 700 | initial not matched by source + 9 | 900 | initial not matched by source + 11 | 1100 | initial not matched by source + 13 | 1300 | initial not matched by source + 15 | 1500 | initial not matched by source +(11 rows) + +-- Re-populate the Citus tables; this time, the source table has fewer rows +DELETE FROM citus_local_source; +DELETE FROM citus_reference_source; +INSERT INTO citus_reference_source SELECT id, id * 10 FROM generate_series(1,4) AS id; +INSERT INTO citus_local_source SELECT id, id * 10 FROM generate_series(1,4) AS id; +SET citus.shard_count to 32; +CREATE TABLE citus_distributed_target32 (tid integer, balance float, val text); +SELECT create_distributed_table('citus_distributed_target32', 'tid'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO citus_distributed_target32 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +-- Distributed-Local +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Distributed-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_1'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_2'); + compare_tables +--------------------------------------------------------------------- + t +(1 row) + +ROLLBACK; +-- Test that MERGE with NOT MATCHED BY SOURCE runs on all shards of +-- a distributed table when the source is a repartition query with +-- rows that do not match the distributed target +set citus.shard_count = 32; +CREATE TABLE dist_target (tid integer, balance float); +CREATE TABLE dist_src1(sid integer, tid integer, val float); +CREATE TABLE dist_src2(sid integer); +CREATE TABLE dist_ref(sid integer); +INSERT INTO dist_target SELECT id, 0 FROM generate_series(1,9,2) AS id; +INSERT INTO dist_src1 SELECT id, id%3 + 1, id*10 FROM generate_series(1,15) AS id; +INSERT INTO dist_src2 SELECT id FROM generate_series(1,100) AS id; +INSERT INTO dist_ref SELECT id FROM generate_series(1,10) AS id; +-- Run a MERGE command with dist_target as target and an aggregating query +-- as source; note that at this point all tables are vanilla Postgres tables +BEGIN; +SELECT * FROM dist_target ORDER BY tid; + tid | balance +--------------------------------------------------------------------- + 1 | 0 + 3 | 0 + 5 | 0 + 7 | 0 + 9 | 0 +(5 rows) + +MERGE INTO dist_target t +USING (SELECT dt.tid, avg(dt.val) as av, min(dt.val) as m, max(dt.val) as x + FROM dist_src1 dt INNER JOIN dist_src2 dt2 on dt.sid=dt2.sid + INNER JOIN dist_ref dr ON dt.sid=dr.sid + GROUP BY dt.tid) dv ON (t.tid=dv.tid) +WHEN MATCHED THEN + UPDATE SET balance = dv.av +WHEN NOT MATCHED THEN + INSERT (tid, balance) VALUES (dv.tid, dv.m) +WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET balance = 99.95; +SELECT * FROM dist_target ORDER BY tid; + tid | balance +--------------------------------------------------------------------- + 1 | 60 + 2 | 10 + 3 | 50 + 5 | 99.95 + 7 | 99.95 + 9 | 99.95 +(6 rows) + +ROLLBACK; +-- Distribute the tables +SELECT create_distributed_table('dist_target', 'tid'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg17.dist_target$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('dist_src1', 'sid'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg17.dist_src1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('dist_src2', 'sid'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg17.dist_src2$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('dist_ref'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg17.dist_ref$$) + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Re-run the merge; the target is now distributed and the source is a +-- distributed query that is repartitioned. +BEGIN; +SELECT * FROM dist_target ORDER BY tid; + tid | balance +--------------------------------------------------------------------- + 1 | 0 + 3 | 0 + 5 | 0 + 7 | 0 + 9 | 0 +(5 rows) + +MERGE INTO dist_target t +USING (SELECT dt.tid, avg(dt.val) as av, min(dt.val) as m, max(dt.val) as x + FROM dist_src1 dt INNER JOIN dist_src2 dt2 on dt.sid=dt2.sid + INNER JOIN dist_ref dr ON dt.sid=dr.sid + GROUP BY dt.tid) dv ON (t.tid=dv.tid) +WHEN MATCHED THEN + UPDATE SET balance = dv.av +WHEN NOT MATCHED THEN + INSERT (tid, balance) VALUES (dv.tid, dv.m) +WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET balance = 99.95; +-- Data in dist_target is as it was with vanilla Postgres tables: +SELECT * FROM dist_target ORDER BY tid; + tid | balance +--------------------------------------------------------------------- + 1 | 60 + 2 | 10 + 3 | 50 + 5 | 99.95 + 7 | 99.95 + 9 | 99.95 +(6 rows) + +ROLLBACK; +-- Reset shard_count for the DEBUG output in the following test +SET citus.shard_count to 4; -- Complex repartition query example with a mix of tables -- Example from blog post -- https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge @@ -2670,8 +2949,10 @@ DEBUG: Using column - index:0 from the source list to redistribute DEBUG: Executing subplans of the source query and storing the results at the respective node(s) DEBUG: Redistributing source result rows across nodes DEBUG: Executing final MERGE on workers using intermediate results -DEBUG: -DEBUG: +DEBUG: +DEBUG: +DEBUG: +DEBUG: RESET client_min_messages; -- Expected output is: -- reading_id | sensor_id | reading_value | reading_timestamp diff --git a/src/test/regress/sql/pg17.sql b/src/test/regress/sql/pg17.sql index f55d50d17..72998fce0 100644 --- a/src/test/regress/sql/pg17.sql +++ b/src/test/regress/sql/pg17.sql @@ -1336,6 +1336,177 @@ MERGE INTO citus_reference_target t WHEN NOT MATCHED BY SOURCE THEN UPDATE SET val = val || ' not matched by source'; +-- Test Distributed-reference and distributed-local when the source table has fewer rows +-- than distributed target; this tests that MERGE with NOT MATCHED BY SOURCE needs to run +-- on all shards of the distributed target, regardless of whether or not the reshuffled +-- source table has data in the corresponding shard. + +-- Re-populate the Postgres tables; +DELETE FROM postgres_source; +DELETE FROM postgres_target_1; +DELETE FROM postgres_target_2; + +-- This time, the source table has fewer rows +INSERT INTO postgres_target_1 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_target_2 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; +INSERT INTO postgres_source SELECT id, id * 10 FROM generate_series(1,4) AS id; + +-- try simple MERGE +MERGE INTO postgres_target_1 t + USING postgres_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_1 ORDER BY tid, val; + +-- same with a constant qual +MERGE INTO postgres_target_2 t + USING postgres_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT * FROM postgres_target_2 ORDER BY tid, val; + +-- Re-populate the Citus tables; this time, the source table has fewer rows +DELETE FROM citus_local_source; +DELETE FROM citus_reference_source; +INSERT INTO citus_reference_source SELECT id, id * 10 FROM generate_series(1,4) AS id; +INSERT INTO citus_local_source SELECT id, id * 10 FROM generate_series(1,4) AS id; + +SET citus.shard_count to 32; +CREATE TABLE citus_distributed_target32 (tid integer, balance float, val text); +SELECT create_distributed_table('citus_distributed_target32', 'tid'); +INSERT INTO citus_distributed_target32 SELECT id, id * 100, 'initial' FROM generate_series(1,15,2) AS id; + +-- Distributed-Local +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_local_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_local_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED BY TARGET THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_2'); +ROLLBACK; + +-- Distributed-Reference +-- try simple MERGE +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_reference_source s + ON t.tid = s.sid + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_1'); +ROLLBACK; + +-- same with a constant qual +BEGIN; +MERGE INTO citus_distributed_target32 t + USING citus_reference_source s + ON t.tid = s.sid AND tid = 1 + WHEN MATCHED THEN + UPDATE SET balance = balance + delta, val = val || ' updated by merge' + WHEN NOT MATCHED THEN + INSERT VALUES (sid, delta, 'inserted by merge') + WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET val = val || ' not matched by source'; +SELECT compare_tables('citus_distributed_target32', 'postgres_target_2'); +ROLLBACK; + +-- Test that MERGE with NOT MATCHED BY SOURCE runs on all shards of +-- a distributed table when the source is a repartition query with +-- rows that do not match the distributed target + +set citus.shard_count = 32; + +CREATE TABLE dist_target (tid integer, balance float); +CREATE TABLE dist_src1(sid integer, tid integer, val float); +CREATE TABLE dist_src2(sid integer); +CREATE TABLE dist_ref(sid integer); + +INSERT INTO dist_target SELECT id, 0 FROM generate_series(1,9,2) AS id; +INSERT INTO dist_src1 SELECT id, id%3 + 1, id*10 FROM generate_series(1,15) AS id; +INSERT INTO dist_src2 SELECT id FROM generate_series(1,100) AS id; +INSERT INTO dist_ref SELECT id FROM generate_series(1,10) AS id; + +-- Run a MERGE command with dist_target as target and an aggregating query +-- as source; note that at this point all tables are vanilla Postgres tables +BEGIN; +SELECT * FROM dist_target ORDER BY tid; +MERGE INTO dist_target t +USING (SELECT dt.tid, avg(dt.val) as av, min(dt.val) as m, max(dt.val) as x + FROM dist_src1 dt INNER JOIN dist_src2 dt2 on dt.sid=dt2.sid + INNER JOIN dist_ref dr ON dt.sid=dr.sid + GROUP BY dt.tid) dv ON (t.tid=dv.tid) +WHEN MATCHED THEN + UPDATE SET balance = dv.av +WHEN NOT MATCHED THEN + INSERT (tid, balance) VALUES (dv.tid, dv.m) +WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET balance = 99.95; +SELECT * FROM dist_target ORDER BY tid; +ROLLBACK; + +-- Distribute the tables +SELECT create_distributed_table('dist_target', 'tid'); +SELECT create_distributed_table('dist_src1', 'sid'); +SELECT create_distributed_table('dist_src2', 'sid'); +SELECT create_reference_table('dist_ref'); + +-- Re-run the merge; the target is now distributed and the source is a +-- distributed query that is repartitioned. +BEGIN; +SELECT * FROM dist_target ORDER BY tid; +MERGE INTO dist_target t +USING (SELECT dt.tid, avg(dt.val) as av, min(dt.val) as m, max(dt.val) as x + FROM dist_src1 dt INNER JOIN dist_src2 dt2 on dt.sid=dt2.sid + INNER JOIN dist_ref dr ON dt.sid=dr.sid + GROUP BY dt.tid) dv ON (t.tid=dv.tid) +WHEN MATCHED THEN + UPDATE SET balance = dv.av +WHEN NOT MATCHED THEN + INSERT (tid, balance) VALUES (dv.tid, dv.m) +WHEN NOT MATCHED BY SOURCE THEN + UPDATE SET balance = 99.95; + +-- Data in dist_target is as it was with vanilla Postgres tables: +SELECT * FROM dist_target ORDER BY tid; +ROLLBACK; + +-- Reset shard_count for the DEBUG output in the following test + +SET citus.shard_count to 4; -- Complex repartition query example with a mix of tables -- Example from blog post -- https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge From 2b964228bc468ce2f384159ff1dcb01656b14b3d Mon Sep 17 00:00:00 2001 From: Mehmet YILMAZ Date: Tue, 25 Feb 2025 20:49:32 +0300 Subject: [PATCH 153/155] Fix 0-Task Plans in Single-Shard Router When Updating a Local Table with Reference Table in Subquery (#7897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes an issue #7891 in the Citus planner where an `UPDATE` on a local table with a subquery referencing a reference table could produce a 0-task plan. Historically, the planner sometimes failed to detect that both the target and referenced tables were effectively “local,” assigning `INVALID_SHARD_ID `and yielding a no-op plan. ### Root Cause - In the Citus router logic (`PlanRouterQuery`), we relied on `shardId` to determine whether a query should be routed to a single shard. - If `shardId == INVALID_SHARD_ID`, but we also had not marked the query as a “local table modification,” the code path would produce zero tasks. - Local + reference tables do not require multi-shard routing. Failing to detect this “purely local” scenario caused Citus to incorrectly route to zero tasks. ### Changes **Enhanced Local Table Detection** - Updated `IsLocalTableModification` and related checks to consider both local and reference tables as “local” for planning, preventing the 0-task scenario. - Expanded `ContainsOnlyLocalOrReferenceTables` to return true if there are no fully distributed tables in the query. **Added Regress Test** - Introduced a new regress test (`issue_7891.sql`) which reproduces the scenario. - Verifies we get a valid single- or local-task plan rather than a 0-task plan. --- citus-tools | 1 + .../distributed/planner/merge_planner.c | 2 +- .../planner/multi_router_planner.c | 10 +- .../distributed/multi_router_planner.h | 2 +- src/test/regress/expected/issue_7891.out | 211 ++++++++++++++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/issue_7891.sql | 169 ++++++++++++++ 7 files changed, 390 insertions(+), 7 deletions(-) create mode 160000 citus-tools create mode 100644 src/test/regress/expected/issue_7891.out create mode 100644 src/test/regress/sql/issue_7891.sql diff --git a/citus-tools b/citus-tools new file mode 160000 index 000000000..3376bd684 --- /dev/null +++ b/citus-tools @@ -0,0 +1 @@ +Subproject commit 3376bd6845f0614908ed304f5033bd644c82d3bf diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index e1d917ca0..6f3993794 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -1583,7 +1583,7 @@ IsLocalTableModification(Oid targetRelationId, Query *query, uint64 shardId, return true; } - if (shardId == INVALID_SHARD_ID && ContainsOnlyLocalTables(rteProperties)) + if (shardId == INVALID_SHARD_ID && ContainsOnlyLocalOrReferenceTables(rteProperties)) { return true; } diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 72d1dddd1..13d62c765 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -2556,13 +2556,15 @@ AllShardsColocated(List *relationShardList) /* - * ContainsOnlyLocalTables returns true if there is only - * local tables and not any distributed or reference table. + * ContainsOnlyLocalOrReferenceTables returns true if there are no distributed + * tables in the query. In other words, the query might reference only local + * tables and/or reference tables, but no fully distributed tables. */ bool -ContainsOnlyLocalTables(RTEListProperties *rteProperties) +ContainsOnlyLocalOrReferenceTables(RTEListProperties *rteProperties) { - return !rteProperties->hasDistributedTable && !rteProperties->hasReferenceTable; + /* If hasDistributedTable is false, then all tables are either local or reference. */ + return !rteProperties->hasDistributedTable; } diff --git a/src/include/distributed/multi_router_planner.h b/src/include/distributed/multi_router_planner.h index ae75ee631..44be2736e 100644 --- a/src/include/distributed/multi_router_planner.h +++ b/src/include/distributed/multi_router_planner.h @@ -118,7 +118,7 @@ extern bool HasDangerousJoinUsing(List *rtableList, Node *jtnode); extern Job * RouterJob(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, DeferredErrorMessage **planningError); -extern bool ContainsOnlyLocalTables(RTEListProperties *rteProperties); +extern bool ContainsOnlyLocalOrReferenceTables(RTEListProperties *rteProperties); extern RangeTblEntry * ExtractSourceResultRangeTableEntry(Query *query); #endif /* MULTI_ROUTER_PLANNER_H */ diff --git a/src/test/regress/expected/issue_7891.out b/src/test/regress/expected/issue_7891.out new file mode 100644 index 000000000..0e3148d87 --- /dev/null +++ b/src/test/regress/expected/issue_7891.out @@ -0,0 +1,211 @@ +-- This test validates that the query planner correctly handles nested subqueries involving both a +-- local table (t4_pg) and a reference table (t2_ref). The steps are as follows: +-- +-- 1. A dedicated schema (issue_7891) is created, and three tables (t2_ref, t4_pg, t6_pg) are set up. +-- 2. The table t2_ref is designated as a reference table using the create_reference_table() function. +-- 3. Sample data is inserted into all tables. +-- 4. An UPDATE is executed on t6_pg. The update uses an EXISTS clause with a nested subquery: +-- - The outer subquery iterates over every row in t4_pg. +-- - The inner subquery selects c15 from t2_ref. +-- 5. The update should occur if the nested subquery returns any row, effectively updating t6_pg's vkey to 43. +-- 6. The final state of t6_pg is displayed to confirm that the update was applied. +-- +-- Note: This test was originally designed to detect a planner bug where the nested structure might +-- lead to an incorrect plan (such as a 0-task plan), ensuring proper handling of reference and local tables. +-- https://github.com/citusdata/citus/issues/7891 +CREATE SCHEMA issue_7891; +SET search_path TO issue_7891; +-- Create tables +CREATE TABLE t2_ref ( + vkey INT, + pkey INT, + c15 TIMESTAMP +); +CREATE TABLE t2_ref2 ( + vkey INT, + pkey INT, + c15 TIMESTAMP +); +CREATE TABLE t4_pg ( + vkey INT, + pkey INT, + c22 NUMERIC, + c23 TEXT, + c24 TIMESTAMP +); +CREATE TABLE t6_pg ( + vkey INT, + pkey INT, + c26 TEXT +); +-- Mark t2_ref and t2_ref2 as a reference table +SELECT create_reference_table('t2_ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('t2_ref2'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Insert sample data +INSERT INTO t6_pg (vkey, pkey, c26) VALUES + (2, 12000, 'initial'), + (3, 13000, 'will_be_deleted'), + (4, 14000, 'to_merge'); +INSERT INTO t4_pg (vkey, pkey, c22, c23, c24) + VALUES (5, 15000, 0.0, ']]?', MAKE_TIMESTAMP(2071, 10, 26, 16, 20, 5)); +INSERT INTO t2_ref (vkey, pkey, c15) + VALUES (14, 24000, NULL::timestamp); +-- Show initial data +SELECT 't6_pg before' AS label, * FROM t6_pg; + label | vkey | pkey | c26 +--------------------------------------------------------------------- + t6_pg before | 2 | 12000 | initial + t6_pg before | 3 | 13000 | will_be_deleted + t6_pg before | 4 | 14000 | to_merge +(3 rows) + +SELECT 't4_pg data' AS label, * FROM t4_pg; + label | vkey | pkey | c22 | c23 | c24 +--------------------------------------------------------------------- + t4_pg data | 5 | 15000 | 0.0 | ]]? | Mon Oct 26 16:20:05 2071 +(1 row) + +SELECT 't2_ref data' AS label, * FROM t2_ref; + label | vkey | pkey | c15 +--------------------------------------------------------------------- + t2_ref data | 14 | 24000 | +(1 row) + +-- +-- The problematic query: update t6_pg referencing t4_pg and sub-subquery on t2_ref. +-- Historically might produce a 0-task plan if the planner incorrectly fails to +-- treat t4_pg/t2_ref as local/reference. +-- +-- The outer subquery iterates over every row in table t4_pg. +UPDATE t6_pg + SET vkey = 43 + WHERE EXISTS ( + SELECT (SELECT c15 FROM t2_ref) + FROM t4_pg +); +SELECT 't6_pg after' AS label, * FROM t6_pg; + label | vkey | pkey | c26 +--------------------------------------------------------------------- + t6_pg after | 43 | 12000 | initial + t6_pg after | 43 | 13000 | will_be_deleted + t6_pg after | 43 | 14000 | to_merge +(3 rows) + +-- +-- DELETE with a similar nested subquery approach +-- Here, let's delete any rows for which t4_pg is non-empty (like a trivial check). +-- We'll specifically target the row with c26='will_be_deleted' to confirm it's removed. +-- +DELETE FROM t6_pg + WHERE EXISTS ( + SELECT (SELECT c15 FROM t2_ref) + FROM t4_pg + ) + AND c26 = 'will_be_deleted'; +SELECT 't6_pg after DELETE' AS label, * FROM t6_pg; + label | vkey | pkey | c26 +--------------------------------------------------------------------- + t6_pg after DELETE | 43 | 12000 | initial + t6_pg after DELETE | 43 | 14000 | to_merge +(2 rows) + +-- +-- We'll merge from t4_pg into t6_pg. The merge will update c26 for pkey=14000. +-- +-- Anticipate an error indicating non-IMMUTABLE functions are not supported in MERGE statements on distributed tables. +-- Retain this comment to highlight the current limitation. +-- +MERGE INTO t6_pg AS tgt +USING t4_pg AS src +ON (tgt.pkey = 14000) +WHEN MATCHED THEN + UPDATE SET c26 = 'merged_' || (SELECT pkey FROM t2_ref WHERE pkey=24000 LIMIT 1) +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c26) + VALUES (99, src.pkey, 'inserted_via_merge'); +ERROR: non-IMMUTABLE functions are not yet supported in MERGE sql with distributed tables +MERGE INTO t2_ref AS tgt +USING t4_pg AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c15 = '2088-01-01 00:00:00'::timestamp +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c15) + VALUES (src.vkey, src.pkey, '2099-12-31 23:59:59'::timestamp); +ERROR: Reference table as target is not allowed in MERGE command +-- Show the final state of t2_ref: +SELECT 't2_ref after MERGE (using t4_pg)' AS label, * FROM t2_ref; + label | vkey | pkey | c15 +--------------------------------------------------------------------- + t2_ref after MERGE (using t4_pg) | 14 | 24000 | +(1 row) + +MERGE INTO t2_ref2 AS tgt +USING t2_ref AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c15 = '2077-07-07 07:07:07'::timestamp +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c15) + VALUES (src.vkey, src.pkey, '2066-06-06 06:06:06'::timestamp); +ERROR: Reference table as target is not allowed in MERGE command +-- Show the final state of t2_ref2: +SELECT 't2_ref2 after MERGE (using t2_ref)' AS label, * FROM t2_ref2; + label | vkey | pkey | c15 +--------------------------------------------------------------------- +(0 rows) + +MERGE INTO t6_pg AS tgt +USING t4_pg AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c26 = 'merged_value' +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c26) + VALUES (src.vkey, src.pkey, 'inserted_via_merge'); +SELECT 't6_pg after MERGE' AS label, * FROM t6_pg; + label | vkey | pkey | c26 +--------------------------------------------------------------------- + t6_pg after MERGE | 43 | 12000 | initial + t6_pg after MERGE | 43 | 14000 | to_merge + t6_pg after MERGE | 5 | 15000 | inserted_via_merge +(3 rows) + +-- +-- Update the REFERENCE table itself and verify the change +-- This is to ensure that the reference table is correctly handled. +UPDATE t2_ref + SET c15 = '2099-01-01 00:00:00'::timestamp + WHERE pkey = 24000; +SELECT 't2_ref after self-update' AS label, * FROM t2_ref; + label | vkey | pkey | c15 +--------------------------------------------------------------------- + t2_ref after self-update | 14 | 24000 | Thu Jan 01 00:00:00 2099 +(1 row) + +UPDATE t2_ref + SET c15 = '2099-01-01 00:00:00'::timestamp + WHERE EXISTS ( + SELECT 1 + FROM t4_pg + ); +ERROR: relation t4_pg is not distributed +SELECT 't2_ref after UPDATE' AS label, * FROM t2_ref; + label | vkey | pkey | c15 +--------------------------------------------------------------------- + t2_ref after UPDATE | 14 | 24000 | Thu Jan 01 00:00:00 2099 +(1 row) + +-- Cleanup +SET client_min_messages TO WARNING; +DROP SCHEMA issue_7891 CASCADE; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index db2794171..429806dd6 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -103,7 +103,7 @@ test: multi_dropped_column_aliases foreign_key_restriction_enforcement test: binary_protocol test: alter_table_set_access_method test: alter_distributed_table -test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 +test: issue_5248 issue_5099 issue_5763 issue_6543 issue_6758 issue_7477 issue_7891 test: object_propagation_debug test: undistribute_table test: run_command_on_all_nodes diff --git a/src/test/regress/sql/issue_7891.sql b/src/test/regress/sql/issue_7891.sql new file mode 100644 index 000000000..2002cb9b3 --- /dev/null +++ b/src/test/regress/sql/issue_7891.sql @@ -0,0 +1,169 @@ +-- This test validates that the query planner correctly handles nested subqueries involving both a +-- local table (t4_pg) and a reference table (t2_ref). The steps are as follows: +-- +-- 1. A dedicated schema (issue_7891) is created, and three tables (t2_ref, t4_pg, t6_pg) are set up. +-- 2. The table t2_ref is designated as a reference table using the create_reference_table() function. +-- 3. Sample data is inserted into all tables. +-- 4. An UPDATE is executed on t6_pg. The update uses an EXISTS clause with a nested subquery: +-- - The outer subquery iterates over every row in t4_pg. +-- - The inner subquery selects c15 from t2_ref. +-- 5. The update should occur if the nested subquery returns any row, effectively updating t6_pg's vkey to 43. +-- 6. The final state of t6_pg is displayed to confirm that the update was applied. +-- +-- Note: This test was originally designed to detect a planner bug where the nested structure might +-- lead to an incorrect plan (such as a 0-task plan), ensuring proper handling of reference and local tables. +-- https://github.com/citusdata/citus/issues/7891 +CREATE SCHEMA issue_7891; +SET search_path TO issue_7891; + +-- Create tables +CREATE TABLE t2_ref ( + vkey INT, + pkey INT, + c15 TIMESTAMP +); + +CREATE TABLE t2_ref2 ( + vkey INT, + pkey INT, + c15 TIMESTAMP +); + + +CREATE TABLE t4_pg ( + vkey INT, + pkey INT, + c22 NUMERIC, + c23 TEXT, + c24 TIMESTAMP +); + +CREATE TABLE t6_pg ( + vkey INT, + pkey INT, + c26 TEXT +); + +-- Mark t2_ref and t2_ref2 as a reference table +SELECT create_reference_table('t2_ref'); +SELECT create_reference_table('t2_ref2'); + +-- Insert sample data +INSERT INTO t6_pg (vkey, pkey, c26) VALUES + (2, 12000, 'initial'), + (3, 13000, 'will_be_deleted'), + (4, 14000, 'to_merge'); +INSERT INTO t4_pg (vkey, pkey, c22, c23, c24) + VALUES (5, 15000, 0.0, ']]?', MAKE_TIMESTAMP(2071, 10, 26, 16, 20, 5)); +INSERT INTO t2_ref (vkey, pkey, c15) + VALUES (14, 24000, NULL::timestamp); + +-- Show initial data +SELECT 't6_pg before' AS label, * FROM t6_pg; +SELECT 't4_pg data' AS label, * FROM t4_pg; +SELECT 't2_ref data' AS label, * FROM t2_ref; + +-- +-- The problematic query: update t6_pg referencing t4_pg and sub-subquery on t2_ref. +-- Historically might produce a 0-task plan if the planner incorrectly fails to +-- treat t4_pg/t2_ref as local/reference. +-- + +-- The outer subquery iterates over every row in table t4_pg. +UPDATE t6_pg + SET vkey = 43 + WHERE EXISTS ( + SELECT (SELECT c15 FROM t2_ref) + FROM t4_pg +); + +SELECT 't6_pg after' AS label, * FROM t6_pg; + +-- +-- DELETE with a similar nested subquery approach +-- Here, let's delete any rows for which t4_pg is non-empty (like a trivial check). +-- We'll specifically target the row with c26='will_be_deleted' to confirm it's removed. +-- +DELETE FROM t6_pg + WHERE EXISTS ( + SELECT (SELECT c15 FROM t2_ref) + FROM t4_pg + ) + AND c26 = 'will_be_deleted'; + +SELECT 't6_pg after DELETE' AS label, * FROM t6_pg; + +-- +-- We'll merge from t4_pg into t6_pg. The merge will update c26 for pkey=14000. +-- +-- Anticipate an error indicating non-IMMUTABLE functions are not supported in MERGE statements on distributed tables. +-- Retain this comment to highlight the current limitation. +-- +MERGE INTO t6_pg AS tgt +USING t4_pg AS src +ON (tgt.pkey = 14000) +WHEN MATCHED THEN + UPDATE SET c26 = 'merged_' || (SELECT pkey FROM t2_ref WHERE pkey=24000 LIMIT 1) +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c26) + VALUES (99, src.pkey, 'inserted_via_merge'); + +MERGE INTO t2_ref AS tgt +USING t4_pg AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c15 = '2088-01-01 00:00:00'::timestamp +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c15) + VALUES (src.vkey, src.pkey, '2099-12-31 23:59:59'::timestamp); + +-- Show the final state of t2_ref: +SELECT 't2_ref after MERGE (using t4_pg)' AS label, * FROM t2_ref; + +MERGE INTO t2_ref2 AS tgt +USING t2_ref AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c15 = '2077-07-07 07:07:07'::timestamp +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c15) + VALUES (src.vkey, src.pkey, '2066-06-06 06:06:06'::timestamp); + +-- Show the final state of t2_ref2: +SELECT 't2_ref2 after MERGE (using t2_ref)' AS label, * FROM t2_ref2; + + +MERGE INTO t6_pg AS tgt +USING t4_pg AS src + ON (tgt.pkey = src.pkey) +WHEN MATCHED THEN + UPDATE SET c26 = 'merged_value' +WHEN NOT MATCHED THEN + INSERT (vkey, pkey, c26) + VALUES (src.vkey, src.pkey, 'inserted_via_merge'); + +SELECT 't6_pg after MERGE' AS label, * FROM t6_pg; + +-- +-- Update the REFERENCE table itself and verify the change +-- This is to ensure that the reference table is correctly handled. + +UPDATE t2_ref + SET c15 = '2099-01-01 00:00:00'::timestamp + WHERE pkey = 24000; + +SELECT 't2_ref after self-update' AS label, * FROM t2_ref; + + +UPDATE t2_ref + SET c15 = '2099-01-01 00:00:00'::timestamp + WHERE EXISTS ( + SELECT 1 + FROM t4_pg + ); + +SELECT 't2_ref after UPDATE' AS label, * FROM t2_ref; + +-- Cleanup +SET client_min_messages TO WARNING; +DROP SCHEMA issue_7891 CASCADE; From 86107ca191f21ed202f85bcb507b5a4b8130fc59 Mon Sep 17 00:00:00 2001 From: Colm Date: Thu, 27 Feb 2025 10:54:39 +0000 Subject: [PATCH 154/155] #7782 - catch when Postgres planning removes all Citus tables (#7907) DESCRIPTION: fix a planning error caused by a redundant WHERE clause Fix a Citus planning glitch that occurs in a DML query when the WHERE clause of the query is of the form: ` WHERE true OR ` and this is the only place in the query referencing a citus table. Postgres' standard planner transforms the WHERE clause to: ` WHERE true ` So the query now has no citus tables, confusing the Citus planner as described in issues #7782 and #7783. The fix is to check, after Postgres standard planner, if the Query has been transformed as shown, and re-run the check of whether or not the query needs distributed planning. --- .../distributed/planner/distributed_planner.c | 48 +++++++- .../regress/expected/subquery_in_where.out | 110 ++++++++++++++++++ src/test/regress/sql/subquery_in_where.sql | 84 +++++++++++++ 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index a388c767c..36da10e8c 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -151,7 +151,10 @@ static RouterPlanType GetRouterPlanType(Query *query, bool hasUnresolvedParams); static void ConcatenateRTablesAndPerminfos(PlannedStmt *mainPlan, PlannedStmt *concatPlan); - +static bool CheckPostPlanDistribution(bool isDistributedQuery, + Query *origQuery, + List *rangeTableList, + Query *plannedQuery); /* Distributed planner hook */ PlannedStmt * @@ -272,6 +275,11 @@ distributed_planner(Query *parse, planContext.plan = standard_planner(planContext.query, NULL, planContext.cursorOptions, planContext.boundParams); + needsDistributedPlanning = CheckPostPlanDistribution(needsDistributedPlanning, + planContext.originalQuery, + rangeTableList, + planContext.query); + if (needsDistributedPlanning) { result = PlanDistributedStmt(&planContext, rteIdCounter); @@ -2729,3 +2737,41 @@ WarnIfListHasForeignDistributedTable(List *rangeTableList) } } } + + +static bool +CheckPostPlanDistribution(bool isDistributedQuery, + Query *origQuery, List *rangeTableList, + Query *plannedQuery) +{ + if (isDistributedQuery) + { + Node *origQuals = origQuery->jointree->quals; + Node *plannedQuals = plannedQuery->jointree->quals; + + #if PG_VERSION_NUM >= PG_VERSION_17 + if (IsMergeQuery(origQuery)) + { + origQuals = origQuery->mergeJoinCondition; + plannedQuals = plannedQuery->mergeJoinCondition; + } + #endif + + /* + * The WHERE quals have been eliminated by the Postgres planner, possibly by + * an OR clause that was simplified to TRUE. In such cases, we need to check + * if the planned query still requires distributed planning. + */ + if (origQuals != NULL && plannedQuals == NULL) + { + List *rtesPostPlan = ExtractRangeTableEntryList(plannedQuery); + if (list_length(rtesPostPlan) < list_length(rangeTableList)) + { + isDistributedQuery = ListContainsDistributedTableRTE( + rtesPostPlan, NULL); + } + } + } + + return isDistributedQuery; +} diff --git a/src/test/regress/expected/subquery_in_where.out b/src/test/regress/expected/subquery_in_where.out index 990c29084..901954265 100644 --- a/src/test/regress/expected/subquery_in_where.out +++ b/src/test/regress/expected/subquery_in_where.out @@ -1146,7 +1146,117 @@ WHERE (SELECT COUNT(DISTINCT e1.value_2) (1 row) +-- Test redundant WHERE clause (fix #7782, #7783) +CREATE TABLE t0 (vkey int4, pkey int4, c0 timestamp); +CREATE TABLE t1 (vkey int4, pkey int4, c4 timestamp, c5 text, c6 text); +CREATE TABLE t3 (vkey int4, pkey int4, c9 timestamp); +CREATE TABLE t7 (vkey int4, pkey int4); +-- DEBUG messages not needed for these tests SET client_min_messages TO DEFAULT; +INSERT INTO t0 (vkey, pkey, c0) values +(3, 13000, make_timestamp(2032, 9, 4, 13, 38, 0)); +INSERT INTO t7 (vkey, pkey) values +(3, 59525); +SELECT create_reference_table('t1'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('t3', 'c9'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +UPDATE t0 set vkey = 117 +where (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t3.pkey from t3 where false + ))) + or TRUE; +-- Local table t0 is updated +SELECT vkey, pkey, c0 FROM t0; + vkey | pkey | c0 +--------------------------------------------------------------------- + 117 | 13000 | Sat Sep 04 13:38:00 2032 +(1 row) + +-- MERGE command with redundant join can be planned locally +EXPLAIN (costs off, timing off) +MERGE INTO t0 USING t7 ON + (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t1.pkey from t1 where false + ))) + or TRUE +WHEN MATCHED THEN + UPDATE SET vkey = 113; + QUERY PLAN +--------------------------------------------------------------------- + Merge on t0 + -> Nested Loop + -> Seq Scan on t7 + -> Materialize + -> Seq Scan on t0 +(5 rows) + +-- UPDATE via MERGE with redundant join clause: +MERGE INTO t0 USING t7 ON + (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t1.pkey from t1 where false + ))) + or TRUE +WHEN MATCHED THEN + UPDATE SET vkey = 113; +-- Local table t0 is updated +SELECT vkey, pkey, c0 FROM t0; + vkey | pkey | c0 +--------------------------------------------------------------------- + 113 | 13000 | Sat Sep 04 13:38:00 2032 +(1 row) + +DELETE FROM t0 +where TRUE or (((t0.vkey) >= (select + pg_catalog.regexp_count(ref_0.c5, ref_0.c6) + from t1 as ref_0 where true))); +-- Local table t0 is now empty (0 rows) +SELECT vkey, pkey, c0 FROM t0; + vkey | pkey | c0 +--------------------------------------------------------------------- +(0 rows) + +INSERT INTO t3 (vkey, pkey, c9) values +(3, 13000, make_timestamp(2032, 9, 4, 13, 38, 0)); +-- Distributed table update with redundant WHERE +UPDATE t3 set vkey = 117 +where (((t3.pkey) in (select t1.vkey from t1 where false + union all + select t0.pkey from t0 join t7 on t0.pkey=t7.vkey where false + ))) + or TRUE; +SELECT vkey, pkey FROM t3; + vkey | pkey +--------------------------------------------------------------------- + 117 | 13000 +(1 row) + +-- Distributed table delete with redundant WHERE +DELETE FROM t3 +where TRUE or (((t3.vkey) >= (select + pg_catalog.regexp_count(ref_0.c5, ref_0.c6) + from t1 as ref_0 where true)) and (select max(vkey) from t0) > 0); +-- Distributed table t3 is now empty +SELECT vkey, pkey FROM t3; + vkey | pkey +--------------------------------------------------------------------- +(0 rows) + DROP TABLE local_table; +DROP TABLE t0; +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t7; DROP SCHEMA subquery_in_where CASCADE; SET search_path TO public; diff --git a/src/test/regress/sql/subquery_in_where.sql b/src/test/regress/sql/subquery_in_where.sql index 8316508b7..ebdb60890 100644 --- a/src/test/regress/sql/subquery_in_where.sql +++ b/src/test/regress/sql/subquery_in_where.sql @@ -847,8 +847,92 @@ WHERE (SELECT COUNT(DISTINCT e1.value_2) WHERE e1.user_id = u1.user_id ) > 115 AND false; +-- Test redundant WHERE clause (fix #7782, #7783) +CREATE TABLE t0 (vkey int4, pkey int4, c0 timestamp); +CREATE TABLE t1 (vkey int4, pkey int4, c4 timestamp, c5 text, c6 text); +CREATE TABLE t3 (vkey int4, pkey int4, c9 timestamp); +CREATE TABLE t7 (vkey int4, pkey int4); + +-- DEBUG messages not needed for these tests SET client_min_messages TO DEFAULT; +INSERT INTO t0 (vkey, pkey, c0) values +(3, 13000, make_timestamp(2032, 9, 4, 13, 38, 0)); + +INSERT INTO t7 (vkey, pkey) values +(3, 59525); + +SELECT create_reference_table('t1'); +SELECT create_distributed_table('t3', 'c9'); + +UPDATE t0 set vkey = 117 +where (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t3.pkey from t3 where false + ))) + or TRUE; + +-- Local table t0 is updated +SELECT vkey, pkey, c0 FROM t0; + +-- MERGE command with redundant join can be planned locally +EXPLAIN (costs off, timing off) +MERGE INTO t0 USING t7 ON + (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t1.pkey from t1 where false + ))) + or TRUE +WHEN MATCHED THEN + UPDATE SET vkey = 113; + +-- UPDATE via MERGE with redundant join clause: +MERGE INTO t0 USING t7 ON + (((t0.pkey) in (select t7.vkey from t7 where false + union all + select t1.pkey from t1 where false + ))) + or TRUE +WHEN MATCHED THEN + UPDATE SET vkey = 113; + +-- Local table t0 is updated +SELECT vkey, pkey, c0 FROM t0; + +DELETE FROM t0 +where TRUE or (((t0.vkey) >= (select + pg_catalog.regexp_count(ref_0.c5, ref_0.c6) + from t1 as ref_0 where true))); + +-- Local table t0 is now empty (0 rows) +SELECT vkey, pkey, c0 FROM t0; + +INSERT INTO t3 (vkey, pkey, c9) values +(3, 13000, make_timestamp(2032, 9, 4, 13, 38, 0)); + +-- Distributed table update with redundant WHERE +UPDATE t3 set vkey = 117 +where (((t3.pkey) in (select t1.vkey from t1 where false + union all + select t0.pkey from t0 join t7 on t0.pkey=t7.vkey where false + ))) + or TRUE; + +SELECT vkey, pkey FROM t3; + +-- Distributed table delete with redundant WHERE +DELETE FROM t3 +where TRUE or (((t3.vkey) >= (select + pg_catalog.regexp_count(ref_0.c5, ref_0.c6) + from t1 as ref_0 where true)) and (select max(vkey) from t0) > 0); + +-- Distributed table t3 is now empty +SELECT vkey, pkey FROM t3; + DROP TABLE local_table; +DROP TABLE t0; +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t7; DROP SCHEMA subquery_in_where CASCADE; SET search_path TO public; From 43f3786c1fd4710285cac087e43c7b0c33964a0a Mon Sep 17 00:00:00 2001 From: Muhammad Usama Date: Tue, 4 Mar 2025 15:11:01 +0500 Subject: [PATCH 155/155] Fix Deadlock with transaction recovery is possible during Citus upgrades (#7910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESCRIPTION: Fixes deadlock with transaction recovery that is possible during Citus upgrades. Fixes #7875. This commit addresses two interrelated deadlock issues uncovered during Citus upgrades: 1. Local Deadlock: - **Problem:** In `RecoverWorkerTransactions()`, a new connection is created for each worker node to perform transaction recovery by locking the `pg_dist_transaction` catalog table until the end of the transaction. When `RecoverTwoPhaseCommits()` calls this function for each worker node, the order of acquiring locks on `pg_dist_authinfo` and `pg_dist_transaction` can alternate. This reversal can lead to a deadlock if any concurrent process requires locks on these tables. - **Fix:** Pre-establish all worker node connections upfront so that `RecoverWorkerTransactions()` operates with a single, consistent connection. This ensures that locks on `pg_dist_authinfo` and `pg_dist_transaction` are always acquired in the correct order, thereby preventing the local deadlock. 2. Distributed Deadlock: - **Problem:** After resolving the local deadlock, a distributed deadlock issue emerges. The maintenance daemon calls `RecoverWorkerTransactions()` on each worker node— including the local node—which leads to a complex locking sequence: - A RowExclusiveLock is taken on the `pg_dist_transaction` table in `RecoverWorkerTransactions()`. - An update extension then tries to acquire an AccessExclusiveLock on the same table, getting blocked by the RowExclusiveLock. - A subsequent query (e.g., a SELECT on `pg_prepared_xacts`) issued using a separate connection on the local node gets blocked due to locks held during a call to `BuildCitusTableCacheEntry()`. - The maintenance daemon waits for this query, resulting in a circular wait and stalling the entire cluster. - **Fix:** Avoid cache lookups for internal PostgreSQL tables by implementing an early bailout for relation IDs below `FirstNormalObjectId` (system objects). This eliminates unnecessary calls to `BuildCitusTableCache`, reducing lock contention and mitigating the distributed deadlock. Furthermore, this optimization improves performance in fast connect→query_catalog→disconnect cycles by eliminating redundant cache creation and lookups. 3. Also reverts the commit that disabled the relevant test cases. --- .../distributed/metadata/metadata_cache.c | 12 +++++ .../transaction/transaction_recovery.c | 51 +++++++++++++++++-- .../citus_tests/upgrade/citus_upgrade_test.py | 18 ------- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 3451415d1..eb5a0c22d 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -660,6 +660,18 @@ GetTableTypeName(Oid tableId) bool IsCitusTable(Oid relationId) { + /* + * PostgreSQL's OID generator assigns user operation OIDs starting + * from FirstNormalObjectId. This means no user object can have + * an OID lower than FirstNormalObjectId. Therefore, if the + * relationId is less than FirstNormalObjectId + * (i.e. in PostgreSQL's reserved range), we can immediately + * return false, since such objects cannot be Citus tables. + */ + if (relationId < FirstNormalObjectId) + { + return false; + } return LookupCitusTableCacheEntry(relationId) != NULL; } diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index f25823b30..4a1503b55 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -53,7 +53,8 @@ PG_FUNCTION_INFO_V1(recover_prepared_transactions); /* Local functions forward declarations */ -static int RecoverWorkerTransactions(WorkerNode *workerNode); +static int RecoverWorkerTransactions(WorkerNode *workerNode, + MultiConnection *connection); static List * PendingWorkerTransactionList(MultiConnection *connection); static bool IsTransactionInProgress(HTAB *activeTransactionNumberSet, char *preparedTransactionName); @@ -123,10 +124,51 @@ RecoverTwoPhaseCommits(void) LockTransactionRecovery(ShareUpdateExclusiveLock); List *workerList = ActivePrimaryNodeList(NoLock); + List *workerConnections = NIL; WorkerNode *workerNode = NULL; + MultiConnection *connection = NULL; + + /* + * Pre-establish all connections to worker nodes. + * + * We do this to enforce a consistent lock acquisition order and prevent deadlocks. + * Currently, during extension updates, we take strong locks on the Citus + * catalog tables in a specific order: first on pg_dist_authinfo, then on + * pg_dist_transaction. It's critical that any operation locking these two + * tables adheres to this order, or a deadlock could occur. + * + * Note that RecoverWorkerTransactions() retains its lock until the end + * of the transaction, while GetNodeConnection() releases its lock after + * the catalog lookup. So when there are multiple workers in the active primary + * node list, the lock acquisition order may reverse in subsequent iterations + * of the loop calling RecoverWorkerTransactions(), increasing the risk + * of deadlock. + * + * By establishing all worker connections upfront, we ensure that + * RecoverWorkerTransactions() deals with a single distributed catalog table, + * thereby preventing deadlocks regardless of the lock acquisition sequence + * used in the upgrade extension script. + */ + foreach_declared_ptr(workerNode, workerList) { - recoveredTransactionCount += RecoverWorkerTransactions(workerNode); + int connectionFlags = 0; + char *nodeName = workerNode->workerName; + int nodePort = workerNode->workerPort; + + connection = GetNodeConnection(connectionFlags, nodeName, nodePort); + Assert(connection != NULL); + + /* + * We don't verify connection validity here. + * Instead, RecoverWorkerTransactions() performs the necessary + * sanity checks on the connection state. + */ + workerConnections = lappend(workerConnections, connection); + } + forboth_ptr(workerNode, workerList, connection, workerConnections) + { + recoveredTransactionCount += RecoverWorkerTransactions(workerNode, connection); } return recoveredTransactionCount; @@ -138,7 +180,7 @@ RecoverTwoPhaseCommits(void) * started by this node on the specified worker. */ static int -RecoverWorkerTransactions(WorkerNode *workerNode) +RecoverWorkerTransactions(WorkerNode *workerNode, MultiConnection *connection) { int recoveredTransactionCount = 0; @@ -156,8 +198,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode) bool recoveryFailed = false; - int connectionFlags = 0; - MultiConnection *connection = GetNodeConnection(connectionFlags, nodeName, nodePort); + Assert(connection != NULL); if (connection->pgConn == NULL || PQstatus(connection->pgConn) != CONNECTION_OK) { ereport(WARNING, (errmsg("transaction recovery cannot connect to %s:%d", diff --git a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py index c25a34482..1ab448031 100755 --- a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py +++ b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py @@ -62,16 +62,10 @@ def run_citus_upgrade_tests(config, before_upgrade_schedule, after_upgrade_sched install_citus(config.post_tar_path) - # disable 2pc recovery for all nodes to work around https://github.com/citusdata/citus/issues/7875 - disable_2pc_recovery_for_all_nodes(config.bindir, config) - restart_databases(config.bindir, config.datadir, config.mixed_mode, config) run_alter_citus(config.bindir, config.mixed_mode, config) verify_upgrade(config, config.mixed_mode, config.node_name_to_ports.values()) - # re-enable 2pc recovery for all nodes - enable_2pc_recovery_for_all_nodes(config.bindir, config) - run_test_on_coordinator(config, after_upgrade_schedule) remove_citus(config.post_tar_path) @@ -152,18 +146,6 @@ def restart_database(pg_path, abs_data_path, node_name, node_ports, logfile_pref subprocess.run(command, check=True) -def disable_2pc_recovery_for_all_nodes(pg_path, config): - for port in config.node_name_to_ports.values(): - utils.psql(pg_path, port, "ALTER SYSTEM SET citus.recover_2pc_interval TO -1;") - utils.psql(pg_path, port, "SELECT pg_reload_conf();") - - -def enable_2pc_recovery_for_all_nodes(pg_path, config): - for port in config.node_name_to_ports.values(): - utils.psql(pg_path, port, "ALTER SYSTEM RESET citus.recover_2pc_interval;") - utils.psql(pg_path, port, "SELECT pg_reload_conf();") - - def run_alter_citus(pg_path, mixed_mode, config): for port in config.node_name_to_ports.values(): if mixed_mode and port in (