diff --git a/.circleci/config.yml b/.circleci/config.yml index 08cfbc6ce..08fcc2922 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,16 +6,19 @@ orbs: parameters: image_suffix: type: string - default: '-vbab548a' + default: '-v0c8d80c' pg14_version: type: string - default: '14.8' + default: '14.9' pg15_version: type: string - default: '15.3' + default: '15.4' + pg16_version: + type: string + default: '16beta3' upgrade_pg_versions: type: string - default: '14.8-15.3' + default: '14.9-15.4-16beta3' style_checker_tools_version: type: string default: '0.8.18' @@ -721,6 +724,10 @@ workflows: name: build-15 pg_major: 15 image_tag: '<< pipeline.parameters.pg15_version >>' + - build: + name: build-16 + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' - check-style - check-sql-snapshots @@ -871,6 +878,79 @@ workflows: image: citus/failtester make: check-failure + - test-citus: &test-citus-16 + name: 'test-16_check-split' + make: check-split + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' + requires: [build-16] + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise' + make: check-enterprise + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise-isolation' + make: check-enterprise-isolation + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise-isolation-logicalrep-1' + make: check-enterprise-isolation-logicalrep-1 + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise-isolation-logicalrep-2' + make: check-enterprise-isolation-logicalrep-2 + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise-isolation-logicalrep-3' + make: check-enterprise-isolation-logicalrep-3 + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-enterprise-failure' + image: citus/failtester + make: check-enterprise-failure + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-multi' + make: check-multi + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-multi-1' + make: check-multi-1 + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-mx' + make: check-multi-mx + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-vanilla' + make: check-vanilla + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-isolation' + make: check-isolation + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-operations' + make: check-operations + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-follower-cluster' + make: check-follower-cluster + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-columnar' + make: check-columnar + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-columnar-isolation' + make: check-columnar-isolation + - test-citus: + <<: *test-citus-16 + name: 'test-16_check-failure' + image: citus/failtester + make: check-failure + - test-pytest: name: 'test-14_pytest' pg_major: 14 @@ -883,6 +963,12 @@ workflows: image_tag: '<< pipeline.parameters.pg15_version >>' requires: [build-15] + - test-pytest: + name: 'test-16_pytest' + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' + requires: [build-16] + - tap-test-citus: name: 'test-15_tap-cdc' suite: cdc @@ -890,6 +976,13 @@ workflows: image_tag: '<< pipeline.parameters.pg15_version >>' requires: [build-15] + - tap-test-citus: + name: 'test-16_tap-cdc' + suite: cdc + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' + requires: [build-16] + - test-arbitrary-configs: name: 'test-14_check-arbitrary-configs' pg_major: 14 @@ -902,6 +995,12 @@ workflows: image_tag: '<< pipeline.parameters.pg15_version >>' requires: [build-15] + - test-arbitrary-configs: + name: 'test-16_check-arbitrary-configs' + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' + requires: [build-16] + - test-query-generator: name: 'test-14_check-query-generator' pg_major: 14 @@ -914,6 +1013,12 @@ workflows: image_tag: '<< pipeline.parameters.pg15_version >>' requires: [build-15] + - test-query-generator: + name: 'test-16_check-query-generator' + pg_major: 16 + image_tag: '<< pipeline.parameters.pg16_version >>' + requires: [build-16] + - test-pg-upgrade: name: 'test-14-15_check-pg-upgrade' old_pg_major: 14 @@ -921,6 +1026,20 @@ workflows: image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' requires: [build-14, build-15] + - test-pg-upgrade: + name: 'test-15-16_check-pg-upgrade' + old_pg_major: 15 + new_pg_major: 16 + image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' + requires: [build-15, build-16] + + - test-pg-upgrade: + name: 'test-14-16_check-pg-upgrade' + old_pg_major: 14 + new_pg_major: 16 + image_tag: '<< pipeline.parameters.upgrade_pg_versions >>' + requires: [build-14, build-16] + - test-citus-upgrade: name: test-14_check-citus-upgrade pg_major: 14 @@ -967,7 +1086,28 @@ workflows: - test-15_check-split - test-15_check-arbitrary-configs - test-15_check-query-generator + - test-16_check-multi + - test-16_check-multi-1 + - test-16_check-mx + - test-16_check-vanilla + - test-16_check-isolation + - test-16_check-operations + - test-16_check-follower-cluster + - test-16_check-columnar + - test-16_check-columnar-isolation + - test-16_check-failure + - test-16_check-enterprise + - test-16_check-enterprise-isolation + - test-16_check-enterprise-isolation-logicalrep-1 + - test-16_check-enterprise-isolation-logicalrep-2 + - test-16_check-enterprise-isolation-logicalrep-3 + - test-16_check-enterprise-failure + - test-16_check-split + - test-16_check-arbitrary-configs + - test-16_check-query-generator - test-14-15_check-pg-upgrade + - test-15-16_check-pg-upgrade + - test-14-16_check-pg-upgrade - test-14_check-citus-upgrade - ch_benchmark: diff --git a/configure b/configure index 9eb7abee2..c98037d89 100755 --- a/configure +++ b/configure @@ -2588,7 +2588,7 @@ fi if test "$with_pg_version_check" = no; then { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;} -elif test "$version_num" != '14' -a "$version_num" != '15'; then +elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.ac b/configure.ac index 7aa2fcb4a..7e5619857 100644 --- a/configure.ac +++ b/configure.ac @@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check) if test "$with_pg_version_check" = no; then AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)]) -elif test "$version_num" != '14' -a "$version_num" != '15'; then +elif test "$version_num" != '14' -a "$version_num" != '15' -a "$version_num" != '16'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) diff --git a/src/backend/distributed/cdc/cdc_decoder_utils.c b/src/backend/distributed/cdc/cdc_decoder_utils.c index d781f171c..f5b23aa12 100644 --- a/src/backend/distributed/cdc/cdc_decoder_utils.c +++ b/src/backend/distributed/cdc/cdc_decoder_utils.c @@ -72,7 +72,7 @@ DistShardRelationId(void) /* - * DistShardRelationId returns the relation id of the pg_dist_shard + * DistShardShardidIndexId returns the relation id of the pg_dist_shard_shardid_index */ static Oid DistShardShardidIndexId(void) @@ -87,7 +87,7 @@ DistShardShardidIndexId(void) /* - * DistShardRelationId returns the relation id of the pg_dist_shard + * DistPartitionRelationId returns the relation id of the pg_dist_partition */ static Oid DistPartitionRelationId(void) @@ -376,7 +376,8 @@ CdcIsReferenceTableViaCatalog(Oid relationId) * A table is a reference table when its partition method is 'none' * and replication model is 'two phase commit' */ - return partitionMethodChar == 'n' && replicationModelChar == 't'; + return partitionMethodChar == DISTRIBUTE_BY_NONE && + replicationModelChar == REPLICATION_MODEL_2PC; } diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index ae0a939a5..8271cc4f4 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -561,7 +561,7 @@ ReindexStmtFindRelationOid(ReindexStmt *reindexStmt, bool missingOk) { relationId = RangeVarGetRelidExtended(reindexStmt->relation, lockmode, (missingOk) ? RVR_MISSING_OK : 0, - RANGE_VAR_TABLE_CALLBACK, NULL); + RangeVarCallbackOwnsTable, NULL); } return relationId; diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 4e274d270..99ce4fb9f 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -819,12 +819,15 @@ GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options) grantRoleStmt->grantee_roles = list_make1(roleSpec); } -#if PG_VERSION_NUM < PG_VERSION_16 if (strcmp(option->defname, "adminmembers") == 0) { +#if PG_VERSION_NUM >= PG_VERSION_16 + DefElem *opt = makeDefElem("admin", (Node *) makeBoolean(true), -1); + grantRoleStmt->opt = list_make1(opt); +#else grantRoleStmt->admin_opt = true; - } #endif + } stmts = lappend(stmts, grantRoleStmt); } @@ -871,7 +874,13 @@ GenerateGrantRoleStmtsOfRole(Oid roleid) grantRoleStmt->grantor = NULL; -#if PG_VERSION_NUM < PG_VERSION_16 +#if PG_VERSION_NUM >= PG_VERSION_16 + if (membership->admin_option) + { + DefElem *opt = makeDefElem("admin", (Node *) makeBoolean(true), -1); + grantRoleStmt->opt = list_make1(opt); + } +#else grantRoleStmt->admin_opt = membership->admin_option; #endif diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index 6bc76b7b8..ee03aeae1 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -42,6 +42,9 @@ typedef struct CitusVacuumParams VacOptValue truncate; VacOptValue index_cleanup; int nworkers; +#if PG_VERSION_NUM >= PG_VERSION_16 + int ring_size; +#endif } CitusVacuumParams; /* Local functions forward declarations for processing distributed table commands */ @@ -318,13 +321,26 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) } /* if no flags remain, exit early */ - if (vacuumFlags == 0 && - vacuumParams.truncate == VACOPTVALUE_UNSPECIFIED && - vacuumParams.index_cleanup == VACOPTVALUE_UNSPECIFIED && - vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET - ) +#if PG_VERSION_NUM >= PG_VERSION_16 + if (vacuumFlags & VACOPT_PROCESS_TOAST && + vacuumFlags & VACOPT_PROCESS_MAIN) { - return vacuumPrefix->data; + /* process toast and process main are true by default */ + if (((vacuumFlags & ~VACOPT_PROCESS_TOAST) & ~VACOPT_PROCESS_MAIN) == 0 && + vacuumParams.ring_size == -1 && +#else + if (vacuumFlags & VACOPT_PROCESS_TOAST) + { + /* process toast is true by default */ + if ((vacuumFlags & ~VACOPT_PROCESS_TOAST) == 0 && +#endif + vacuumParams.truncate == VACOPTVALUE_UNSPECIFIED && + vacuumParams.index_cleanup == VACOPTVALUE_UNSPECIFIED && + vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET + ) + { + return vacuumPrefix->data; + } } /* otherwise, handle options */ @@ -360,11 +376,33 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) appendStringInfoString(vacuumPrefix, "SKIP_LOCKED,"); } - if (vacuumFlags & VACOPT_PROCESS_TOAST) + if (!(vacuumFlags & VACOPT_PROCESS_TOAST)) { - appendStringInfoString(vacuumPrefix, "PROCESS_TOAST,"); + appendStringInfoString(vacuumPrefix, "PROCESS_TOAST FALSE,"); } +#if PG_VERSION_NUM >= PG_VERSION_16 + if (!(vacuumFlags & VACOPT_PROCESS_MAIN)) + { + appendStringInfoString(vacuumPrefix, "PROCESS_MAIN FALSE,"); + } + + if (vacuumFlags & VACOPT_SKIP_DATABASE_STATS) + { + appendStringInfoString(vacuumPrefix, "SKIP_DATABASE_STATS,"); + } + + if (vacuumFlags & VACOPT_ONLY_DATABASE_STATS) + { + appendStringInfoString(vacuumPrefix, "ONLY_DATABASE_STATS,"); + } + + if (vacuumParams.ring_size != -1) + { + appendStringInfo(vacuumPrefix, "BUFFER_USAGE_LIMIT %d,", vacuumParams.ring_size); + } +#endif + if (vacuumParams.truncate != VACOPTVALUE_UNSPECIFIED) { appendStringInfoString(vacuumPrefix, @@ -499,7 +537,14 @@ VacuumStmtParams(VacuumStmt *vacstmt) bool freeze = false; bool full = false; bool disable_page_skipping = false; - bool process_toast = false; + bool process_toast = true; + +#if PG_VERSION_NUM >= PG_VERSION_16 + bool process_main = true; + bool skip_database_stats = false; + bool only_database_stats = false; + params.ring_size = -1; +#endif /* Set default value */ params.index_cleanup = VACOPTVALUE_UNSPECIFIED; @@ -519,6 +564,13 @@ VacuumStmtParams(VacuumStmt *vacstmt) { skip_locked = defGetBoolean(opt); } +#if PG_VERSION_NUM >= PG_VERSION_16 + else if (strcmp(opt->defname, "buffer_usage_limit") == 0) + { + char *vac_buffer_size = defGetString(opt); + parse_int(vac_buffer_size, ¶ms.ring_size, GUC_UNIT_KB, NULL); + } +#endif else if (!vacstmt->is_vacuumcmd) { ereport(ERROR, @@ -543,6 +595,20 @@ VacuumStmtParams(VacuumStmt *vacstmt) { disable_page_skipping = defGetBoolean(opt); } +#if PG_VERSION_NUM >= PG_VERSION_16 + else if (strcmp(opt->defname, "process_main") == 0) + { + process_main = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "skip_database_stats") == 0) + { + skip_database_stats = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "only_database_stats") == 0) + { + only_database_stats = defGetBoolean(opt); + } +#endif else if (strcmp(opt->defname, "process_toast") == 0) { process_toast = defGetBoolean(opt); @@ -613,6 +679,11 @@ VacuumStmtParams(VacuumStmt *vacstmt) (analyze ? VACOPT_ANALYZE : 0) | (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | +#if PG_VERSION_NUM >= PG_VERSION_16 + (process_main ? VACOPT_PROCESS_MAIN : 0) | + (skip_database_stats ? VACOPT_SKIP_DATABASE_STATS : 0) | + (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0) | +#endif (process_toast ? VACOPT_PROCESS_TOAST : 0) | (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0); return params; diff --git a/src/backend/distributed/deparser/deparse_role_stmts.c b/src/backend/distributed/deparser/deparse_role_stmts.c index 620e7606d..4ad4f4c7f 100644 --- a/src/backend/distributed/deparser/deparse_role_stmts.c +++ b/src/backend/distributed/deparser/deparse_role_stmts.c @@ -17,6 +17,7 @@ #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" +#include "distributed/listutils.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" @@ -349,7 +350,20 @@ AppendGrantRoleStmt(StringInfo buf, GrantRoleStmt *stmt) { appendStringInfo(buf, "%s ", stmt->is_grant ? "GRANT" : "REVOKE"); -#if PG_VERSION_NUM < PG_VERSION_16 +#if PG_VERSION_NUM >= PG_VERSION_16 + if (!stmt->is_grant) + { + DefElem *opt = NULL; + foreach_ptr(opt, stmt->opt) + { + if (strcmp(opt->defname, "admin") == 0) + { + appendStringInfo(buf, "ADMIN OPTION FOR "); + break; + } + } + } +#else if (!stmt->is_grant && stmt->admin_opt) { appendStringInfo(buf, "ADMIN OPTION FOR "); @@ -364,7 +378,17 @@ AppendGrantRoleStmt(StringInfo buf, GrantRoleStmt *stmt) if (stmt->is_grant) { -#if PG_VERSION_NUM < PG_VERSION_16 +#if PG_VERSION_NUM >= PG_VERSION_16 + DefElem *opt = NULL; + foreach_ptr(opt, stmt->opt) + { + if (strcmp(opt->defname, "admin") == 0) + { + appendStringInfo(buf, " WITH ADMIN OPTION"); + break; + } + } +#else if (stmt->admin_opt) { appendStringInfo(buf, " WITH ADMIN OPTION"); diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index d0578dfdc..f970cecd1 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -1022,13 +1022,12 @@ GetUndistributableDependency(const ObjectAddress *objectAddress) if (!SupportedDependencyByCitus(dependency)) { /* - * Skip roles and text search templates. - * - * Roles should be handled manually with Citus community whereas text search - * templates should be handled manually in both community and enterprise + * Since we do not yet support distributed TS TEMPLATE and AM objects, we skip + * dependency checks for text search templates. The user is expected to + * manually create the TS TEMPLATE and AM objects. */ - if (getObjectClass(dependency) != OCLASS_ROLE && - getObjectClass(dependency) != OCLASS_TSTEMPLATE) + if (getObjectClass(dependency) != OCLASS_TSTEMPLATE && + getObjectClass(dependency) != OCLASS_AM) { return dependency; } diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 0feb2e2ae..79c210301 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -4987,8 +4987,8 @@ CitusTableTypeIdList(CitusTableType citusTableType) Datum replicationModelDatum = datumArray[Anum_pg_dist_partition_repmodel - 1]; Datum colocationIdDatum = datumArray[Anum_pg_dist_partition_colocationid - 1]; - Oid partitionMethod = DatumGetChar(partMethodDatum); - Oid replicationModel = DatumGetChar(replicationModelDatum); + char partitionMethod = DatumGetChar(partMethodDatum); + char replicationModel = DatumGetChar(replicationModelDatum); uint32 colocationId = DatumGetUInt32(colocationIdDatum); if (IsCitusTableTypeInternal(partitionMethod, replicationModel, colocationId, diff --git a/src/backend/distributed/planner/deparse_shard_query.c b/src/backend/distributed/planner/deparse_shard_query.c index 5743ab1c1..ac37b1399 100644 --- a/src/backend/distributed/planner/deparse_shard_query.c +++ b/src/backend/distributed/planner/deparse_shard_query.c @@ -358,6 +358,11 @@ ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte) subquery->jointree = joinTree; rte->rtekind = RTE_SUBQUERY; +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* no permission checking for this RTE */ + rte->perminfoindex = 0; +#endif rte->subquery = subquery; rte->alias = copyObject(rte->eref); } diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 3b6a8f9f7..65278d1ea 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -56,6 +56,9 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" +#if PG_VERSION_NUM >= PG_VERSION_16 +#include "parser/parse_relation.h" +#endif #include "parser/parsetree.h" #include "parser/parse_type.h" #include "optimizer/optimizer.h" @@ -108,6 +111,7 @@ static int AssignRTEIdentities(List *rangeTableList, int rteIdCounter); static void AssignRTEIdentity(RangeTblEntry *rangeTableEntry, int rteIdentifier); static void AdjustPartitioningForDistributedPlanning(List *rangeTableList, bool setPartitionedTablesInherited); +static bool RTEWentThroughAdjustPartitioning(RangeTblEntry *rangeTableEntry); static PlannedStmt * FinalizeNonRouterPlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan, CustomScan *customScan); @@ -144,6 +148,8 @@ static void WarnIfListHasForeignDistributedTable(List *rangeTableList); static RouterPlanType GetRouterPlanType(Query *query, Query *originalQuery, bool hasUnresolvedParams); +static void ConcatenateRTablesAndPerminfos(PlannedStmt *mainPlan, + PlannedStmt *concatPlan); /* Distributed planner hook */ @@ -494,6 +500,20 @@ AdjustPartitioningForDistributedPlanning(List *rangeTableList, } +/* + * RTEWentThroughAdjustPartitioning returns true if the given rangetableentry + * has been modified through AdjustPartitioningForDistributedPlanning + * function, false otherwise. + */ +static bool +RTEWentThroughAdjustPartitioning(RangeTblEntry *rangeTableEntry) +{ + return (rangeTableEntry->rtekind == RTE_RELATION && + PartitionedTable(rangeTableEntry->relid) && + rangeTableEntry->inh == false); +} + + /* * AssignRTEIdentity assigns the given rteIdentifier to the given range table * entry. @@ -1066,6 +1086,11 @@ CreateDistributedPlan(uint64 planId, bool allowRecursivePlanning, Query *origina /* * Plan subqueries and CTEs that cannot be pushed down by recursively * calling the planner and return the resulting plans to subPlanList. + * Note that GenerateSubplansForSubqueriesAndCTEs will reset perminfoindexes + * for some RTEs in originalQuery->rtable list, while not changing + * originalQuery->rteperminfos. That's fine because we will go through + * standard_planner again, which will adjust things accordingly in + * set_plan_references>add_rtes_to_flat_rtable>add_rte_to_flat_rtable. */ List *subPlanList = GenerateSubplansForSubqueriesAndCTEs(planId, originalQuery, plannerRestrictionContext); @@ -1465,12 +1490,42 @@ FinalizeNonRouterPlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan, finalPlan->utilityStmt = localPlan->utilityStmt; /* add original range table list for access permission checks */ - finalPlan->rtable = list_concat(finalPlan->rtable, localPlan->rtable); + ConcatenateRTablesAndPerminfos(finalPlan, localPlan); return finalPlan; } +static void +ConcatenateRTablesAndPerminfos(PlannedStmt *mainPlan, PlannedStmt *concatPlan) +{ + mainPlan->rtable = list_concat(mainPlan->rtable, concatPlan->rtable); +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* + * concatPlan's range table list is concatenated to mainPlan's range table list + * therefore all the perminfoindexes should be updated to their value + * PLUS the highest perminfoindex in mainPlan's perminfos, which is exactly + * the list length. + */ + int mainPlan_highest_perminfoindex = list_length(mainPlan->permInfos); + + ListCell *lc; + foreach(lc, concatPlan->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + if (rte->perminfoindex != 0) + { + rte->perminfoindex = rte->perminfoindex + mainPlan_highest_perminfoindex; + } + } + + /* finally, concatenate perminfos as well */ + mainPlan->permInfos = list_concat(mainPlan->permInfos, concatPlan->permInfos); +#endif +} + + /* * FinalizeRouterPlan gets a CustomScan node which already wrapped distributed * part of a router plan and sets it as the direct child of the router plan @@ -1502,7 +1557,7 @@ FinalizeRouterPlan(PlannedStmt *localPlan, CustomScan *customScan) routerPlan->rtable = list_make1(remoteScanRangeTableEntry); /* add original range table list for access permission checks */ - routerPlan->rtable = list_concat(routerPlan->rtable, localPlan->rtable); + ConcatenateRTablesAndPerminfos(routerPlan, localPlan); routerPlan->canSetTag = true; routerPlan->relationOids = NIL; @@ -1976,6 +2031,62 @@ multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo, } +/* + * multi_get_relation_info_hook modifies the relation's indexlist + * if necessary, to avoid a crash in PG16 caused by our + * Citus function AdjustPartitioningForDistributedPlanning(). + * + * AdjustPartitioningForDistributedPlanning() is a hack that we use + * to prevent Postgres' standard_planner() to expand all the partitions + * for the distributed planning when a distributed partitioned table + * is queried. It is required for both correctness and performance + * reasons. Although we can eliminate the use of the function for + * the correctness (e.g., make sure that rest of the planner can handle + * partitions), it's performance implication is hard to avoid. Certain + * planning logic of Citus (such as router or query pushdown) relies + * heavily on the relationRestrictionList. If + * AdjustPartitioningForDistributedPlanning() is removed, all the + * partitions show up in the relationRestrictionList, causing high + * planning times for such queries. + */ +void +multi_get_relation_info_hook(PlannerInfo *root, Oid relationObjectId, bool inhparent, + RelOptInfo *rel) +{ + if (!CitusHasBeenLoaded()) + { + return; + } + + Index varno = rel->relid; + RangeTblEntry *rangeTableEntry = planner_rt_fetch(varno, root); + + if (RTEWentThroughAdjustPartitioning(rangeTableEntry)) + { + ListCell *lc = NULL; + foreach(lc, rel->indexlist) + { + IndexOptInfo *indexOptInfo = (IndexOptInfo *) lfirst(lc); + if (get_rel_relkind(indexOptInfo->indexoid) == RELKIND_PARTITIONED_INDEX) + { + /* + * Normally, we should not need this. However, the combination of + * Postgres commit 3c569049b7b502bb4952483d19ce622ff0af5fd6 and + * Citus function AdjustPartitioningForDistributedPlanning() + * forces us to do this. The commit expects partitioned indexes + * to belong to relations with "inh" flag set properly. Whereas, the + * function overrides "inh" flag. To avoid a crash, + * we go over the list of indexinfos and remove all partitioned indexes. + * Partitioned indexes were ignored pre PG16 anyway, we are essentially + * not breaking any logic. + */ + rel->indexlist = foreach_delete_current(rel->indexlist, lc); + } + } + } +} + + /* * TranslatedVars deep copies the translated vars for the given relation index * if there is any append rel list. diff --git a/src/backend/distributed/planner/fast_path_router_planner.c b/src/backend/distributed/planner/fast_path_router_planner.c index 41802ee83..933ee7425 100644 --- a/src/backend/distributed/planner/fast_path_router_planner.c +++ b/src/backend/distributed/planner/fast_path_router_planner.c @@ -136,6 +136,9 @@ GeneratePlaceHolderPlannedStmt(Query *parse) result->stmt_len = parse->stmt_len; result->rtable = copyObject(parse->rtable); +#if PG_VERSION_NUM >= PG_VERSION_16 + result->permInfos = copyObject(parse->rteperminfos); +#endif result->planTree = (Plan *) plan; result->hasReturning = (parse->returningList != NIL); diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index b71d95432..1b7f468f8 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -604,6 +604,22 @@ CreateCombineQueryForRouterPlan(DistributedPlan *distPlan) combineQuery->querySource = QSRC_ORIGINAL; combineQuery->canSetTag = true; combineQuery->rtable = list_make1(rangeTableEntry); + +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* + * This part of the code is more of a sanity check for readability, + * it doesn't really do anything. + * We know that Only relation RTEs and subquery RTEs that were once relation + * RTEs (views) have their perminfoindex set. (see ExecCheckPermissions function) + * DerivedRangeTableEntry sets the rtekind to RTE_FUNCTION + * Hence we should have no perminfos here. + */ + Assert(rangeTableEntry->rtekind == RTE_FUNCTION && + rangeTableEntry->perminfoindex == 0); + combineQuery->rteperminfos = NIL; +#endif + combineQuery->targetList = targetList; combineQuery->jointree = joinTree; return combineQuery; @@ -1533,6 +1549,20 @@ WrapSubquery(Query *subquery) selectAlias, false, true)); outerQuery->rtable = list_make1(newRangeTableEntry); +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* + * This part of the code is more of a sanity check for readability, + * it doesn't really do anything. + * addRangeTableEntryForSubquery doesn't add permission info + * because the range table is set to be RTE_SUBQUERY. + * Hence we should also have no perminfos here. + */ + Assert(newRangeTableEntry->rtekind == RTE_SUBQUERY && + newRangeTableEntry->perminfoindex == 0); + outerQuery->rteperminfos = NIL; +#endif + /* set the FROM expression to the subquery */ RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); newRangeTableRef->rtindex = 1; diff --git a/src/backend/distributed/planner/local_distributed_join_planner.c b/src/backend/distributed/planner/local_distributed_join_planner.c index 2c6a63de1..d93921966 100644 --- a/src/backend/distributed/planner/local_distributed_join_planner.c +++ b/src/backend/distributed/planner/local_distributed_join_planner.c @@ -107,6 +107,7 @@ #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" +#include "parser/parse_relation.h" #include "parser/parsetree.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -136,6 +137,9 @@ typedef struct RangeTableEntryDetails RangeTblEntry *rangeTableEntry; List *requiredAttributeNumbers; bool hasConstantFilterOnUniqueColumn; +#if PG_VERSION_NUM >= PG_VERSION_16 + RTEPermissionInfo *perminfo; +#endif } RangeTableEntryDetails; /* @@ -176,7 +180,8 @@ static bool HasConstantFilterOnUniqueColumn(RangeTblEntry *rangeTableEntry, static ConversionCandidates * CreateConversionCandidates(PlannerRestrictionContext * plannerRestrictionContext, List *rangeTableList, - int resultRTEIdentity); + int resultRTEIdentity, + List *rteperminfos); static void AppendUniqueIndexColumnsToList(Form_pg_index indexForm, List **uniqueIndexes, int flags); static ConversionChoice GetConversionChoice(ConversionCandidates * @@ -205,10 +210,17 @@ RecursivelyPlanLocalTableJoins(Query *query, GetPlannerRestrictionContext(context); List *rangeTableList = query->rtable; +#if PG_VERSION_NUM >= PG_VERSION_16 + List *rteperminfos = query->rteperminfos; +#endif int resultRTEIdentity = ResultRTEIdentity(query); ConversionCandidates *conversionCandidates = CreateConversionCandidates(plannerRestrictionContext, - rangeTableList, resultRTEIdentity); +#if PG_VERSION_NUM >= PG_VERSION_16 + rangeTableList, resultRTEIdentity, rteperminfos); +#else + rangeTableList, resultRTEIdentity, NIL); +#endif ConversionChoice conversionChoise = GetConversionChoice(conversionCandidates, plannerRestrictionContext); @@ -323,7 +335,12 @@ ConvertRTEsToSubquery(List *rangeTableEntryDetailsList, RecursivePlanningContext RangeTblEntry *rangeTableEntry = rangeTableEntryDetails->rangeTableEntry; List *requiredAttributeNumbers = rangeTableEntryDetails->requiredAttributeNumbers; ReplaceRTERelationWithRteSubquery(rangeTableEntry, - requiredAttributeNumbers, context); +#if PG_VERSION_NUM >= PG_VERSION_16 + requiredAttributeNumbers, context, + rangeTableEntryDetails->perminfo); +#else + requiredAttributeNumbers, context, NULL); +#endif } } @@ -530,7 +547,9 @@ RequiredAttrNumbersForRelationInternal(Query *queryToProcess, int rteIndex) */ static ConversionCandidates * CreateConversionCandidates(PlannerRestrictionContext *plannerRestrictionContext, - List *rangeTableList, int resultRTEIdentity) + List *rangeTableList, + int resultRTEIdentity, + List *rteperminfos) { ConversionCandidates *conversionCandidates = palloc0(sizeof(ConversionCandidates)); @@ -564,6 +583,14 @@ CreateConversionCandidates(PlannerRestrictionContext *plannerRestrictionContext, RequiredAttrNumbersForRelation(rangeTableEntry, plannerRestrictionContext); rangeTableEntryDetails->hasConstantFilterOnUniqueColumn = HasConstantFilterOnUniqueColumn(rangeTableEntry, relationRestriction); +#if PG_VERSION_NUM >= PG_VERSION_16 + rangeTableEntryDetails->perminfo = NULL; + if (rangeTableEntry->perminfoindex) + { + rangeTableEntryDetails->perminfo = getRTEPermissionInfo(rteperminfos, + rangeTableEntry); + } +#endif bool referenceOrDistributedTable = IsCitusTableType(rangeTableEntry->relid, REFERENCE_TABLE) || diff --git a/src/backend/distributed/planner/merge_planner.c b/src/backend/distributed/planner/merge_planner.c index 06126cdf3..3cadea23a 100644 --- a/src/backend/distributed/planner/merge_planner.c +++ b/src/backend/distributed/planner/merge_planner.c @@ -15,6 +15,7 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" +#include "parser/parse_relation.h" #include "parser/parsetree.h" #include "tcop/tcopprot.h" #include "utils/lsyscache.h" @@ -777,6 +778,11 @@ ConvertCteRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte) Query *cteQuery = (Query *) copyObject(sourceCte->ctequery); sourceRte->rtekind = RTE_SUBQUERY; +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* sanity check - sourceRte was RTE_CTE previously so it should have no perminfo */ + Assert(sourceRte->perminfoindex == 0); +#endif /* * As we are delinking the CTE from main query, we have to walk through the @@ -827,6 +833,20 @@ ConvertRelationRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, RangeTblEntry *newRangeTableEntry = copyObject(sourceRte); sourceResultsQuery->rtable = list_make1(newRangeTableEntry); +#if PG_VERSION_NUM >= PG_VERSION_16 + sourceResultsQuery->rteperminfos = NIL; + if (sourceRte->perminfoindex) + { + /* create permission info for newRangeTableEntry */ + RTEPermissionInfo *perminfo = getRTEPermissionInfo(mergeQuery->rteperminfos, + sourceRte); + + /* update the sourceResultsQuery's rteperminfos accordingly */ + newRangeTableEntry->perminfoindex = 1; + sourceResultsQuery->rteperminfos = list_make1(perminfo); + } +#endif + /* set the FROM expression to the subquery */ newRangeTableRef->rtindex = SINGLE_RTE_INDEX; sourceResultsQuery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); @@ -852,6 +872,9 @@ ConvertRelationRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, /* replace the function with the constructed subquery */ sourceRte->rtekind = RTE_SUBQUERY; +#if PG_VERSION_NUM >= PG_VERSION_16 + sourceRte->perminfoindex = 0; +#endif sourceRte->subquery = sourceResultsQuery; sourceRte->inh = false; } diff --git a/src/backend/distributed/planner/query_colocation_checker.c b/src/backend/distributed/planner/query_colocation_checker.c index f5701fdb1..77baab197 100644 --- a/src/backend/distributed/planner/query_colocation_checker.c +++ b/src/backend/distributed/planner/query_colocation_checker.c @@ -83,7 +83,16 @@ CreateColocatedJoinChecker(Query *subquery, PlannerRestrictionContext *restricti * functions (i.e., FilterPlannerRestrictionForQuery()) rely on queries * not relations. */ - anchorSubquery = WrapRteRelationIntoSubquery(anchorRangeTblEntry, NIL); +#if PG_VERSION_NUM >= PG_VERSION_16 + RTEPermissionInfo *perminfo = NULL; + if (anchorRangeTblEntry->perminfoindex) + { + perminfo = getRTEPermissionInfo(subquery->rteperminfos, anchorRangeTblEntry); + } + anchorSubquery = WrapRteRelationIntoSubquery(anchorRangeTblEntry, NIL, perminfo); +#else + anchorSubquery = WrapRteRelationIntoSubquery(anchorRangeTblEntry, NIL, NULL); +#endif } else if (anchorRangeTblEntry->rtekind == RTE_SUBQUERY) { @@ -266,7 +275,9 @@ SubqueryColocated(Query *subquery, ColocatedJoinChecker *checker) * designed for generating a stub query. */ Query * -WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, List *requiredAttributes) +WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, + List *requiredAttributes, + RTEPermissionInfo *perminfo) { Query *subquery = makeNode(Query); RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); @@ -277,6 +288,14 @@ WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, List *requiredAttributes RangeTblEntry *newRangeTableEntry = copyObject(rteRelation); subquery->rtable = list_make1(newRangeTableEntry); +#if PG_VERSION_NUM >= PG_VERSION_16 + if (perminfo) + { + newRangeTableEntry->perminfoindex = 1; + subquery->rteperminfos = list_make1(perminfo); + } +#endif + /* set the FROM expression to the subquery */ newRangeTableRef = makeNode(RangeTblRef); newRangeTableRef->rtindex = SINGLE_RTE_INDEX; diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index cbe6a3606..3bad73459 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -1414,6 +1414,12 @@ RelationInfoContainsOnlyRecurringTuples(PlannerInfo *plannerInfo, Relids relids) while ((relationId = bms_next_member(relids, relationId)) >= 0) { + /* outer join RTE check in PG16 */ + if (IsRelOptOuterJoin(plannerInfo, relationId)) + { + continue; + } + RangeTblEntry *rangeTableEntry = plannerInfo->simple_rte_array[relationId]; if (FindNodeMatchingCheckFunctionInRangeTableList(list_make1(rangeTableEntry), @@ -1915,6 +1921,9 @@ SubqueryPushdownMultiNodeTree(Query *originalQuery) pushedDownQuery->targetList = subqueryTargetEntryList; pushedDownQuery->jointree = copyObject(queryTree->jointree); pushedDownQuery->rtable = copyObject(queryTree->rtable); +#if PG_VERSION_NUM >= PG_VERSION_16 + pushedDownQuery->rteperminfos = copyObject(queryTree->rteperminfos); +#endif pushedDownQuery->setOperations = copyObject(queryTree->setOperations); pushedDownQuery->querySource = queryTree->querySource; pushedDownQuery->hasSubLinks = queryTree->hasSubLinks; diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index f582fd9df..c2426cf5f 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -80,6 +80,7 @@ #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" +#include "parser/parse_relation.h" #include "parser/parsetree.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -886,8 +887,19 @@ RecursivelyPlanDistributedJoinNode(Node *node, Query *query, List *requiredAttributes = RequiredAttrNumbersForRelation(distributedRte, restrictionContext); +#if PG_VERSION_NUM >= PG_VERSION_16 + RTEPermissionInfo *perminfo = NULL; + if (distributedRte->perminfoindex) + { + perminfo = getRTEPermissionInfo(query->rteperminfos, distributedRte); + } + ReplaceRTERelationWithRteSubquery(distributedRte, requiredAttributes, - recursivePlanningContext); + recursivePlanningContext, perminfo); +#else + ReplaceRTERelationWithRteSubquery(distributedRte, requiredAttributes, + recursivePlanningContext, NULL); +#endif } else if (distributedRte->rtekind == RTE_SUBQUERY) { @@ -1751,9 +1763,11 @@ NodeContainsSubqueryReferencingOuterQuery(Node *node) void ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, List *requiredAttrNumbers, - RecursivePlanningContext *context) + RecursivePlanningContext *context, + RTEPermissionInfo *perminfo) { - Query *subquery = WrapRteRelationIntoSubquery(rangeTableEntry, requiredAttrNumbers); + Query *subquery = WrapRteRelationIntoSubquery(rangeTableEntry, requiredAttrNumbers, + perminfo); List *outerQueryTargetList = CreateAllTargetListForRelation(rangeTableEntry->relid, requiredAttrNumbers); @@ -1778,6 +1792,9 @@ ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, /* replace the function with the constructed subquery */ rangeTableEntry->rtekind = RTE_SUBQUERY; +#if PG_VERSION_NUM >= PG_VERSION_16 + rangeTableEntry->perminfoindex = 0; +#endif rangeTableEntry->subquery = subquery; /* @@ -1850,6 +1867,15 @@ CreateOuterSubquery(RangeTblEntry *rangeTableEntry, List *outerSubqueryTargetLis innerSubqueryRTE->eref->colnames = innerSubqueryColNames; outerSubquery->rtable = list_make1(innerSubqueryRTE); +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* sanity check */ + Assert(innerSubqueryRTE->rtekind == RTE_SUBQUERY && + innerSubqueryRTE->perminfoindex == 0); + outerSubquery->rteperminfos = NIL; +#endif + + /* set the FROM expression to the subquery */ RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); newRangeTableRef->rtindex = 1; @@ -2022,6 +2048,15 @@ TransformFunctionRTE(RangeTblEntry *rangeTblEntry) /* set the FROM expression to the subquery */ subquery->rtable = list_make1(newRangeTableEntry); + +#if PG_VERSION_NUM >= PG_VERSION_16 + + /* sanity check */ + Assert(newRangeTableEntry->rtekind == RTE_FUNCTION && + newRangeTableEntry->perminfoindex == 0); + subquery->rteperminfos = NIL; +#endif + newRangeTableRef->rtindex = 1; subquery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); @@ -2392,6 +2427,9 @@ BuildReadIntermediateResultsQuery(List *targetEntryList, List *columnAliasList, Query *resultQuery = makeNode(Query); resultQuery->commandType = CMD_SELECT; resultQuery->rtable = list_make1(rangeTableEntry); +#if PG_VERSION_NUM >= PG_VERSION_16 + resultQuery->rteperminfos = NIL; +#endif resultQuery->jointree = joinTree; resultQuery->targetList = targetList; diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index af2443b19..368ba2026 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -171,8 +171,6 @@ static bool FindQueryContainingRTEIdentityInternal(Node *node, static int ParentCountPriorToAppendRel(List *appendRelList, AppendRelInfo *appendRelInfo); -static bool IsVarRelOptOuterJoin(PlannerInfo *root, Var *varToBeAdded); - /* * AllDistributionKeysInQueryAreEqual returns true if either @@ -1241,7 +1239,7 @@ AddToAttributeEquivalenceClass(AttributeEquivalenceClass *attributeEquivalenceCl } /* outer join checks in PG16 */ - if (IsVarRelOptOuterJoin(root, varToBeAdded)) + if (IsRelOptOuterJoin(root, varToBeAdded->varno)) { return; } @@ -1388,19 +1386,19 @@ GetTargetSubquery(PlannerInfo *root, RangeTblEntry *rangeTableEntry, Var *varToB /* - * IsVarRelOptOuterJoin returns true if the Var to be added - * is an outer join, false otherwise. + * IsRelOptOuterJoin returns true if the RelOpt referenced + * by varNo is an outer join, false otherwise. */ -static bool -IsVarRelOptOuterJoin(PlannerInfo *root, Var *varToBeAdded) +bool +IsRelOptOuterJoin(PlannerInfo *root, int varNo) { #if PG_VERSION_NUM >= PG_VERSION_16 - if (root->simple_rel_array_size <= varToBeAdded->varno) + if (root->simple_rel_array_size <= varNo) { return true; } - RelOptInfo *rel = root->simple_rel_array[varToBeAdded->varno]; + RelOptInfo *rel = root->simple_rel_array[varNo]; if (rel == NULL) { /* must be an outer join */ diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index a2b1811b8..e84aa282a 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -104,6 +104,7 @@ #include "replication/walsender.h" #include "storage/ipc.h" #include "optimizer/planner.h" +#include "optimizer/plancat.h" #include "optimizer/paths.h" #include "tcop/tcopprot.h" #include "utils/guc.h" @@ -452,6 +453,7 @@ _PG_init(void) /* register for planner hook */ set_rel_pathlist_hook = multi_relation_restriction_hook; + get_relation_info_hook = multi_get_relation_info_hook; set_join_pathlist_hook = multi_join_restriction_hook; ExecutorStart_hook = CitusExecutorStart; ExecutorRun_hook = CitusExecutorRun; diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.1-1.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.1-1.sql index 358fb638d..766e86a2e 100644 --- a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.1-1.sql +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.1-1.sql @@ -106,6 +106,15 @@ BEGIN -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we need to regenerate the partkeys because they will include varnullingrels as well. + UPDATE pg_catalog.pg_dist_partition + SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) + FROM public.pg_dist_partkeys_pre_16_upgrade + WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; + DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; + INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql index 358fb638d..766e86a2e 100644 --- a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql @@ -106,6 +106,15 @@ BEGIN -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we need to regenerate the partkeys because they will include varnullingrels as well. + UPDATE pg_catalog.pg_dist_partition + SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) + FROM public.pg_dist_partkeys_pre_16_upgrade + WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; + DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; + INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.1-1.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.1-1.sql index 099bad716..b4bc653f2 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.1-1.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.1-1.sql @@ -84,6 +84,13 @@ BEGIN objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we will need to regenerate the partkeys because they will include varnullingrels as well. + -- so we save the partkeys as column names here + CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS + SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name + FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; END; $cppu$; diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql index 099bad716..b4bc653f2 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql @@ -84,6 +84,13 @@ BEGIN objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; + + -- if we are upgrading from PG14/PG15 to PG16+, + -- we will need to regenerate the partkeys because they will include varnullingrels as well. + -- so we save the partkeys as column names here + CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS + SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name + FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; END; $cppu$; diff --git a/src/backend/distributed/utils/relation_utils.c b/src/backend/distributed/utils/relation_utils.c index ef02736ea..d39c1f071 100644 --- a/src/backend/distributed/utils/relation_utils.c +++ b/src/backend/distributed/utils/relation_utils.c @@ -45,9 +45,6 @@ RelationGetNamespaceName(Relation relation) * we are dealing with GetUserId(). * Currently the following entries are filled like this: * perminfo->checkAsUser = GetUserId(); - * perminfo->selectedCols = NULL; - * perminfo->insertedCols = NULL; - * perminfo->updatedCols = NULL; */ RTEPermissionInfo * GetFilledPermissionInfo(Oid relid, bool inh, AclMode requiredPerms) @@ -57,9 +54,6 @@ GetFilledPermissionInfo(Oid relid, bool inh, AclMode requiredPerms) perminfo->inh = inh; perminfo->requiredPerms = requiredPerms; perminfo->checkAsUser = GetUserId(); - perminfo->selectedCols = NULL; - perminfo->insertedCols = NULL; - perminfo->updatedCols = NULL; return perminfo; } diff --git a/src/include/distributed/coordinator_protocol.h b/src/include/distributed/coordinator_protocol.h index 7f90eadda..e2f1e9c52 100644 --- a/src/include/distributed/coordinator_protocol.h +++ b/src/include/distributed/coordinator_protocol.h @@ -156,7 +156,7 @@ struct TableDDLCommand /* * This union contains one (1) typed field for every implementation for - * TableDDLCommand. A union enforces no overloading of fields but instead requiers at + * TableDDLCommand. A union enforces no overloading of fields but instead requires at * most one of the fields to be used at any time. */ union diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index aac936a98..d46fbf2e6 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -234,6 +234,8 @@ extern List * TranslatedVarsForRteIdentity(int rteIdentity); extern struct DistributedPlan * GetDistributedPlan(CustomScan *node); extern void multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo, Index restrictionIndex, RangeTblEntry *rte); +extern void multi_get_relation_info_hook(PlannerInfo *root, Oid relationObjectId, bool + inhparent, RelOptInfo *rel); extern void multi_join_restriction_hook(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, diff --git a/src/include/distributed/query_colocation_checker.h b/src/include/distributed/query_colocation_checker.h index 969ecbcf9..562869a92 100644 --- a/src/include/distributed/query_colocation_checker.h +++ b/src/include/distributed/query_colocation_checker.h @@ -35,7 +35,8 @@ extern ColocatedJoinChecker CreateColocatedJoinChecker(Query *subquery, restrictionContext); extern bool SubqueryColocated(Query *subquery, ColocatedJoinChecker *context); extern Query * WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, - List *requiredAttributes); + List *requiredAttributes, + RTEPermissionInfo *perminfo); extern List * CreateAllTargetListForRelation(Oid relationId, List *requiredAttributes); #endif /* QUERY_COLOCATION_CHECKER_H */ diff --git a/src/include/distributed/recursive_planning.h b/src/include/distributed/recursive_planning.h index 8943443aa..a883047f6 100644 --- a/src/include/distributed/recursive_planning.h +++ b/src/include/distributed/recursive_planning.h @@ -42,7 +42,8 @@ extern bool GeneratingSubplans(void); extern bool ContainsLocalTableDistributedTableJoin(List *rangeTableList); extern void ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, List *requiredAttrNumbers, - RecursivePlanningContext *context); + RecursivePlanningContext *context, + RTEPermissionInfo *perminfo); extern bool IsRecursivelyPlannableRelation(RangeTblEntry *rangeTableEntry); extern bool IsRelationLocalTableOrMatView(Oid relationId); extern bool ContainsReferencesToOuterQuery(Query *query); diff --git a/src/include/distributed/relation_restriction_equivalence.h b/src/include/distributed/relation_restriction_equivalence.h index 42b2b801f..f3a7e2b94 100644 --- a/src/include/distributed/relation_restriction_equivalence.h +++ b/src/include/distributed/relation_restriction_equivalence.h @@ -20,6 +20,7 @@ extern bool AllDistributionKeysInQueryAreEqual(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); +extern bool IsRelOptOuterJoin(PlannerInfo *root, int varNo); extern bool SafeToPushdownUnionSubquery(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); extern bool ContainsUnionSubquery(Query *queryTree); diff --git a/src/include/pg_version_compat.h b/src/include/pg_version_compat.h index 3177aec28..2b0320003 100644 --- a/src/include/pg_version_compat.h +++ b/src/include/pg_version_compat.h @@ -27,8 +27,6 @@ #define float_abs(a) fabs(a) -#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackMaintainsTable - #define tuplesort_getdatum_compat(a, b, c, d, e, f) tuplesort_getdatum(a, b, c, d, e, f) static inline struct config_generic ** @@ -57,8 +55,6 @@ get_guc_variables_compat(int *gucCount) #include "utils/guc.h" #include "utils/guc_tables.h" -#define RANGE_VAR_TABLE_CALLBACK RangeVarCallbackOwnsTable - #define pg_clean_ascii_compat(a, b) pg_clean_ascii(a) #define RelationPhysicalIdentifier_compat(a) ((a)->rd_node) @@ -144,6 +140,13 @@ object_aclcheck(Oid classid, Oid objectid, Oid roleid, AclMode mode) typedef bool TU_UpdateIndexes; +/* + * we define RTEPermissionInfo for PG16 compatibility + * There are some functions that need to include RTEPermissionInfo in their signature + * for PG14/PG15 we pass a NULL argument in these functions + */ +typedef RangeTblEntry RTEPermissionInfo; + #endif #if PG_VERSION_NUM >= PG_VERSION_15 diff --git a/src/test/cdc/t/016_cdc_wal2json.pl b/src/test/cdc/t/016_cdc_wal2json.pl index ab384df64..10475ba85 100644 --- a/src/test/cdc/t/016_cdc_wal2json.pl +++ b/src/test/cdc/t/016_cdc_wal2json.pl @@ -9,6 +9,13 @@ use cdctestlib; use threads; +my $pg_major_version = int($ENV{'pg_major_version'}); +print("working with PG major version : $pg_major_version\n"); +if ($pg_major_version >= 16) { + plan skip_all => 'wal2json is not available for PG16 yet'; + exit 0; +} + # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;); my $result = 0; diff --git a/src/test/regress/Pipfile.lock b/src/test/regress/Pipfile.lock index ed604c2c0..e6717e5fc 100644 --- a/src/test/regress/Pipfile.lock +++ b/src/test/regress/Pipfile.lock @@ -21,7 +21,6 @@ "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9", "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214" ], - "markers": "python_version >= '3.6'", "version": "==3.4.1" }, "blinker": { @@ -119,11 +118,10 @@ }, "certifi": { "hashes": [ - "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", - "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" ], - "markers": "python_version >= '3.6'", - "version": "==2022.12.7" + "version": "==2023.7.22" }, "cffi": { "hashes": [ @@ -199,7 +197,6 @@ "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], - "markers": "python_version >= '3.6'", "version": "==8.0.4" }, "construct": { @@ -211,28 +208,32 @@ }, "cryptography": { "hashes": [ - "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440", - "sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288", - "sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b", - "sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958", - "sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b", - "sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d", - "sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a", - "sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404", - "sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b", - "sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e", - "sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2", - "sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c", - "sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b", - "sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9", - "sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b", - "sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636", - "sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99", - "sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e", - "sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9" + "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306", + "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84", + "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47", + "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d", + "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116", + "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207", + "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81", + "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087", + "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd", + "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507", + "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858", + "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae", + "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34", + "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906", + "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd", + "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922", + "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7", + "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4", + "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574", + "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1", + "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c", + "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e", + "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de" ], "index": "pypi", - "version": "==40.0.2" + "version": "==41.0.3" }, "docopt": { "hashes": [ @@ -243,34 +244,32 @@ }, "exceptiongroup": { "hashes": [ - "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e", - "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785" + "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9", + "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3" ], "markers": "python_version < '3.11'", - "version": "==1.1.1" + "version": "==1.1.3" }, "execnet": { "hashes": [ - "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5", - "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142" + "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41", + "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.9.0" + "version": "==2.0.2" }, "filelock": { "hashes": [ - "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9", - "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718" + "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", + "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec" ], "index": "pypi", - "version": "==3.12.0" + "version": "==3.12.2" }, "flask": { "hashes": [ "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f", "sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d" ], - "markers": "python_version >= '3.6'", "version": "==2.0.3" }, "h11": { @@ -278,7 +277,6 @@ "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6", "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042" ], - "markers": "python_version >= '3.6'", "version": "==0.12.0" }, "h2": { @@ -286,7 +284,6 @@ "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d", "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb" ], - "markers": "python_full_version >= '3.6.1'", "version": "==4.1.0" }, "hpack": { @@ -294,7 +291,6 @@ "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c", "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095" ], - "markers": "python_full_version >= '3.6.1'", "version": "==4.0.0" }, "hyperframe": { @@ -302,7 +298,6 @@ "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15", "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914" ], - "markers": "python_full_version >= '3.6.1'", "version": "==6.0.1" }, "iniconfig": { @@ -310,7 +305,6 @@ "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" ], - "markers": "python_version >= '3.7'", "version": "==2.0.0" }, "itsdangerous": { @@ -318,7 +312,6 @@ "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" ], - "markers": "python_version >= '3.7'", "version": "==2.1.2" }, "jinja2": { @@ -326,7 +319,6 @@ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], - "markers": "python_version >= '3.7'", "version": "==3.1.2" }, "kaitaistruct": { @@ -347,59 +339,58 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], - "markers": "python_version >= '3.7'", - "version": "==2.1.2" + "version": "==2.1.3" }, "mitmproxy": { "editable": true, @@ -479,7 +470,6 @@ "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], - "markers": "python_version >= '3.7'", "version": "==23.1" }, "passlib": { @@ -491,11 +481,10 @@ }, "pluggy": { "hashes": [ - "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", - "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849", + "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3" ], - "markers": "python_version >= '3.6'", - "version": "==1.0.0" + "version": "==1.2.0" }, "protobuf": { "hashes": [ @@ -521,16 +510,15 @@ "sha256:e68ad00695547d9397dd14abd3efba23cb31cef67228f4512d41396971889812", "sha256:e9bffd52d6ee039a1cafb72475b2900c6fd0f0dca667fb7a09af0a3e119e78cb" ], - "markers": "python_version >= '3.5'", "version": "==3.18.3" }, "psycopg": { "hashes": [ - "sha256:59b4a71536b146925513c0234dfd1dc42b81e65d56ce5335dff4813434dbc113", - "sha256:b1500c42063abaa01d30b056f0b300826b8dd8d586900586029a294ce74af327" + "sha256:15b25741494344c24066dc2479b0f383dd1b82fa5e75612fa4fa5bb30726e9b6", + "sha256:8bbeddae5075c7890b2fa3e3553440376d3c5e28418335dee3c3656b06fa2b52" ], "index": "pypi", - "version": "==3.1.8" + "version": "==3.1.10" }, "publicsuffix2": { "hashes": [ @@ -544,7 +532,6 @@ "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57", "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==0.5.0" }, "pycparser": { @@ -556,18 +543,16 @@ }, "pyopenssl": { "hashes": [ - "sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7", - "sha256:9e0c526404a210df9d2b18cd33364beadb0dc858a739b885677bc65e105d4a4c" + "sha256:24f0dc5227396b3e831f4c7f602b950a5e9833d292c8e4a2e06b709292806ae2", + "sha256:276f931f55a452e7dea69c7173e984eb2a4407ce413c918aa34b55f82f9b8bac" ], - "markers": "python_version >= '3.6'", - "version": "==23.1.1" + "version": "==23.2.0" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.4.7" }, "pyperclip": { @@ -578,19 +563,19 @@ }, "pytest": { "hashes": [ - "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362", - "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3" + "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", + "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a" ], "index": "pypi", - "version": "==7.3.1" + "version": "==7.4.0" }, "pytest-asyncio": { "hashes": [ - "sha256:2b38a496aef56f56b0e87557ec313e11e1ab9276fc3863f6a7be0f1d0e415e1b", - "sha256:f2b3366b7cd501a4056858bd39349d5af19742aed2d81660b7998b6341c7eb9c" + "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d", + "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b" ], "index": "pypi", - "version": "==0.21.0" + "version": "==0.21.1" }, "pytest-repeat": { "hashes": [ @@ -610,64 +595,63 @@ }, "pytest-xdist": { "hashes": [ - "sha256:1849bd98d8b242b948e472db7478e090bf3361912a8fed87992ed94085f54727", - "sha256:37290d161638a20b672401deef1cba812d110ac27e35d213f091d15b8beb40c9" + "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93", + "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2" ], "index": "pypi", - "version": "==3.2.1" + "version": "==3.3.1" }, "pyyaml": { "hashes": [ - "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", - "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", - "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", - "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", - "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", - "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", - "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", - "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", - "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", - "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", - "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", - "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", - "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", - "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", - "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", - "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", - "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", - "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", - "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", - "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", - "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", - "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", - "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", - "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", - "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", - "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", - "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", - "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", - "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", - "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", - "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", - "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", - "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", - "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", - "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", - "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", - "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", - "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", - "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", - "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" ], "index": "pypi", - "version": "==6.0" + "version": "==6.0.1" }, "ruamel.yaml": { "hashes": [ "sha256:1a771fc92d3823682b7f0893ad56cb5a5c87c48e62b5399d6f42c8759a583b33", "sha256:ea21da1198c4b41b8e7a259301cc9710d3b972bf8ba52f06218478e6802dd1f1" ], - "markers": "python_version >= '3'", "version": "==0.17.16" }, "ruamel.yaml.clib": { @@ -675,6 +659,7 @@ "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e", "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3", "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5", + "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81", "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497", "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f", "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac", @@ -685,13 +670,13 @@ "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1", "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072", "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9", - "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5", "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231", "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93", "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b", "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb", "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f", "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307", + "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf", "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8", "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b", "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b", @@ -729,28 +714,26 @@ }, "tornado": { "hashes": [ - "sha256:4546003dc8b5733489139d3bff5fa6a0211be505faf819bd9970e7c2b32e8122", - "sha256:4d349846931557b7ec92f224b5d598b160e2ba26ae1812480b42e9622c884bf7", - "sha256:6164571f5b9f73143d1334df4584cb9ac86d20c461e17b6c189a19ead8bb93c1", - "sha256:6cfff1e9c15c79e106b8352269d201f8fc0815914a6260f3893ca18b724ea94b", - "sha256:720f53e6367b38190ae7fa398c25c086c69d88b3c6535bd6021a126b727fb5cd", - "sha256:912df5712024564e362ecce43c8d5862e14c78c8dd3846c9d889d44fbd7f4951", - "sha256:c37b6a384d54ce6a31168d40ab21ad2591ddaf34973075cc0cad154402ecd9e8", - "sha256:c659ab04d5aa477dbe44152c67d93f3ad3243b992d94f795ca1d5c73c37337ce", - "sha256:c9114a61a4588c09065b9996ae05462350d17160b92b9bf9a1e93689cc0424dc", - "sha256:d68f3192936ff2c4add04dc21a436a43b4408d466746b78bb2b9d0a53a18683f", - "sha256:d7b737e18f701de3e4a3b0824260b4d740e4d60607b8089bb80e80ffd464780e" + "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f", + "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5", + "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d", + "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3", + "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2", + "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a", + "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16", + "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a", + "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17", + "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0", + "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe" ], - "markers": "python_version >= '3.8'", - "version": "==6.3" + "version": "==6.3.3" }, "typing-extensions": { "hashes": [ - "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", - "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], - "markers": "python_version >= '3.7'", - "version": "==4.5.0" + "version": "==4.7.1" }, "urwid": { "hashes": [ @@ -760,18 +743,16 @@ }, "werkzeug": { "hashes": [ - "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", - "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612" + "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8", + "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528" ], - "markers": "python_version >= '3.7'", - "version": "==2.2.3" + "version": "==2.3.7" }, "wsproto": { "hashes": [ "sha256:868776f8456997ad0d9720f7322b746bbe9193751b5b290b7f924659377c8c38", "sha256:d8345d1808dd599b5ffb352c25a367adb6157e664e140dbecba3f9bc007edb9f" ], - "markers": "python_full_version >= '3.6.1'", "version": "==1.0.0" }, "zstandard": { @@ -825,7 +806,6 @@ "sha256:f98fc5750aac2d63d482909184aac72a979bfd123b112ec53fd365104ea15b1c", "sha256:ff5b75f94101beaa373f1511319580a010f6e03458ee51b1a386d7de5331440a" ], - "markers": "python_version >= '3.5'", "version": "==0.15.2" } }, @@ -835,63 +815,58 @@ "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" ], - "markers": "python_version >= '3.7'", "version": "==23.1.0" }, "black": { "hashes": [ - "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5", - "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915", - "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326", - "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940", - "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b", - "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30", - "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c", - "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c", - "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab", - "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27", - "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2", - "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961", - "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9", - "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb", - "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70", - "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331", - "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2", - "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266", - "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d", - "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6", - "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b", - "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925", - "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8", - "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4", - "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3" + "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3", + "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb", + "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087", + "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320", + "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6", + "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3", + "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc", + "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f", + "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587", + "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91", + "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a", + "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad", + "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926", + "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9", + "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be", + "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd", + "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96", + "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491", + "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2", + "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a", + "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f", + "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995" ], "index": "pypi", - "version": "==23.3.0" + "version": "==23.7.0" }, "click": { "hashes": [ "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], - "markers": "python_version >= '3.6'", "version": "==8.0.4" }, "flake8": { "hashes": [ - "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", - "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23", + "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5" ], "index": "pypi", - "version": "==6.0.0" + "version": "==6.1.0" }, "flake8-bugbear": { "hashes": [ - "sha256:8a218d13abd6904800970381057ce8e72cde8eea743240c4ef3ede4dd0bc9cfb", - "sha256:ea565bdb87b96b56dc499edd6cc3ba7f695373d902a5f56c989b74fad7c7719d" + "sha256:0ebdc7d8ec1ca8bd49347694562381f099f4de2f8ec6bda7a7dca65555d9e0d4", + "sha256:d99d005114020fbef47ed5e4aebafd22f167f9a0fbd0d8bf3c9e90612cb25c34" ], "index": "pypi", - "version": "==23.3.23" + "version": "==23.7.10" }, "isort": { "hashes": [ @@ -906,7 +881,6 @@ "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "markers": "python_version >= '3.6'", "version": "==0.7.0" }, "mypy-extensions": { @@ -914,7 +888,6 @@ "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782" ], - "markers": "python_version >= '3.5'", "version": "==1.0.0" }, "packaging": { @@ -922,40 +895,35 @@ "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], - "markers": "python_version >= '3.7'", "version": "==23.1" }, "pathspec": { "hashes": [ - "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687", - "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293" + "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20", + "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3" ], - "markers": "python_version >= '3.7'", - "version": "==0.11.1" + "version": "==0.11.2" }, "platformdirs": { "hashes": [ - "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08", - "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e" + "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d", + "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d" ], - "markers": "python_version >= '3.7'", - "version": "==3.2.0" + "version": "==3.10.0" }, "pycodestyle": { "hashes": [ - "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", - "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0", + "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8" ], - "markers": "python_version >= '3.6'", - "version": "==2.10.0" + "version": "==2.11.0" }, "pyflakes": { "hashes": [ - "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", - "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774", + "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.1" + "version": "==3.1.0" }, "tomli": { "hashes": [ @@ -967,11 +935,10 @@ }, "typing-extensions": { "hashes": [ - "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", - "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], - "markers": "python_version >= '3.7'", - "version": "==4.5.0" + "version": "==4.7.1" } } } diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 7bed04edf..efa9e310f 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -301,5 +301,10 @@ s/(NOTICE: issuing CREATE EXTENSION IF NOT EXISTS citus_columnar WITH SCHEMA p # (This is not preprocessor directive, but a reminder for the developer that will drop PG14&15 support ) s/, password_required=false//g +s/provide the file or change sslmode/provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode/g +s/(:varcollid [0-9]+) :varlevelsup 0/\1 :varnullingrels (b) :varlevelsup 0/g +s/table_name_for_view\.([_a-z0-9]+)(,| |$)/\1\2/g +s/permission denied to terminate process/must be a superuser to terminate superuser process/g +s/permission denied to cancel query/must be a superuser to cancel superuser query/g #endif /* PG_VERSION_NUM < PG_VERSION_16 */ diff --git a/src/test/regress/citus_tests/common.py b/src/test/regress/citus_tests/common.py index 9ff30ebf9..66ff044d2 100644 --- a/src/test/regress/citus_tests/common.py +++ b/src/test/regress/citus_tests/common.py @@ -92,6 +92,7 @@ PG_MAJOR_VERSION = get_pg_major_version() OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 14: "10.2.0", 15: "11.1.5", + 16: "12.1devel", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION] diff --git a/src/test/regress/expected/citus_local_tables.out b/src/test/regress/expected/citus_local_tables.out index aca2548a5..4f3053094 100644 --- a/src/test/regress/expected/citus_local_tables.out +++ b/src/test/regress/expected/citus_local_tables.out @@ -635,7 +635,7 @@ FROM pg_dist_partition WHERE logicalrelid = 'citus_local_table_4'::regclass; SELECT column_name_to_column('citus_local_table_4', 'a'); column_name_to_column --------------------------------------------------------------------- - {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} + {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} (1 row) SELECT master_update_shard_statistics(shardid) diff --git a/src/test/regress/expected/citus_local_tables_mx.out b/src/test/regress/expected/citus_local_tables_mx.out index 363e87e58..8b3cb953f 100644 --- a/src/test/regress/expected/citus_local_tables_mx.out +++ b/src/test/regress/expected/citus_local_tables_mx.out @@ -769,8 +769,8 @@ SELECT logicalrelid, partmethod, partkey FROM pg_dist_partition ORDER BY logicalrelid; logicalrelid | partmethod | partkey --------------------------------------------------------------------- - parent_dropped_col | h | {VAR :varno 1 :varattno 1 :vartype 1082 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} - parent_dropped_col_2 | h | {VAR :varno 1 :varattno 5 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 5 :location -1} + parent_dropped_col | h | {VAR :varno 1 :varattno 1 :vartype 1082 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} + parent_dropped_col_2 | h | {VAR :varno 1 :varattno 5 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 5 :location -1} (2 rows) -- some tests for view propagation on citus local tables @@ -912,12 +912,12 @@ select run_command_on_workers($$SELECT count(*)=0 from citus_local_tables_mx.v10 (2 rows) CREATE TABLE loc_tb_2 (a int); -CREATE VIEW v104 AS SELECT * from loc_tb_2; +CREATE VIEW v104 AS SELECT * from loc_tb_2 table_name_for_view; SET client_min_messages TO DEBUG1; -- verify the CREATE command for the view is generated correctly ALTER TABLE loc_tb_2 ADD CONSTRAINT fkey_2 FOREIGN KEY (a) references ref_tb(a); -DEBUG: executing "CREATE OR REPLACE VIEW citus_local_tables_mx.v104 (a) AS SELECT loc_tb_2.a - FROM citus_local_tables_mx.loc_tb_2; ALTER VIEW citus_local_tables_mx.v104 OWNER TO postgres" +DEBUG: executing "CREATE OR REPLACE VIEW citus_local_tables_mx.v104 (a) AS SELECT a + FROM citus_local_tables_mx.loc_tb_2 table_name_for_view; ALTER VIEW citus_local_tables_mx.v104 OWNER TO postgres" DEBUG: "view v104" has dependency to "table loc_tb_2" that is not in Citus' metadata DEBUG: validating foreign key constraint "fkey_2_1330083" SET client_min_messages TO WARNING; diff --git a/src/test/regress/expected/columnar_chunk_filtering.out b/src/test/regress/expected/columnar_chunk_filtering.out index 0d0534ccc..3acdd957d 100644 --- a/src/test/regress/expected/columnar_chunk_filtering.out +++ b/src/test/regress/expected/columnar_chunk_filtering.out @@ -1,6 +1,10 @@ -- -- Test chunk filtering in columnar using min/max values in stripe skip lists. -- +-- It has an alternative test output file +-- because PG16 changed the order of some Filters in EXPLAIN +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/2489d76c4906f4461a364ca8ad7e0751ead8aa0d -- -- filtered_row_count returns number of rows filtered by the WHERE clause. -- If chunks get filtered by columnar, less rows are passed to WHERE @@ -370,10 +374,10 @@ SELECT * FROM r1, coltest WHERE Filter: ((n1 % 10) = 0) Rows Removed by Filter: 1 -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=4) - Filter: ((x1 > 15000) AND (r1.id1 = id) AND ((x1)::text > '000000'::text)) + Filter: ((x1 > 15000) AND (id = r1.id1) AND ((x1)::text > '000000'::text)) Rows Removed by Filter: 999 Columnar Projected Columns: id, x1, x2, x3 - Columnar Chunk Group Filters: ((x1 > 15000) AND (r1.id1 = id)) + Columnar Chunk Group Filters: ((x1 > 15000) AND (id = r1.id1)) Columnar Chunk Groups Removed by Filter: 19 (10 rows) @@ -413,10 +417,10 @@ SELECT * FROM r1, r2, r3, r4, r5, r6, r7, coltest WHERE -> Seq Scan on r2 (actual rows=5 loops=5) -> Seq Scan on r3 (actual rows=5 loops=5) -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=5) - Filter: (r1.id1 = id) + Filter: (id = r1.id1) Rows Removed by Filter: 999 Columnar Projected Columns: id, x1, x2, x3 - Columnar Chunk Group Filters: (r1.id1 = id) + Columnar Chunk Group Filters: (id = r1.id1) Columnar Chunk Groups Removed by Filter: 19 -> Seq Scan on r4 (actual rows=1 loops=5) -> Seq Scan on r5 (actual rows=1 loops=1) @@ -588,10 +592,10 @@ DETAIL: parameterized by rels {r3}; 2 clauses pushed down -> Nested Loop (actual rows=3 loops=1) -> Seq Scan on r1 (actual rows=5 loops=1) -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=5) - Filter: ((r1.n1 > x1) AND (r1.id1 = id)) + Filter: ((r1.n1 > x1) AND (id = r1.id1)) Rows Removed by Filter: 799 Columnar Projected Columns: id, x1, x2, x3 - Columnar Chunk Group Filters: ((r1.n1 > x1) AND (r1.id1 = id)) + Columnar Chunk Group Filters: ((r1.n1 > x1) AND (id = r1.id1)) Columnar Chunk Groups Removed by Filter: 19 -> Seq Scan on r2 (actual rows=5 loops=3) -> Seq Scan on r3 (actual rows=5 loops=3) @@ -618,10 +622,10 @@ SELECT * FROM r1, coltest_part WHERE -> Seq Scan on r1 (actual rows=5 loops=1) -> Append (actual rows=1 loops=5) -> Custom Scan (ColumnarScan) on coltest_part0 coltest_part_1 (actual rows=1 loops=3) - Filter: ((r1.n1 > x1) AND (r1.id1 = id)) + Filter: ((r1.n1 > x1) AND (id = r1.id1)) Rows Removed by Filter: 999 Columnar Projected Columns: id, x1, x2, x3 - Columnar Chunk Group Filters: ((r1.n1 > x1) AND (r1.id1 = id)) + Columnar Chunk Group Filters: ((r1.n1 > x1) AND (id = r1.id1)) Columnar Chunk Groups Removed by Filter: 9 -> Seq Scan on coltest_part1 coltest_part_2 (actual rows=0 loops=2) Filter: ((r1.n1 > x1) AND (r1.id1 = id)) diff --git a/src/test/regress/expected/columnar_chunk_filtering_0.out b/src/test/regress/expected/columnar_chunk_filtering_0.out new file mode 100644 index 000000000..746f3406f --- /dev/null +++ b/src/test/regress/expected/columnar_chunk_filtering_0.out @@ -0,0 +1,1138 @@ +-- +-- Test chunk filtering in columnar using min/max values in stripe skip lists. +-- +-- It has an alternative test output file +-- because PG16 changed the order of some Filters in EXPLAIN +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/2489d76c4906f4461a364ca8ad7e0751ead8aa0d +-- +-- filtered_row_count returns number of rows filtered by the WHERE clause. +-- If chunks get filtered by columnar, less rows are passed to WHERE +-- clause, so this function should return a lower number. +-- +CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS +$$ + DECLARE + result bigint; + rec text; + BEGIN + result := 0; + + FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP + IF rec ~ '^\s+Rows Removed by Filter' then + result := regexp_replace(rec, '[^0-9]*', '', 'g'); + END IF; + END LOOP; + + RETURN result; + END; +$$ LANGUAGE PLPGSQL; +set columnar.qual_pushdown_correlation = 0.0; +-- Create and load data +-- chunk_group_row_limit '1000', stripe_row_limit '2000' +set columnar.stripe_row_limit = 2000; +set columnar.chunk_group_row_limit = 1000; +CREATE TABLE test_chunk_filtering (a int) + USING columnar; +INSERT INTO test_chunk_filtering SELECT generate_series(1,10000); +-- Verify that filtered_row_count is less than 1000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering'); + filtered_row_count +--------------------------------------------------------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200'); + filtered_row_count +--------------------------------------------------------------------- + 801 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 200'); + filtered_row_count +--------------------------------------------------------------------- + 200 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 9900'); + filtered_row_count +--------------------------------------------------------------------- + 101 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 9900'); + filtered_row_count +--------------------------------------------------------------------- + 900 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0'); + filtered_row_count +--------------------------------------------------------------------- + 0 +(1 row) + +-- Verify that filtered_row_count is less than 2000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 1 AND 10'); + filtered_row_count +--------------------------------------------------------------------- + 990 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +--------------------------------------------------------------------- + 1979 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN -10 AND 0'); + filtered_row_count +--------------------------------------------------------------------- + 0 +(1 row) + +-- Load data for second time and verify that filtered_row_count is exactly twice as before +INSERT INTO test_chunk_filtering SELECT generate_series(1,10000); +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200'); + filtered_row_count +--------------------------------------------------------------------- + 1602 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0'); + filtered_row_count +--------------------------------------------------------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +--------------------------------------------------------------------- + 3958 +(1 row) + +set columnar.stripe_row_limit to default; +set columnar.chunk_group_row_limit to default; +-- Verify that we are fine with collations which use a different alphabet order +CREATE TABLE collation_chunk_filtering_test(A text collate "da_DK") + USING columnar; +COPY collation_chunk_filtering_test FROM STDIN; +SELECT * FROM collation_chunk_filtering_test WHERE A > 'B'; + a +--------------------------------------------------------------------- + Å +(1 row) + +CREATE TABLE simple_chunk_filtering(i int) USING COLUMNAR; +INSERT INTO simple_chunk_filtering SELECT generate_series(0,234567); +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT * FROM simple_chunk_filtering WHERE i > 123456; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on simple_chunk_filtering (actual rows=111111 loops=1) + Filter: (i > 123456) + Rows Removed by Filter: 3457 + Columnar Projected Columns: i + Columnar Chunk Group Filters: (i > 123456) + Columnar Chunk Groups Removed by Filter: 12 +(6 rows) + +SET columnar.enable_qual_pushdown = false; +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT * FROM simple_chunk_filtering WHERE i > 123456; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on simple_chunk_filtering (actual rows=111111 loops=1) + Filter: (i > 123456) + Rows Removed by Filter: 123457 + Columnar Projected Columns: i +(4 rows) + +SET columnar.enable_qual_pushdown TO DEFAULT; +-- https://github.com/citusdata/citus/issues/4555 +TRUNCATE simple_chunk_filtering; +INSERT INTO simple_chunk_filtering SELECT generate_series(0,200000); +COPY (SELECT * FROM simple_chunk_filtering WHERE i > 180000) TO '/dev/null'; +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT * FROM simple_chunk_filtering WHERE i > 180000; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on simple_chunk_filtering (actual rows=20000 loops=1) + Filter: (i > 180000) + Rows Removed by Filter: 1 + Columnar Projected Columns: i + Columnar Chunk Group Filters: (i > 180000) + Columnar Chunk Groups Removed by Filter: 18 +(6 rows) + +DROP TABLE simple_chunk_filtering; +CREATE TABLE multi_column_chunk_filtering(a int, b int) USING columnar; +INSERT INTO multi_column_chunk_filtering SELECT i,i+1 FROM generate_series(0,234567) i; +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT count(*) FROM multi_column_chunk_filtering WHERE a > 50000; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=184567 loops=1) + Filter: (a > 50000) + Rows Removed by Filter: 1 + Columnar Projected Columns: a + Columnar Chunk Group Filters: (a > 50000) + Columnar Chunk Groups Removed by Filter: 5 +(7 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT count(*) FROM multi_column_chunk_filtering WHERE a > 50000 AND b > 50000; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=184567 loops=1) + Filter: ((a > 50000) AND (b > 50000)) + Rows Removed by Filter: 1 + Columnar Projected Columns: a, b + Columnar Chunk Group Filters: ((a > 50000) AND (b > 50000)) + Columnar Chunk Groups Removed by Filter: 5 +(7 rows) + +-- make next tests faster +TRUNCATE multi_column_chunk_filtering; +INSERT INTO multi_column_chunk_filtering SELECT generate_series(0,5); +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT b FROM multi_column_chunk_filtering WHERE a > 50000 AND b > 50000; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=0 loops=1) + Filter: ((a > 50000) AND (b > 50000)) + Columnar Projected Columns: a, b + Columnar Chunk Group Filters: ((a > 50000) AND (b > 50000)) + Columnar Chunk Groups Removed by Filter: 1 +(5 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT b, a FROM multi_column_chunk_filtering WHERE b > 50000; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=0 loops=1) + Filter: (b > 50000) + Rows Removed by Filter: 6 + Columnar Projected Columns: a, b + Columnar Chunk Group Filters: (b > 50000) + Columnar Chunk Groups Removed by Filter: 0 +(6 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT FROM multi_column_chunk_filtering WHERE a > 50000; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=0 loops=1) + Filter: (a > 50000) + Columnar Projected Columns: a + Columnar Chunk Group Filters: (a > 50000) + Columnar Chunk Groups Removed by Filter: 1 +(5 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT FROM multi_column_chunk_filtering; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=6 loops=1) + Columnar Projected Columns: +(2 rows) + +BEGIN; + ALTER TABLE multi_column_chunk_filtering DROP COLUMN a; + ALTER TABLE multi_column_chunk_filtering DROP COLUMN b; + EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT * FROM multi_column_chunk_filtering; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=6 loops=1) + Columnar Projected Columns: +(2 rows) + +ROLLBACK; +CREATE TABLE another_columnar_table(x int, y int) USING columnar; +INSERT INTO another_columnar_table SELECT generate_series(0,5); +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT a, y FROM multi_column_chunk_filtering, another_columnar_table WHERE x > 1; + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop (actual rows=24 loops=1) + -> Custom Scan (ColumnarScan) on another_columnar_table (actual rows=4 loops=1) + Filter: (x > 1) + Rows Removed by Filter: 2 + Columnar Projected Columns: x, y + Columnar Chunk Group Filters: (x > 1) + Columnar Chunk Groups Removed by Filter: 0 + -> Custom Scan (ColumnarScan) on multi_column_chunk_filtering (actual rows=6 loops=4) + Columnar Projected Columns: a +(9 rows) + +EXPLAIN (costs off, timing off, summary off) + SELECT y, * FROM another_columnar_table; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on another_columnar_table + Columnar Projected Columns: x, y +(2 rows) + +EXPLAIN (costs off, timing off, summary off) + SELECT *, x FROM another_columnar_table; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on another_columnar_table + Columnar Projected Columns: x, y +(2 rows) + +EXPLAIN (costs off, timing off, summary off) + SELECT y, another_columnar_table FROM another_columnar_table; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on another_columnar_table + Columnar Projected Columns: x, y +(2 rows) + +EXPLAIN (costs off, timing off, summary off) + SELECT another_columnar_table, x FROM another_columnar_table; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on another_columnar_table + Columnar Projected Columns: x, y +(2 rows) + +DROP TABLE multi_column_chunk_filtering, another_columnar_table; +-- +-- https://github.com/citusdata/citus/issues/4780 +-- +create table part_table (id int) partition by range (id); +create table part_1_row partition of part_table for values from (150000) to (160000); +create table part_2_columnar partition of part_table for values from (0) to (150000) using columnar; +insert into part_table select generate_series(1,159999); +select filtered_row_count('select count(*) from part_table where id > 75000'); + filtered_row_count +--------------------------------------------------------------------- + 5000 +(1 row) + +drop table part_table; +-- test join parameterization +set columnar.stripe_row_limit = 2000; +set columnar.chunk_group_row_limit = 1000; +create table r1(id1 int, n1 int); -- row +create table r2(id2 int, n2 int); -- row +create table r3(id3 int, n3 int); -- row +create table r4(id4 int, n4 int); -- row +create table r5(id5 int, n5 int); -- row +create table r6(id6 int, n6 int); -- row +create table r7(id7 int, n7 int); -- row +create table coltest(id int, x1 int, x2 int, x3 int) using columnar; +create table coltest_part(id int, x1 int, x2 int, x3 int) + partition by range (id); +create table coltest_part0 + partition of coltest_part for values from (0) to (10000) + using columnar; +create table coltest_part1 + partition of coltest_part for values from (10000) to (20000); -- row +set columnar.stripe_row_limit to default; +set columnar.chunk_group_row_limit to default; +insert into r1 values(1234, 12350); +insert into r1 values(4567, 45000); +insert into r1 values(9101, 176000); +insert into r1 values(14202, 7); +insert into r1 values(18942, 189430); +insert into r2 values(1234, 123502); +insert into r2 values(4567, 450002); +insert into r2 values(9101, 1760002); +insert into r2 values(14202, 72); +insert into r2 values(18942, 1894302); +insert into r3 values(1234, 1235075); +insert into r3 values(4567, 4500075); +insert into r3 values(9101, 17600075); +insert into r3 values(14202, 775); +insert into r3 values(18942, 18943075); +insert into r4 values(1234, -1); +insert into r5 values(1234, -1); +insert into r6 values(1234, -1); +insert into r7 values(1234, -1); +insert into coltest + select g, g*10, g*100, g*1000 from generate_series(0, 19999) g; +insert into coltest_part + select g, g*10, g*100, g*1000 from generate_series(0, 19999) g; +ANALYZE r1, r2, r3, coltest, coltest_part; +-- force nested loop +set enable_mergejoin=false; +set enable_hashjoin=false; +set enable_material=false; +-- test different kinds of expressions +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM r1, coltest WHERE + id1 = id AND x1 > 15000 AND x1::text > '000000' AND n1 % 10 = 0; + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop (actual rows=3 loops=1) + -> Seq Scan on r1 (actual rows=4 loops=1) + Filter: ((n1 % 10) = 0) + Rows Removed by Filter: 1 + -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=4) + Filter: ((x1 > 15000) AND (r1.id1 = id) AND ((x1)::text > '000000'::text)) + Rows Removed by Filter: 999 + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: ((x1 > 15000) AND (r1.id1 = id)) + Columnar Chunk Groups Removed by Filter: 19 +(10 rows) + +SELECT * FROM r1, coltest WHERE + id1 = id AND x1 > 15000 AND x1::text > '000000' AND n1 % 10 = 0; + id1 | n1 | id | x1 | x2 | x3 +--------------------------------------------------------------------- + 4567 | 45000 | 4567 | 45670 | 456700 | 4567000 + 9101 | 176000 | 9101 | 91010 | 910100 | 9101000 + 18942 | 189430 | 18942 | 189420 | 1894200 | 18942000 +(3 rows) + +-- test equivalence classes +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM r1, r2, r3, r4, r5, r6, r7, coltest WHERE + id = id1 AND id1 = id2 AND id2 = id3 AND id3 = id4 AND + id4 = id5 AND id5 = id6 AND id6 = id7; + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop (actual rows=1 loops=1) + Join Filter: (coltest.id = r7.id7) + -> Nested Loop (actual rows=1 loops=1) + Join Filter: (coltest.id = r6.id6) + -> Nested Loop (actual rows=1 loops=1) + Join Filter: (coltest.id = r5.id5) + -> Nested Loop (actual rows=1 loops=1) + Join Filter: (coltest.id = r4.id4) + Rows Removed by Join Filter: 4 + -> Nested Loop (actual rows=5 loops=1) + -> Nested Loop (actual rows=5 loops=1) + Join Filter: (r1.id1 = r3.id3) + Rows Removed by Join Filter: 20 + -> Nested Loop (actual rows=5 loops=1) + Join Filter: (r1.id1 = r2.id2) + Rows Removed by Join Filter: 20 + -> Seq Scan on r1 (actual rows=5 loops=1) + -> Seq Scan on r2 (actual rows=5 loops=5) + -> Seq Scan on r3 (actual rows=5 loops=5) + -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=5) + Filter: (r1.id1 = id) + Rows Removed by Filter: 999 + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: (r1.id1 = id) + Columnar Chunk Groups Removed by Filter: 19 + -> Seq Scan on r4 (actual rows=1 loops=5) + -> Seq Scan on r5 (actual rows=1 loops=1) + -> Seq Scan on r6 (actual rows=1 loops=1) + -> Seq Scan on r7 (actual rows=1 loops=1) +(29 rows) + +SELECT * FROM r1, r2, r3, r4, r5, r6, r7, coltest WHERE + id = id1 AND id1 = id2 AND id2 = id3 AND id3 = id4 AND + id4 = id5 AND id5 = id6 AND id6 = id7; + id1 | n1 | id2 | n2 | id3 | n3 | id4 | n4 | id5 | n5 | id6 | n6 | id7 | n7 | id | x1 | x2 | x3 +--------------------------------------------------------------------- + 1234 | 12350 | 1234 | 123502 | 1234 | 1235075 | 1234 | -1 | 1234 | -1 | 1234 | -1 | 1234 | -1 | 1234 | 12340 | 123400 | 1234000 +(1 row) + +-- test path generation with different thresholds +set columnar.planner_debug_level = 'notice'; +set columnar.max_custom_scan_paths to 10; +EXPLAIN (costs off, timing off, summary off) + SELECT * FROM coltest c1, coltest c2, coltest c3, coltest c4 WHERE + c1.id = c2.id and c1.id = c3.id and c1.id = c4.id; +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c2}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c2, c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c2, c3, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c2, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c3, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: parameterized by rels {c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c1}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c1, c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c1, c3, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c1, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c3, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: parameterized by rels {c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c1}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c1, c2}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c1, c2, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c1, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c2}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c2, c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: parameterized by rels {c4}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c1}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c1, c2}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c1, c2, c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c1, c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c2}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c2, c3}; 1 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: parameterized by rels {c3}; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop + -> Nested Loop + -> Nested Loop + -> Custom Scan (ColumnarScan) on coltest c1 + Columnar Projected Columns: id, x1, x2, x3 + -> Custom Scan (ColumnarScan) on coltest c2 + Filter: (c1.id = id) + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: (c1.id = id) + -> Custom Scan (ColumnarScan) on coltest c3 + Filter: (c1.id = id) + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: (c1.id = id) + -> Custom Scan (ColumnarScan) on coltest c4 + Filter: (c1.id = id) + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: (c1.id = id) +(17 rows) + +set columnar.max_custom_scan_paths to 2; +EXPLAIN (costs off, timing off, summary off) + SELECT * FROM coltest c1, coltest c2, coltest c3, coltest c4 WHERE + c1.id = c2.id and c1.id = c3.id and c1.id = c4.id; +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c2 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c3 +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for c4 +DETAIL: unparameterized; 0 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop + Join Filter: (c1.id = c4.id) + -> Nested Loop + Join Filter: (c1.id = c3.id) + -> Nested Loop + Join Filter: (c1.id = c2.id) + -> Custom Scan (ColumnarScan) on coltest c1 + Columnar Projected Columns: id, x1, x2, x3 + -> Custom Scan (ColumnarScan) on coltest c2 + Columnar Projected Columns: id, x1, x2, x3 + -> Custom Scan (ColumnarScan) on coltest c3 + Columnar Projected Columns: id, x1, x2, x3 + -> Custom Scan (ColumnarScan) on coltest c4 + Columnar Projected Columns: id, x1, x2, x3 +(14 rows) + +set columnar.max_custom_scan_paths to default; +set columnar.planner_debug_level to default; +-- test more complex parameterization +set columnar.planner_debug_level = 'notice'; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM r1, r2, r3, coltest WHERE + id1 = id2 AND id2 = id3 AND id3 = id AND + n1 > x1 AND n2 > x2 AND n3 > x3; +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r1}; 2 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r1, r2}; 3 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r1, r2, r3}; 4 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r1, r3}; 3 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r2}; 2 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r2, r3}; 3 clauses pushed down +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: parameterized by rels {r3}; 2 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop (actual rows=3 loops=1) + Join Filter: ((r3.n3 > coltest.x3) AND (r1.id1 = r3.id3)) + Rows Removed by Join Filter: 12 + -> Nested Loop (actual rows=3 loops=1) + Join Filter: ((r2.n2 > coltest.x2) AND (r1.id1 = r2.id2)) + Rows Removed by Join Filter: 12 + -> Nested Loop (actual rows=3 loops=1) + -> Seq Scan on r1 (actual rows=5 loops=1) + -> Custom Scan (ColumnarScan) on coltest (actual rows=1 loops=5) + Filter: ((r1.n1 > x1) AND (r1.id1 = id)) + Rows Removed by Filter: 799 + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: ((r1.n1 > x1) AND (r1.id1 = id)) + Columnar Chunk Groups Removed by Filter: 19 + -> Seq Scan on r2 (actual rows=5 loops=3) + -> Seq Scan on r3 (actual rows=5 loops=3) +(16 rows) + +set columnar.planner_debug_level to default; +SELECT * FROM r1, r2, r3, coltest WHERE + id1 = id2 AND id2 = id3 AND id3 = id AND + n1 > x1 AND n2 > x2 AND n3 > x3; + id1 | n1 | id2 | n2 | id3 | n3 | id | x1 | x2 | x3 +--------------------------------------------------------------------- + 1234 | 12350 | 1234 | 123502 | 1234 | 1235075 | 1234 | 12340 | 123400 | 1234000 + 9101 | 176000 | 9101 | 1760002 | 9101 | 17600075 | 9101 | 91010 | 910100 | 9101000 + 18942 | 189430 | 18942 | 1894302 | 18942 | 18943075 | 18942 | 189420 | 1894200 | 18942000 +(3 rows) + +-- test partitioning parameterization +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM r1, coltest_part WHERE + id1 = id AND n1 > x1; + QUERY PLAN +--------------------------------------------------------------------- + Nested Loop (actual rows=3 loops=1) + -> Seq Scan on r1 (actual rows=5 loops=1) + -> Append (actual rows=1 loops=5) + -> Custom Scan (ColumnarScan) on coltest_part0 coltest_part_1 (actual rows=1 loops=3) + Filter: ((r1.n1 > x1) AND (r1.id1 = id)) + Rows Removed by Filter: 999 + Columnar Projected Columns: id, x1, x2, x3 + Columnar Chunk Group Filters: ((r1.n1 > x1) AND (r1.id1 = id)) + Columnar Chunk Groups Removed by Filter: 9 + -> Seq Scan on coltest_part1 coltest_part_2 (actual rows=0 loops=2) + Filter: ((r1.n1 > x1) AND (r1.id1 = id)) + Rows Removed by Filter: 10000 +(12 rows) + +SELECT * FROM r1, coltest_part WHERE + id1 = id AND n1 > x1; + id1 | n1 | id | x1 | x2 | x3 +--------------------------------------------------------------------- + 1234 | 12350 | 1234 | 12340 | 123400 | 1234000 + 9101 | 176000 | 9101 | 91010 | 910100 | 9101000 + 18942 | 189430 | 18942 | 189420 | 1894200 | 18942000 +(3 rows) + +set enable_mergejoin to default; +set enable_hashjoin to default; +set enable_material to default; +set columnar.planner_debug_level = 'notice'; +alter table coltest add column x5 int default (random()*20000)::int; +analyze coltest; +-- test that expressions on whole-row references are not pushed down +select * from coltest where coltest = (1,1,1,1); +NOTICE: columnar planner: cannot push down clause: var is whole-row reference or system column +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: unparameterized; 0 clauses pushed down + id | x1 | x2 | x3 | x5 +--------------------------------------------------------------------- +(0 rows) + +-- test that expressions on uncorrelated attributes are not pushed down +set columnar.qual_pushdown_correlation to default; +select * from coltest where x5 = 23484; +NOTICE: columnar planner: cannot push down clause: absolute correlation (X.YZ) of var attribute 5 is smaller than the value configured in "columnar.qual_pushdown_correlation_threshold" (0.900) +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: unparameterized; 0 clauses pushed down + id | x1 | x2 | x3 | x5 +--------------------------------------------------------------------- +(0 rows) + +-- test that expressions on volatile functions are not pushed down +create function vol() returns int language plpgsql as $$ +BEGIN + RETURN 1; +END; +$$; +select * from coltest where x3 = vol(); +NOTICE: columnar planner: cannot push down clause: expr contains volatile functions +NOTICE: columnar planner: adding CustomScan path for coltest +DETAIL: unparameterized; 0 clauses pushed down + id | x1 | x2 | x3 | x5 +--------------------------------------------------------------------- +(0 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) + SELECT * FROM coltest c1 WHERE ceil(x1) > 4222; +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: adding CustomScan path for c1 +DETAIL: unparameterized; 0 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on coltest c1 (actual rows=19577 loops=1) + Filter: (ceil((x1)::double precision) > '4222'::double precision) + Rows Removed by Filter: 423 + Columnar Projected Columns: id, x1, x2, x3, x5 +(4 rows) + +set columnar.planner_debug_level to default; +-- +-- https://github.com/citusdata/citus/issues/4488 +-- +create table columnar_prepared_stmt (x int, y int) using columnar; +insert into columnar_prepared_stmt select s, s from generate_series(1,5000000) s; +prepare foo (int) as select x from columnar_prepared_stmt where x = $1; +execute foo(3); + x +--------------------------------------------------------------------- + 3 +(1 row) + +execute foo(3); + x +--------------------------------------------------------------------- + 3 +(1 row) + +execute foo(3); + x +--------------------------------------------------------------------- + 3 +(1 row) + +execute foo(3); + x +--------------------------------------------------------------------- + 3 +(1 row) + +select filtered_row_count('execute foo(3)'); + filtered_row_count +--------------------------------------------------------------------- + 9999 +(1 row) + +select filtered_row_count('execute foo(3)'); + filtered_row_count +--------------------------------------------------------------------- + 9999 +(1 row) + +select filtered_row_count('execute foo(3)'); + filtered_row_count +--------------------------------------------------------------------- + 9999 +(1 row) + +select filtered_row_count('execute foo(3)'); + filtered_row_count +--------------------------------------------------------------------- + 9999 +(1 row) + +drop table columnar_prepared_stmt; +-- +-- https://github.com/citusdata/citus/issues/5258 +-- +set default_table_access_method to columnar; +CREATE TABLE atest1 ( a int, b text ); +CREATE TABLE atest2 (col1 varchar(10), col2 boolean); +INSERT INTO atest1 VALUES (1, 'one'); +SELECT * FROM atest1; -- ok + a | b +--------------------------------------------------------------------- + 1 | one +(1 row) + +SELECT * FROM atest2; -- ok + col1 | col2 +--------------------------------------------------------------------- +(0 rows) + +INSERT INTO atest1 VALUES (2, 'two'); -- ok +INSERT INTO atest1 SELECT 1, b FROM atest1; -- ok +SELECT * FROM atest2 WHERE ( col1 IN ( SELECT b FROM atest1 ) ); + col1 | col2 +--------------------------------------------------------------------- +(0 rows) + +CREATE TABLE t1 (name TEXT, n INTEGER); +CREATE TABLE t2 (name TEXT, n INTEGER); +CREATE TABLE t3 (name TEXT, n INTEGER); +INSERT INTO t1 VALUES ( 'bb', 11 ); +INSERT INTO t2 VALUES ( 'bb', 12 ); +INSERT INTO t2 VALUES ( 'cc', 22 ); +INSERT INTO t2 VALUES ( 'ee', 42 ); +INSERT INTO t3 VALUES ( 'bb', 13 ); +INSERT INTO t3 VALUES ( 'cc', 23 ); +INSERT INTO t3 VALUES ( 'dd', 33 ); +SELECT * FROM +(SELECT name, n as s1_n, 1 as s1_1 FROM t1) as s1 +NATURAL INNER JOIN +(SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 +NATURAL INNER JOIN +(SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; + name | s1_n | s1_1 | s2_n | s2_2 | s3_n | s3_2 +--------------------------------------------------------------------- + bb | 11 | 1 | 12 | 2 | 13 | 3 +(1 row) + +CREATE TABLE numrange_test (nr NUMRANGE); +INSERT INTO numrange_test VALUES('[,)'); +INSERT INTO numrange_test VALUES('[3,]'); +INSERT INTO numrange_test VALUES('[, 5)'); +INSERT INTO numrange_test VALUES(numrange(1.1, 2.2)); +INSERT INTO numrange_test VALUES('empty'); +INSERT INTO numrange_test VALUES(numrange(1.7, 1.7, '[]')); +create table numrange_test2(nr numrange); +INSERT INTO numrange_test2 VALUES('[, 5)'); +INSERT INTO numrange_test2 VALUES(numrange(1.1, 2.2)); +INSERT INTO numrange_test2 VALUES(numrange(1.1, 2.2)); +INSERT INTO numrange_test2 VALUES(numrange(1.1, 2.2,'()')); +INSERT INTO numrange_test2 VALUES('empty'); +set enable_nestloop=t; +set enable_hashjoin=f; +set enable_mergejoin=f; +select * from numrange_test natural join numrange_test2 order by nr; + nr +--------------------------------------------------------------------- + empty + (,5) + [1.1,2.2) + [1.1,2.2) +(4 rows) + +DROP TABLE atest1, atest2, t1, t2, t3, numrange_test, numrange_test2; +set default_table_access_method to default; +set columnar.planner_debug_level to notice; +BEGIN; + SET LOCAL columnar.stripe_row_limit = 2000; + SET LOCAL columnar.chunk_group_row_limit = 1000; + create table pushdown_test (a int, b int) using columnar; + insert into pushdown_test values (generate_series(1, 200000)); +COMMIT; +SET columnar.max_custom_scan_paths TO 50; +SET columnar.qual_pushdown_correlation_threshold TO 0.0; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556; +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=2 loops=1) + Filter: ((a = 204356) OR (a = 104356) OR (a = 76556)) + Rows Removed by Filter: 1998 + Columnar Projected Columns: a + Columnar Chunk Group Filters: ((a = 204356) OR (a = 104356) OR (a = 76556)) + Columnar Chunk Groups Removed by Filter: 198 +(7 rows) + +SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a = 104356 or a = 76556; +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + 180912 +(1 row) + +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556; +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1) + Filter: ((a = 194356) OR (a = 104356) OR (a = 76556)) + Rows Removed by Filter: 2997 + Columnar Projected Columns: a + Columnar Chunk Group Filters: ((a = 194356) OR (a = 104356) OR (a = 76556)) + Columnar Chunk Groups Removed by Filter: 197 +(7 rows) + +SELECT sum(a) FROM pushdown_test WHERE a = 194356 or a = 104356 or a = 76556; +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + 375268 +(1 row) + +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test WHERE a = 204356 or a > a*-1 + b; +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 0 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1) + Filter: ((a = 204356) OR (a > ((a * '-1'::integer) + b))) + Rows Removed by Filter: 200000 + Columnar Projected Columns: a, b +(5 rows) + +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000); +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=38998 loops=1) + Filter: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000))) + Rows Removed by Filter: 2 + Columnar Projected Columns: a + Columnar Chunk Group Filters: (((a > 1000) AND (a < 10000)) OR ((a > 20000) AND (a < 50000))) + Columnar Chunk Groups Removed by Filter: 161 +(7 rows) + +SELECT sum(a) FROM pushdown_test where (a > 1000 and a < 10000) or (a > 20000 and a < 50000); +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + 1099459500 +(1 row) + +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above +NOTICE: columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 0 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=200000 loops=1) + Filter: ((((a)::double precision > random()) AND (a < (2 * a))) OR (a > 100)) + Columnar Projected Columns: a +(4 rows) + +SELECT sum(a) FROM pushdown_test where (a > random() and a < 2*a) or (a > 100); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: none of the arguments were pushdownable, due to the reason(s) given above +NOTICE: columnar planner: cannot push down clause: all arguments of an OR expression must be pushdownable but one of them was not, due to the reason given above +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 0 clauses pushed down + sum +--------------------------------------------------------------------- + 20000100000 +(1 row) + +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=3010 loops=1) + Filter: ((((a)::double precision > random()) AND (a <= 2000)) OR (a > 198990)) + Rows Removed by Filter: 990 + Columnar Projected Columns: a + Columnar Chunk Group Filters: ((a <= 2000) OR (a > 198990)) + Columnar Chunk Groups Removed by Filter: 196 +(7 rows) + +SELECT sum(a) FROM pushdown_test where (a > random() and a <= 2000) or (a > 200000-1010); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + 203491455 +(1 row) + +SET hash_mem_multiplier = 1.0; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test where +( + a > random() + and + ( + (a < 200 and a not in (select a from pushdown_test)) or + (a > 1000 and a < 2000) + ) +) +or +(a > 200000-2010); +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must not contain a subplan +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=3009 loops=1) + Filter: ((((a)::double precision > random()) AND (((a < 200) AND (NOT (SubPlan 1))) OR ((a > 1000) AND (a < 2000)))) OR (a > 197990)) + Rows Removed by Filter: 1991 + Columnar Projected Columns: a + Columnar Chunk Group Filters: (((a < 200) OR ((a > 1000) AND (a < 2000))) OR (a > 197990)) + Columnar Chunk Groups Removed by Filter: 195 + SubPlan 1 + -> Materialize (actual rows=100 loops=199) + -> Custom Scan (ColumnarScan) on pushdown_test pushdown_test_1 (actual rows=199 loops=1) + Columnar Projected Columns: a +(11 rows) + +RESET hash_mem_multiplier; +SELECT sum(a) FROM pushdown_test where +( + a > random() + and + ( + (a < 200 and a not in (select a from pushdown_test)) or + (a > 1000 and a < 2000) + ) +) +or +(a > 200000-2010); +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 0 clauses pushed down +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must not contain a subplan +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + 401479455 +(1 row) + +create function stable_1(arg int) returns int language plpgsql STRICT IMMUTABLE as +$$ BEGIN RETURN 1+arg; END; $$; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000)); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + QUERY PLAN +--------------------------------------------------------------------- + Aggregate (actual rows=1 loops=1) + -> Custom Scan (ColumnarScan) on pushdown_test (actual rows=0 loops=1) + Filter: ((a < 6001) AND ((a)::double precision = random()) AND (a < stable_1(a))) + Rows Removed by Filter: 6000 + Columnar Projected Columns: a + Columnar Chunk Group Filters: (a < 6001) + Columnar Chunk Groups Removed by Filter: 194 +(7 rows) + +SELECT sum(a) FROM pushdown_test where (a = random() and a < stable_1(a) and a < stable_1(6000)); +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: cannot push down clause: must match 'Var Expr' or 'Expr Var' +HINT: Var must only reference this rel, and Expr must not reference this rel +NOTICE: columnar planner: adding CustomScan path for pushdown_test +DETAIL: unparameterized; 1 clauses pushed down + sum +--------------------------------------------------------------------- + +(1 row) + +RESET columnar.max_custom_scan_paths; +RESET columnar.qual_pushdown_correlation_threshold; +RESET columnar.planner_debug_level; +DROP TABLE pushdown_test; +-- https://github.com/citusdata/citus/issues/5803 +CREATE TABLE pushdown_test(id int, country text) using columnar; +BEGIN; + INSERT INTO pushdown_test VALUES(1, 'AL'); + INSERT INTO pushdown_test VALUES(2, 'AU'); +END; +BEGIN; + INSERT INTO pushdown_test VALUES(3, 'BR'); + INSERT INTO pushdown_test VALUES(4, 'BT'); +END; +BEGIN; + INSERT INTO pushdown_test VALUES(5, 'PK'); + INSERT INTO pushdown_test VALUES(6, 'PA'); +END; +BEGIN; + INSERT INTO pushdown_test VALUES(7, 'USA'); + INSERT INTO pushdown_test VALUES(8, 'ZW'); +END; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT id FROM pushdown_test WHERE country IN ('USA', 'BR', 'ZW'); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1) + Filter: (country = ANY ('{USA,BR,ZW}'::text[])) + Rows Removed by Filter: 1 + Columnar Projected Columns: id, country + Columnar Chunk Group Filters: (country = ANY ('{USA,BR,ZW}'::text[])) + Columnar Chunk Groups Removed by Filter: 2 +(6 rows) + +SELECT id FROM pushdown_test WHERE country IN ('USA', 'BR', 'ZW'); + id +--------------------------------------------------------------------- + 3 + 7 + 8 +(3 rows) + +-- test for volatile functions with IN +CREATE FUNCTION volatileFunction() returns TEXT language plpgsql AS +$$ +BEGIN + return 'AL'; +END; +$$; +EXPLAIN (analyze on, costs off, timing off, summary off) +SELECT * FROM pushdown_test WHERE country IN ('USA', 'ZW', volatileFunction()); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (ColumnarScan) on pushdown_test (actual rows=3 loops=1) + Filter: (country = ANY (ARRAY['USA'::text, 'ZW'::text, volatilefunction()])) + Rows Removed by Filter: 5 + Columnar Projected Columns: id, country +(4 rows) + +SELECT * FROM pushdown_test WHERE country IN ('USA', 'ZW', volatileFunction()); + id | country +--------------------------------------------------------------------- + 1 | AL + 7 | USA + 8 | ZW +(3 rows) + +DROP TABLE pushdown_test; diff --git a/src/test/regress/expected/columnar_memory.out b/src/test/regress/expected/columnar_memory.out index 865472da1..229502437 100644 --- a/src/test/regress/expected/columnar_memory.out +++ b/src/test/regress/expected/columnar_memory.out @@ -77,10 +77,10 @@ FROM columnar_test_helpers.columnar_store_memory_stats(); top_growth | 1 -- before this change, max mem usage while executing inserts was 28MB and --- with this change it's less than 8MB. +-- with this change it's less than 9MB. SELECT - (SELECT max(memusage) < 8 * 1024 * 1024 FROM t WHERE tag='large batch') AS large_batch_ok, - (SELECT max(memusage) < 8 * 1024 * 1024 FROM t WHERE tag='first batch') AS first_batch_ok; + (SELECT max(memusage) < 9 * 1024 * 1024 FROM t WHERE tag='large batch') AS large_batch_ok, + (SELECT max(memusage) < 9 * 1024 * 1024 FROM t WHERE tag='first batch') AS first_batch_ok; -[ RECORD 1 ]--+-- large_batch_ok | t first_batch_ok | t diff --git a/src/test/regress/expected/columnar_paths.out b/src/test/regress/expected/columnar_paths.out index 4b32361fa..07b91a42e 100644 --- a/src/test/regress/expected/columnar_paths.out +++ b/src/test/regress/expected/columnar_paths.out @@ -326,7 +326,7 @@ WHERE w2.a = 123; EXPLAIN (COSTS OFF) SELECT sub_1.b, sub_2.a, sub_3.avg FROM - (SELECT b FROM full_correlated WHERE (a > 2) GROUP BY b HAVING count(DISTINCT a) > 0 ORDER BY 1 DESC LIMIT 5) AS sub_1, + (SELECT b FROM full_correlated WHERE (a > 2) GROUP BY b ORDER BY 1 DESC LIMIT 5) AS sub_1, (SELECT a FROM full_correlated WHERE (a > 10) GROUP BY a HAVING count(DISTINCT a) >= 1 ORDER BY 1 DESC LIMIT 3) AS sub_2, (SELECT avg(a) AS AVG FROM full_correlated WHERE (a > 2) GROUP BY a HAVING sum(a) > 10 ORDER BY (sum(d) - avg(a) - COALESCE(array_upper(ARRAY[max(a)],1) * 5, 0)) DESC LIMIT 3) AS sub_3 WHERE sub_2.a < sub_1.b::integer @@ -341,11 +341,10 @@ LIMIT 100; -> Nested Loop Join Filter: (full_correlated_1.a < (full_correlated.b)::integer) -> Limit - -> GroupAggregate - Group Key: full_correlated.b - Filter: (count(DISTINCT full_correlated.a) > 0) - -> Sort - Sort Key: full_correlated.b DESC + -> Sort + Sort Key: full_correlated.b DESC + -> HashAggregate + Group Key: full_correlated.b -> Custom Scan (ColumnarScan) on full_correlated Filter: (a > 2) Columnar Projected Columns: a, b @@ -366,7 +365,7 @@ LIMIT 100; Filter: (sum(full_correlated_2.a) > 10) -> Index Scan using full_correlated_btree on full_correlated full_correlated_2 Index Cond: (a > 2) -(32 rows) +(31 rows) DROP INDEX full_correlated_btree; CREATE INDEX full_correlated_hash ON full_correlated USING hash(a); diff --git a/src/test/regress/expected/create_role_propagation.out b/src/test/regress/expected/create_role_propagation.out index c5111b05f..59f7948a1 100644 --- a/src/test/regress/expected/create_role_propagation.out +++ b/src/test/regress/expected/create_role_propagation.out @@ -244,13 +244,13 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port); 1 (1 row) -SELECT roleid::regrole::text AS role, member::regrole::text, grantor::regrole::text, admin_option FROM pg_auth_members WHERE roleid::regrole::text LIKE '%dist\_%' ORDER BY 1, 2; - role | member | grantor | admin_option +SELECT roleid::regrole::text AS role, member::regrole::text, (grantor::regrole::text IN ('postgres', 'non_dist_role_1', 'dist_role_1')) AS grantor, admin_option FROM pg_auth_members WHERE roleid::regrole::text LIKE '%dist\_%' ORDER BY 1, 2; + role | member | grantor | admin_option --------------------------------------------------------------------- - dist_role_1 | dist_role_2 | non_dist_role_1 | f - dist_role_3 | non_dist_role_3 | postgres | f - non_dist_role_1 | non_dist_role_2 | dist_role_1 | f - non_dist_role_4 | dist_role_4 | postgres | f + dist_role_1 | dist_role_2 | t | f + dist_role_3 | non_dist_role_3 | t | f + non_dist_role_1 | non_dist_role_2 | t | f + non_dist_role_4 | dist_role_4 | t | f (4 rows) SELECT objid::regrole FROM pg_catalog.pg_dist_object WHERE classid='pg_authid'::regclass::oid AND objid::regrole::text LIKE '%dist\_%' ORDER BY 1; diff --git a/src/test/regress/expected/global_cancel.out b/src/test/regress/expected/global_cancel.out index df50dbe3f..5adeef3c8 100644 --- a/src/test/regress/expected/global_cancel.out +++ b/src/test/regress/expected/global_cancel.out @@ -67,12 +67,12 @@ SELECT pg_typeof(:maintenance_daemon_gpid); bigint (1 row) +\set VERBOSITY terse SELECT pg_cancel_backend(:maintenance_daemon_gpid); ERROR: must be a superuser to cancel superuser query -CONTEXT: while executing command on localhost:xxxxx SELECT pg_terminate_backend(:maintenance_daemon_gpid); ERROR: must be a superuser to terminate superuser process -CONTEXT: while executing command on localhost:xxxxx +\set VERBOSITY default -- we can cancel our own backend SELECT pg_cancel_backend(citus_backend_gpid()); ERROR: canceling statement due to user request diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out index 88acc49e3..476aa8640 100644 --- a/src/test/regress/expected/insert_select_repartition.out +++ b/src/test/regress/expected/insert_select_repartition.out @@ -1214,7 +1214,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, @@ -1232,7 +1232,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, @@ -1247,7 +1247,7 @@ DO UPDATE SET -> Task Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Group Key: c1, c2, c3, c4, '-1'::double precision, insert_select_repartition.dist_func(c1, 4) + Group Key: c1, c2, c3, c4, insert_select_repartition.dist_func(c1, 4) -> Seq Scan on source_table_4213644 source_table (10 rows) diff --git a/src/test/regress/expected/insert_select_repartition_0.out b/src/test/regress/expected/insert_select_repartition_0.out index 7217be3e9..904bd215a 100644 --- a/src/test/regress/expected/insert_select_repartition_0.out +++ b/src/test/regress/expected/insert_select_repartition_0.out @@ -1214,7 +1214,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, @@ -1232,7 +1232,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, @@ -1247,7 +1247,7 @@ DO UPDATE SET -> Task Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Group Key: c1, c2, c3, c4, '-1'::double precision, insert_select_repartition.dist_func(c1, 4) + Group Key: c1, c2, c3, c4, insert_select_repartition.dist_func(c1, 4) -> Seq Scan on source_table_4213644 source_table (10 rows) diff --git a/src/test/regress/expected/isolation_add_remove_node.out b/src/test/regress/expected/isolation_add_remove_node.out index 9248ed5be..15a5c1a58 100644 --- a/src/test/regress/expected/isolation_add_remove_node.out +++ b/src/test/regress/expected/isolation_add_remove_node.out @@ -698,9 +698,17 @@ master_remove_node (1 row) -starting permutation: s1-add-node-1 s1-begin s1-disable-node-1 s2-disable-node-1 s1-abort s1-show-nodes +starting permutation: s1-add-node-1 s1-add-node-2 s1-begin s1-disable-node-2 s2-disable-node-2 s1-abort s2-wait-metadata-sync s1-show-nodes step s1-add-node-1: - SELECT 1 FROM master_add_node('localhost', 57637); + SELECT 1 FROM master_add_node('localhost', 57637); + +?column? +--------------------------------------------------------------------- + 1 +(1 row) + +step s1-add-node-2: + SELECT 1 FROM master_add_node('localhost', 57638); ?column? --------------------------------------------------------------------- @@ -708,11 +716,11 @@ step s1-add-node-1: (1 row) step s1-begin: - BEGIN; + BEGIN; -step s1-disable-node-1: - SELECT 1 FROM master_disable_node('localhost', 57637); - SELECT public.wait_until_metadata_sync(); +step s1-disable-node-2: + SELECT 1 FROM master_disable_node('localhost', 57638); + SELECT public.wait_until_metadata_sync(); ?column? --------------------------------------------------------------------- @@ -724,34 +732,38 @@ wait_until_metadata_sync (1 row) -step s2-disable-node-1: - SELECT 1 FROM master_disable_node('localhost', 57637); - SELECT public.wait_until_metadata_sync(); +step s2-disable-node-2: + SELECT 1 FROM master_disable_node('localhost', 57638); step s1-abort: - ABORT; + ABORT; -step s2-disable-node-1: <... completed> +step s2-disable-node-2: <... completed> ?column? --------------------------------------------------------------------- 1 (1 row) +step s2-wait-metadata-sync: + SELECT public.wait_until_metadata_sync(); + wait_until_metadata_sync --------------------------------------------------------------------- (1 row) step s1-show-nodes: - SELECT nodename, nodeport, isactive FROM pg_dist_node ORDER BY nodename, nodeport; + SELECT nodename, nodeport, isactive FROM pg_dist_node ORDER BY nodename, nodeport; nodename |nodeport|isactive --------------------------------------------------------------------- -localhost| 57637|f -(1 row) +localhost| 57637|t +localhost| 57638|f +(2 rows) master_remove_node --------------------------------------------------------------------- -(1 row) + +(2 rows) diff --git a/src/test/regress/expected/isolation_update_node.out b/src/test/regress/expected/isolation_update_node.out index 86615648c..1a1c65ec8 100644 --- a/src/test/regress/expected/isolation_update_node.out +++ b/src/test/regress/expected/isolation_update_node.out @@ -3,8 +3,8 @@ Parsed test spec with 3 sessions starting permutation: s1-begin s1-update-node-1 s2-update-node-2 s1-commit s1-show-nodes s3-update-node-1-back s3-update-node-2-back s3-manually-fix-metadata nodeid|nodename |nodeport --------------------------------------------------------------------- - 22|localhost| 57638 - 21|localhost| 57637 + 23|localhost| 57638 + 22|localhost| 57637 (2 rows) step s1-begin: @@ -43,8 +43,8 @@ step s1-show-nodes: nodeid|nodename |nodeport|isactive --------------------------------------------------------------------- - 21|localhost| 58637|t - 22|localhost| 58638|t + 22|localhost| 58637|t + 23|localhost| 58638|t (2 rows) step s3-update-node-1-back: @@ -93,8 +93,8 @@ nodeid|nodename|nodeport starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes s3-update-node-1-back s3-manually-fix-metadata nodeid|nodename |nodeport --------------------------------------------------------------------- - 24|localhost| 57638 - 23|localhost| 57637 + 25|localhost| 57638 + 24|localhost| 57637 (2 rows) step s1-begin: @@ -139,8 +139,8 @@ step s1-show-nodes: nodeid|nodename |nodeport|isactive --------------------------------------------------------------------- - 24|localhost| 57638|t - 23|localhost| 58637|t + 25|localhost| 57638|t + 24|localhost| 58637|t (2 rows) step s3-update-node-1-back: @@ -178,8 +178,8 @@ nodeid|nodename|nodeport starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s3-manually-fix-metadata nodeid|nodename |nodeport --------------------------------------------------------------------- - 26|localhost| 57638 - 25|localhost| 57637 + 27|localhost| 57638 + 26|localhost| 57637 (2 rows) step s2-create-table: diff --git a/src/test/regress/expected/local_dist_join_mixed.out b/src/test/regress/expected/local_dist_join_mixed.out index 20287ee35..b8f074c73 100644 --- a/src/test/regress/expected/local_dist_join_mixed.out +++ b/src/test/regress/expected/local_dist_join_mixed.out @@ -357,13 +357,13 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c 101 (1 row) -CREATE VIEW local_regular_view AS SELECT * FROM local; +CREATE VIEW local_regular_view AS SELECT * FROM local table_name_for_view; WARNING: "view local_regular_view" has dependency to "table local" that is not in Citus' metadata DETAIL: "view local_regular_view" will be created only locally HINT: Distribute "table local" first to distribute "view local_regular_view" CREATE VIEW dist_regular_view AS SELECT * FROM distributed; SELECT count(*) FROM distributed JOIN local_regular_view USING (id); -DEBUG: generating subplan XXX_1 for subquery SELECT local.id, local.title FROM local_dist_join_mixed.local +DEBUG: generating subplan XXX_1 for subquery SELECT id, title FROM local_dist_join_mixed.local table_name_for_view DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (local_dist_join_mixed.distributed JOIN (SELECT intermediate_result.id, intermediate_result.title FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint, title text)) local_regular_view USING (id)) count --------------------------------------------------------------------- @@ -380,7 +380,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c (1 row) SELECT count(*) FROM dist_regular_view JOIN local_regular_view USING (id); -DEBUG: generating subplan XXX_1 for subquery SELECT local.id, local.title FROM local_dist_join_mixed.local +DEBUG: generating subplan XXX_1 for subquery SELECT id, title FROM local_dist_join_mixed.local table_name_for_view DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT distributed.id, distributed.name, distributed.created_at FROM local_dist_join_mixed.distributed) dist_regular_view JOIN (SELECT intermediate_result.id, intermediate_result.title FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint, title text)) local_regular_view USING (id)) count --------------------------------------------------------------------- diff --git a/src/test/regress/expected/local_table_join.out b/src/test/regress/expected/local_table_join.out index 7da341207..297959d41 100644 --- a/src/test/regress/expected/local_table_join.out +++ b/src/test/regress/expected/local_table_join.out @@ -1370,9 +1370,6 @@ select typdefault from ( select a from tbl where typdefault > 'a' limit 1) as subq_0 - where ( - select true as bool from pg_catalog.pg_am limit 1 - ) ) as subq_1 ) as subq_2; typdefault @@ -1400,15 +1397,11 @@ select typdefault from ( select a from tbl where typdefault > 'a' limit 1) as subq_0 - where ( - select true as bool from pg_catalog.pg_am limit 1 - ) ) as subq_1 ) as subq_2; -DEBUG: generating subplan XXX_1 for subquery SELECT true AS bool FROM pg_am LIMIT 1 DEBUG: Wrapping relation "custom_pg_type" to a subquery -DEBUG: generating subplan XXX_2 for subquery SELECT typdefault FROM local_table_join.custom_pg_type WHERE true -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT typdefault FROM (SELECT subq_1.typdefault FROM (SELECT custom_pg_type.typdefault FROM (SELECT custom_pg_type_1.typdefault FROM (SELECT intermediate_result.typdefault FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(typdefault text)) custom_pg_type_1) custom_pg_type, LATERAL (SELECT tbl.a FROM local_table_join.tbl WHERE (custom_pg_type.typdefault OPERATOR(pg_catalog.>) 'a'::text) LIMIT 1) subq_0 WHERE (SELECT intermediate_result.bool FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(bool boolean))) subq_1) subq_2 +DEBUG: generating subplan XXX_1 for subquery SELECT typdefault FROM local_table_join.custom_pg_type WHERE true +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT typdefault FROM (SELECT subq_1.typdefault FROM (SELECT custom_pg_type.typdefault FROM (SELECT custom_pg_type_1.typdefault FROM (SELECT intermediate_result.typdefault FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(typdefault text)) custom_pg_type_1) custom_pg_type, LATERAL (SELECT tbl.a FROM local_table_join.tbl WHERE (custom_pg_type.typdefault OPERATOR(pg_catalog.>) 'a'::text) LIMIT 1) subq_0) subq_1) subq_2 ERROR: cannot push down this subquery DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables -- Not supported because of 4470 diff --git a/src/test/regress/expected/metadata_sync_helpers.out b/src/test/regress/expected/metadata_sync_helpers.out index 62268b32f..29d62c46a 100644 --- a/src/test/regress/expected/metadata_sync_helpers.out +++ b/src/test/regress/expected/metadata_sync_helpers.out @@ -1284,8 +1284,17 @@ BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; SET application_name to 'citus_internal gpid=10000000001'; -- with an ugly trick, update the vartype of table from int to bigint -- so that making two tables colocated fails - UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1}' + -- include varnullingrels for PG16 + SHOW server_version \gset + SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 + \gset + \if :server_version_ge_16 + UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 1 :varnoold 1 :varoattno 1 :location -1}' WHERE logicalrelid = 'test_2'::regclass; + \else + UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varlevelsup 1 :varnoold 1 :varoattno 1 :location -1}' + WHERE logicalrelid = 'test_2'::regclass; + \endif SELECT citus_internal_update_relation_colocation('test_2'::regclass, 251); ERROR: cannot colocate tables test_2 and test_3 ROLLBACK; diff --git a/src/test/regress/expected/multi_complex_count_distinct.out b/src/test/regress/expected/multi_complex_count_distinct.out index d4e6ecfa3..baa9c829a 100644 --- a/src/test/regress/expected/multi_complex_count_distinct.out +++ b/src/test/regress/expected/multi_complex_count_distinct.out @@ -1,6 +1,18 @@ -- -- COMPLEX_COUNT_DISTINCT -- +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + t +(1 row) + SET citus.next_shard_id TO 240000; SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; @@ -65,7 +77,7 @@ SELECT GROUP BY l_orderkey ORDER BY 2 DESC, 1 DESC LIMIT 10; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit Output: remote_scan.l_orderkey, remote_scan.count @@ -87,9 +99,12 @@ SELECT -> GroupAggregate Output: l_orderkey, count(DISTINCT l_partkey) Group Key: lineitem_hash.l_orderkey - -> Index Scan Backward using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) + -> Sort + Output: l_orderkey, l_partkey + Sort Key: lineitem_hash.l_orderkey DESC, lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey +(25 rows) -- it is also supported if there is no grouping or grouping is on non-partition field SELECT @@ -108,7 +123,7 @@ SELECT FROM lineitem_hash ORDER BY 1 DESC LIMIT 10; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit Output: (count(DISTINCT remote_scan.count)) @@ -117,19 +132,22 @@ SELECT Sort Key: (count(DISTINCT remote_scan.count)) DESC -> Aggregate Output: count(DISTINCT remote_scan.count) - -> Custom Scan (Citus Adaptive) + -> Sort Output: remote_scan.count - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Query: SELECT l_partkey AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_partkey - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_partkey - Group Key: lineitem_hash.l_partkey - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(19 rows) + Sort Key: remote_scan.count + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_partkey AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_partkey + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_partkey + Group Key: lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) SELECT l_shipmode, count(DISTINCT l_partkey) @@ -167,7 +185,7 @@ SELECT Group Key: remote_scan.l_shipmode -> Sort Output: remote_scan.l_shipmode, remote_scan.count - Sort Key: remote_scan.l_shipmode DESC + Sort Key: remote_scan.l_shipmode DESC, remote_scan.count -> Custom Scan (Citus Adaptive) Output: remote_scan.l_shipmode, remote_scan.count Task Count: 8 @@ -210,7 +228,7 @@ SELECT GROUP BY l_orderkey ORDER BY 3 DESC, 2 DESC, 1 LIMIT 10; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 @@ -232,9 +250,12 @@ SELECT -> GroupAggregate Output: l_orderkey, count(DISTINCT l_partkey), count(DISTINCT l_shipmode) Group Key: lineitem_hash.l_orderkey - -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) + -> Sort + Output: l_orderkey, l_partkey, l_shipmode + Sort Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_shipmode +(25 rows) -- partition/non-partition column count distinct no grouping SELECT @@ -249,23 +270,26 @@ EXPLAIN (COSTS false, VERBOSE true) SELECT count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) FROM lineitem_hash; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Aggregate Output: count(DISTINCT remote_scan.count), count(DISTINCT remote_scan.count_1), count(DISTINCT remote_scan.count_2) - -> Custom Scan (Citus Adaptive) + -> Sort Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Query: SELECT l_orderkey AS count, l_partkey AS count, l_shipmode AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey, l_partkey, l_shipmode - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_orderkey, l_partkey, l_shipmode - Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(14 rows) + Sort Key: remote_scan.count + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_orderkey AS count, l_partkey AS count, l_shipmode AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey, l_partkey, l_shipmode + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_orderkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(17 rows) -- distinct/non-distinct on partition and non-partition columns SELECT @@ -433,7 +457,7 @@ SELECT * Group Key: lineitem_hash.l_partkey -> Sort Output: l_partkey, l_orderkey - Sort Key: lineitem_hash.l_partkey + Sort Key: lineitem_hash.l_partkey, lineitem_hash.l_orderkey -> Seq Scan on public.lineitem_hash_240000 lineitem_hash Output: l_partkey, l_orderkey Task Count: 1 @@ -483,7 +507,7 @@ SELECT GROUP BY l_orderkey ORDER BY 2 DESC, 3 DESC, 1 LIMIT 10; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 @@ -505,9 +529,12 @@ SELECT -> GroupAggregate Output: l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar)), count(DISTINCT l_suppkey) Group Key: lineitem_hash.l_orderkey - -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) + -> Sort + Output: l_orderkey, l_suppkey, l_shipmode + Sort Key: lineitem_hash.l_orderkey, lineitem_hash.l_suppkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_suppkey, l_shipmode +(25 rows) -- group by on non-partition column SELECT @@ -550,7 +577,7 @@ SELECT Group Key: remote_scan.l_suppkey -> Sort Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 - Sort Key: remote_scan.l_suppkey DESC + Sort Key: remote_scan.l_suppkey DESC, remote_scan.count -> Custom Scan (Citus Adaptive) Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 Task Count: 8 diff --git a/src/test/regress/expected/multi_complex_count_distinct_0.out b/src/test/regress/expected/multi_complex_count_distinct_0.out new file mode 100644 index 000000000..36af62e96 --- /dev/null +++ b/src/test/regress/expected/multi_complex_count_distinct_0.out @@ -0,0 +1,1139 @@ +-- +-- COMPLEX_COUNT_DISTINCT +-- +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + f +(1 row) + +SET citus.next_shard_id TO 240000; +SET citus.shard_count TO 8; +SET citus.shard_replication_factor TO 1; +SET citus.coordinator_aggregation_strategy TO 'disabled'; +CREATE TABLE lineitem_hash ( + l_orderkey bigint not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimal(15, 2) not null, + l_extendedprice decimal(15, 2) not null, + l_discount decimal(15, 2) not null, + l_tax decimal(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null, + PRIMARY KEY(l_orderkey, l_linenumber) ); +SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\set lineitem_1_data_file :abs_srcdir '/data/lineitem.1.data' +\set lineitem_2_data_file :abs_srcdir '/data/lineitem.2.data' +\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_1_data_file' ' with delimiter '''|''';' +:client_side_copy_command +\set client_side_copy_command '\\copy lineitem_hash FROM ' :'lineitem_2_data_file' ' with delimiter '''|''';' +:client_side_copy_command +ANALYZE lineitem_hash; +-- count(distinct) is supported on top level query if there +-- is a grouping on the partition key +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey DESC + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_orderkey, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_orderkey, count(DISTINCT l_partkey) AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey ORDER BY (count(DISTINCT l_partkey)) DESC, l_orderkey DESC LIMIT '10'::bigint + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey DESC + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan Backward using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- it is also supported if there is no grouping or grouping is on non-partition field +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + count +--------------------------------------------------------------------- + 11661 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: (count(DISTINCT remote_scan.count)) + -> Sort + Output: (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC + -> Aggregate + Output: count(DISTINCT remote_scan.count) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_partkey AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_partkey + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_partkey + Group Key: lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(19 rows) + +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1757 + MAIL | 1730 + AIR | 1702 + FOB | 1700 + RAIL | 1696 + SHIP | 1684 + REG AIR | 1676 +(7 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + -> Sort + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC, remote_scan.l_shipmode DESC + -> GroupAggregate + Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) + Group Key: remote_scan.l_shipmode + -> Sort + Output: remote_scan.l_shipmode, remote_scan.count + Sort Key: remote_scan.l_shipmode DESC + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_shipmode, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_shipmode, l_partkey AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_shipmode, l_partkey + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_shipmode, l_partkey + Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(23 rows) + +-- mixed mode count distinct, grouped by partition column +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 226 | 7 | 7 + 1316 | 7 | 7 + 1477 | 7 | 7 + 3555 | 7 | 7 + 12258 | 7 | 7 + 12835 | 7 | 7 + 768 | 7 | 6 + 1121 | 7 | 6 + 1153 | 7 | 6 + 1281 | 7 | 6 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count_1 DESC, remote_scan.count DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_orderkey, count(DISTINCT l_partkey) AS count, count(DISTINCT l_shipmode) AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey ORDER BY (count(DISTINCT l_shipmode)) DESC, (count(DISTINCT l_partkey)) DESC, l_orderkey LIMIT '10'::bigint + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + Sort Key: (count(DISTINCT lineitem_hash.l_shipmode)) DESC, (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- partition/non-partition column count distinct no grouping +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 2985 | 11661 | 7 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count), count(DISTINCT remote_scan.count_1), count(DISTINCT remote_scan.count_2) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_orderkey AS count, l_partkey AS count, l_shipmode AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey, l_partkey, l_shipmode + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_orderkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(14 rows) + +-- distinct/non-distinct on partition and non-partition columns +SELECT + count(distinct l_orderkey), count(l_orderkey), + count(distinct l_partkey), count(l_partkey), + count(distinct l_shipmode), count(l_shipmode) + FROM lineitem_hash; + count | count | count | count | count | count +--------------------------------------------------------------------- + 2985 | 12000 | 11661 | 12000 | 7 | 12000 +(1 row) + +-- mixed mode count distinct, grouped by non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 1, 2 DESC, 3 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + FOB | 1700 | 1276 + MAIL | 1730 | 1299 + RAIL | 1696 | 1265 + REG AIR | 1676 | 1275 + SHIP | 1684 | 1289 + TRUCK | 1757 | 1333 +(7 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + TRUCK | 1757 | 1333 +(2 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + TRUCK | 1757 +(2 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_suppkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1564 + FOB | 1700 | 1571 + MAIL | 1730 | 1573 + RAIL | 1696 | 1581 + REG AIR | 1676 | 1557 + SHIP | 1684 | 1554 + TRUCK | 1757 | 1602 +(7 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + FOB | 1700 + MAIL | 1730 + RAIL | 1696 + REG AIR | 1676 + SHIP | 1684 + TRUCK | 1757 +(7 rows) + +-- count distinct is supported on single table subqueries +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_partkey | count +--------------------------------------------------------------------- + 199146 | 3 + 188804 | 3 + 177771 | 3 + 160895 | 3 + 149926 | 3 + 136884 | 3 + 87761 | 3 + 15283 | 3 + 6983 | 3 + 1927 | 3 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Output: remote_scan.l_partkey, remote_scan.count + -> Distributed Subplan XXX_1 + -> HashAggregate + Output: remote_scan.l_partkey, COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) + Group Key: remote_scan.l_partkey + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_partkey, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_partkey, count(DISTINCT l_orderkey) AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_partkey + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Output: l_partkey, count(DISTINCT l_orderkey) + Group Key: lineitem_hash.l_partkey + -> Sort + Output: l_partkey, l_orderkey + Sort Key: lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_partkey, l_orderkey + Task Count: 1 + Tasks Shown: All + -> Task + Query: SELECT l_partkey, count FROM (SELECT intermediate_result.l_partkey, intermediate_result.count FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(l_partkey integer, count bigint)) sub ORDER BY count DESC, l_partkey DESC LIMIT 10 + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: intermediate_result.l_partkey, intermediate_result.count + -> Sort + Output: intermediate_result.l_partkey, intermediate_result.count + Sort Key: intermediate_result.count DESC, intermediate_result.l_partkey DESC + -> Function Scan on pg_catalog.read_intermediate_result intermediate_result + Output: intermediate_result.l_partkey, intermediate_result.count + Function Call: read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) +(34 rows) + +-- count distinct with filters +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 4964 | 4 | 7 + 12005 | 4 | 7 + 5409 | 4 | 6 + 164 | 3 | 7 + 322 | 3 | 7 + 871 | 3 | 7 + 1156 | 3 | 7 + 1574 | 3 | 7 + 2054 | 3 | 7 + 2309 | 3 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count DESC, remote_scan.count_1 DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode OPERATOR(pg_catalog.=) 'AIR'::bpchar)) AS count, count(DISTINCT l_suppkey) AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_orderkey ORDER BY (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode OPERATOR(pg_catalog.=) 'AIR'::bpchar))) DESC, (count(DISTINCT l_suppkey)) DESC, l_orderkey LIMIT '10'::bigint + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_suppkey) FILTER (WHERE (lineitem_hash.l_shipmode = 'AIR'::bpchar))) DESC, (count(DISTINCT lineitem_hash.l_suppkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar)), count(DISTINCT l_suppkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- group by on non-partition column +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_suppkey | count +--------------------------------------------------------------------- + 7680 | 4 + 7703 | 3 + 7542 | 3 + 7072 | 3 + 6335 | 3 + 5873 | 3 + 1318 | 3 + 1042 | 3 + 160 | 3 + 9872 | 2 +(10 rows) + +-- explaining the same query fails +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + -> Sort + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + Sort Key: (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) DESC, remote_scan.l_suppkey DESC + -> GroupAggregate + Output: remote_scan.l_suppkey, count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar)) + Group Key: remote_scan.l_suppkey + -> Sort + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.l_suppkey DESC + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Query: SELECT l_suppkey, l_partkey AS count, l_shipmode AS count FROM public.lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_suppkey, l_partkey, l_shipmode + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_suppkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_suppkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(23 rows) + +-- without group by, on partition column +SELECT + count(DISTINCT l_orderkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1327 +(1 row) + +-- without group by, on non-partition column +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1702 +(1 row) + +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_partkey), + count(DISTINCT l_shipdate) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 1702 | 11661 | 2470 +(1 row) + +-- filter column already exists in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_orderkey > 100) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +-- filter column does not exist in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- case expr in count distinct is supported. +-- count orders partkeys if l_shipmode is air +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode = 'AIR' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- text like operator is also supported +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode like '%A%' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14275 | 7 + 14181 | 7 + 13605 | 7 + 12707 | 7 + 12384 | 7 + 11746 | 7 + 10727 | 7 + 10467 | 7 + 5636 | 7 + 4614 | 7 +(10 rows) + +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT 1) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT (random() * 5)::int) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- even non-const function calls are supported within count distinct +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT (random() * 5)::int = l_linenumber) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 0; + l_orderkey | count +--------------------------------------------------------------------- +(0 rows) + +-- multiple nested subquery +SELECT + total, + avg(avg_count) as total_avg_count + FROM ( + SELECT + number_sum, + count(DISTINCT l_suppkey) as total, + avg(total_count) avg_count + FROM ( + SELECT + l_suppkey, + sum(l_linenumber) as number_sum, + count(DISTINCT l_shipmode) as total_count + FROM + lineitem_hash + WHERE + l_partkey > 100 and + l_quantity > 2 and + l_orderkey < 10000 + GROUP BY + l_suppkey) as distributed_table + WHERE + number_sum >= 10 + GROUP BY + number_sum) as distributed_table_2 + GROUP BY + total + ORDER BY + total_avg_count DESC; + total | total_avg_count +--------------------------------------------------------------------- + 1 | 3.6000000000000000 + 6 | 2.8333333333333333 + 10 | 2.6000000000000000 + 27 | 2.5555555555555556 + 32 | 2.4687500000000000 + 77 | 2.1948051948051948 + 57 | 2.1754385964912281 +(7 rows) + +-- multiple cases query +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + WHEN l_shipmode = 'AIR' THEN l_quantity + WHEN l_shipmode = 'SHIP' THEN l_discount + ELSE l_suppkey + END) as count, + l_shipdate + FROM + lineitem_hash + GROUP BY + l_shipdate) sub + WHERE + count > 0 + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + count | l_shipdate +--------------------------------------------------------------------- + 14 | 07-30-1997 + 13 | 05-26-1998 + 13 | 08-08-1997 + 13 | 11-17-1995 + 13 | 01-09-1993 + 12 | 01-15-1998 + 12 | 10-15-1997 + 12 | 09-07-1997 + 12 | 06-02-1997 + 12 | 03-14-1997 +(10 rows) + +-- count DISTINCT expression +SELECT * + FROM ( + SELECT + l_quantity, count(DISTINCT ((l_orderkey / 1000) * 1000 )) as count + FROM + lineitem_hash + GROUP BY + l_quantity) sub + WHERE + count > 0 + ORDER BY + 2 DESC, 1 DESC + LIMIT 10; + l_quantity | count +--------------------------------------------------------------------- + 48.00 | 13 + 47.00 | 13 + 37.00 | 13 + 33.00 | 13 + 26.00 | 13 + 25.00 | 13 + 23.00 | 13 + 21.00 | 13 + 15.00 | 13 + 12.00 | 13 +(10 rows) + +-- count DISTINCT is part of an expression which includes another aggregate +SELECT * + FROM ( + SELECT + sum(((l_partkey * l_tax) / 100)) / + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE l_suppkey + END) as avg, + l_shipmode + FROM + lineitem_hash + GROUP BY + l_shipmode) sub + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + avg | l_shipmode +--------------------------------------------------------------------- + 44.82904609027336300064 | MAIL + 44.80704536679536679537 | SHIP + 44.68891732736572890026 | AIR + 44.34106724470134874759 | REG AIR + 43.12739987269255251432 | FOB + 43.07299253636938646426 | RAIL + 40.50298377916903813318 | TRUCK +(7 rows) + +-- count DISTINCT CASE WHEN expression +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_linenumber + WHEN l_shipmode = 'AIR' THEN l_linenumber + 10 + ELSE 2 + END) as avg + FROM + lineitem_hash + GROUP BY l_shipdate) sub + ORDER BY 1 DESC + LIMIT 10; + avg +--------------------------------------------------------------------- + 7 + 6 + 6 + 6 + 6 + 6 + 6 + 6 + 5 + 5 +(10 rows) + +-- COUNT DISTINCT (c1, c2) +SELECT * + FROM + (SELECT + l_shipmode, + count(DISTINCT (l_shipdate, l_tax)) + FROM + lineitem_hash + GROUP BY + l_shipmode) t + ORDER BY + 2 DESC,1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1689 + MAIL | 1683 + FOB | 1655 + AIR | 1650 + SHIP | 1644 + RAIL | 1636 + REG AIR | 1607 +(7 rows) + +-- distinct on non-var (type cast/field select) columns are also +-- supported if grouped on distribution column +-- random is added to prevent flattening by postgresql +SELECT + l_orderkey, count(a::int), count(distinct a::int) + FROM ( + SELECT l_orderkey, l_orderkey * 1.5 a, random() b + FROM lineitem_hash) sub + GROUP BY 1 + ORDER BY 1 DESC + LIMIT 5; + l_orderkey | count | count +--------------------------------------------------------------------- + 14947 | 2 | 1 + 14946 | 2 | 1 + 14945 | 6 | 1 + 14944 | 2 | 1 + 14919 | 1 | 1 +(5 rows) + +SELECT user_id, + count(sub.a::int), + count(DISTINCT sub.a::int), + count(DISTINCT (sub).a) +FROM + (SELECT user_id, + unnest(ARRAY[user_id * 1.5])a, + random() b + FROM users_table + ) sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + user_id | count | count | count +--------------------------------------------------------------------- + 6 | 11 | 1 | 1 + 5 | 27 | 1 | 1 + 4 | 24 | 1 | 1 + 3 | 18 | 1 | 1 + 2 | 19 | 1 | 1 +(5 rows) + +CREATE TYPE test_item AS +( + id INTEGER, + duration INTEGER +); +CREATE TABLE test_count_distinct_array (key int, value int , value_arr test_item[]); +SELECT create_distributed_table('test_count_distinct_array', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_count_distinct_array SELECT i, i, ARRAY[(i,i)::test_item] FROM generate_Series(0, 1000) i; +SELECT + key, + count(DISTINCT value), + count(DISTINCT (item)."id"), + count(DISTINCT (item)."id" * 3) +FROM + ( + SELECT key, unnest(value_arr) as item, value FROM test_count_distinct_array + ) as sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + key | count | count | count +--------------------------------------------------------------------- + 1000 | 1 | 1 | 1 + 999 | 1 | 1 | 1 + 998 | 1 | 1 | 1 + 997 | 1 | 1 | 1 + 996 | 1 | 1 | 1 +(5 rows) + +DROP TABLE test_count_distinct_array; +DROP TYPE test_item; +-- other distinct aggregate are not supported +SELECT * + FROM ( + SELECT + l_linenumber, sum(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: table partitioning is unsuitable for aggregate (distinct) +SELECT * + FROM ( + SELECT + l_linenumber, avg(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: table partitioning is unsuitable for aggregate (distinct) +-- whole row references, oid, and ctid are not supported in count distinct +-- test table does not have oid or ctid enabled, so tests for them are skipped +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash.*) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +DROP TABLE lineitem_hash; diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index b3e47474f..17b673607 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -1,6 +1,18 @@ -- -- MULTI_EXPLAIN -- +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + t +(1 row) + SET citus.next_shard_id TO 570000; \a\t SET citus.explain_distributed_queries TO on; @@ -651,7 +663,7 @@ Aggregate -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), events.event_time -> Hash Join Hash Cond: (users.composite_id = events.composite_id) -> Seq Scan on users_1400289 users @@ -737,7 +749,7 @@ HashAggregate -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone, events.event_time -> Hash Left Join Hash Cond: (users.composite_id = subquery_2.composite_id) -> HashAggregate @@ -853,7 +865,7 @@ Sort Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay, events.event_time -> Hash Left Join Hash Cond: (users.composite_id = subquery_2.composite_id) -> HashAggregate @@ -951,7 +963,7 @@ Limit -> GroupAggregate Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), events.event_time -> Nested Loop Left Join -> Limit -> Sort @@ -2381,11 +2393,16 @@ Custom Scan (Citus Adaptive) (actual rows=1 loops=1) Tuple data received from node: 8 bytes Node: host=localhost port=xxxxx dbname=regression -> Aggregate (actual rows=1 loops=1) - -> Hash Join (actual rows=10 loops=1) - Hash Cond: (ref_table.a = intermediate_result.a) - -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) - -> Hash (actual rows=10 loops=1) + -> Merge Join (actual rows=10 loops=1) + Merge Cond: (intermediate_result.a = ref_table.a) + -> Sort (actual rows=10 loops=1) + Sort Key: intermediate_result.a + Sort Method: quicksort Memory: 25kB -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) + -> Sort (actual rows=10 loops=1) + Sort Key: ref_table.a + Sort Method: quicksort Memory: 25kB + -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) EXPLAIN :default_analyze_flags SELECT count(distinct a) FROM (SELECT GREATEST(random(), 2) r, a FROM dist_table) t NATURAL JOIN ref_table; Aggregate (actual rows=1 loops=1) @@ -2442,9 +2459,12 @@ Aggregate (actual rows=1 loops=1) -> Aggregate (actual rows=1 loops=1) InitPlan 1 (returns $0) -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) - -> Result (actual rows=4 loops=1) - One-Time Filter: $0 - -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) + -> Sort (actual rows=4 loops=1) + Sort Key: dist_table.a + Sort Method: quicksort Memory: 25kB + -> Result (actual rows=4 loops=1) + One-Time Filter: $0 + -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) BEGIN; EXPLAIN :default_analyze_flags WITH r AS ( @@ -2486,7 +2506,10 @@ Custom Scan (Citus Adaptive) (actual rows=1 loops=1) Tuple data received from node: 8 bytes Node: host=localhost port=xxxxx dbname=regression -> Aggregate (actual rows=1 loops=1) - -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) + -> Sort (actual rows=10 loops=1) + Sort Key: intermediate_result.a2 + Sort Method: quicksort Memory: 25kB + -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) ROLLBACK; -- https://github.com/citusdata/citus/issues/4074 prepare ref_select(int) AS select * from ref_table where 1 = $1; diff --git a/src/test/regress/expected/multi_explain_0.out b/src/test/regress/expected/multi_explain_0.out new file mode 100644 index 000000000..9534cefb8 --- /dev/null +++ b/src/test/regress/expected/multi_explain_0.out @@ -0,0 +1,3219 @@ +-- +-- MULTI_EXPLAIN +-- +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; + server_version_ge_16 +--------------------------------------------------------------------- + f +(1 row) + +SET citus.next_shard_id TO 570000; +\a\t +SET citus.explain_distributed_queries TO on; +SET citus.enable_repartition_joins to ON; +-- Ensure tuple data in explain analyze output is the same on all PG versions +SET citus.enable_binary_protocol = TRUE; +-- Function that parses explain output as JSON +CREATE OR REPLACE FUNCTION explain_json(query text) +RETURNS jsonb +AS $BODY$ +DECLARE + result jsonb; +BEGIN + EXECUTE format('EXPLAIN (FORMAT JSON) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION explain_analyze_json(query text) +RETURNS jsonb +AS $BODY$ +DECLARE + result jsonb; +BEGIN + EXECUTE format('EXPLAIN (ANALYZE TRUE, FORMAT JSON) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +-- Function that parses explain output as XML +CREATE OR REPLACE FUNCTION explain_xml(query text) +RETURNS xml +AS $BODY$ +DECLARE + result xml; +BEGIN + EXECUTE format('EXPLAIN (FORMAT XML) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +-- Function that parses explain output as XML +CREATE OR REPLACE FUNCTION explain_analyze_xml(query text) +RETURNS xml +AS $BODY$ +DECLARE + result xml; +BEGIN + EXECUTE format('EXPLAIN (ANALYZE true, FORMAT XML) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +-- VACUMM related tables to ensure test outputs are stable +VACUUM ANALYZE lineitem; +VACUUM ANALYZE orders; +-- Test Text format +EXPLAIN (COSTS FALSE, FORMAT TEXT) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + -> HashAggregate + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_quantity + -> Seq Scan on lineitem_360000 lineitem +-- Test disable hash aggregate +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE, FORMAT TEXT) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + -> GroupAggregate + Group Key: remote_scan.l_quantity + -> Sort + Sort Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_quantity + -> Seq Scan on lineitem_360000 lineitem +SET enable_hashagg TO on; +-- Test JSON format +EXPLAIN (COSTS FALSE, FORMAT JSON) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +[ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Sort Key": ["(COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))", "remote_scan.l_quantity"], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Group Key": ["remote_scan.l_quantity"], + "Plans": [ + { + "Node Type": "Custom Scan", + "Parent Relationship": "Outer", + "Custom Plan Provider": "Citus Adaptive", + "Parallel Aware": false, + "Async Capable": false, + "Distributed Query": { + "Job": { + "Task Count": 2, + "Tasks Shown": "One of 2", + "Tasks": [ + { + "Node": "host=localhost port=xxxxx dbname=regression", + "Remote Plan": [ + [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parallel Aware": false, + "Async Capable": false, + "Group Key": ["l_quantity"], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "lineitem_360000", + "Alias": "lineitem" + } + ] + } + } + ] + + ] + } + ] + } + } + } + ] + } + ] + } + } +] +-- Validate JSON format +SELECT true AS valid FROM explain_json($$ + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity$$); +t +SELECT true AS valid FROM explain_analyze_json($$ + WITH a AS ( + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity LIMIT 10) + SELECT count(*) FROM a +$$); +t +-- Test XML format +EXPLAIN (COSTS FALSE, FORMAT XML) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; + + + + Sort + false + false + + (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)) + remote_scan.l_quantity + + + + Aggregate + Hashed + Simple + Outer + false + false + + remote_scan.l_quantity + + + + Custom Scan + Outer + Citus Adaptive + false + false + + + 2 + One of 2 + + + host=localhost port=xxxxx dbname=regression + + + + + Aggregate + Hashed + Simple + false + false + + l_quantity + + + + Seq Scan + Outer + false + false + lineitem_360000 + lineitem + + + + + + + + + + + + + + + + + +-- Validate XML format +SELECT true AS valid FROM explain_xml($$ + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity$$); +t +SELECT true AS valid FROM explain_analyze_xml($$ + WITH a AS ( + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity LIMIT 10) + SELECT count(*) FROM a +$$); +t +-- Test YAML format +EXPLAIN (COSTS FALSE, FORMAT YAML) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +- Plan: + Node Type: "Sort" + Parallel Aware: false + Async Capable: false + Sort Key: + - "(COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))" + - "remote_scan.l_quantity" + Plans: + - Node Type: "Aggregate" + Strategy: "Hashed" + Partial Mode: "Simple" + Parent Relationship: "Outer" + Parallel Aware: false + Async Capable: false + Group Key: + - "remote_scan.l_quantity" + Plans: + - Node Type: "Custom Scan" + Parent Relationship: "Outer" + Custom Plan Provider: "Citus Adaptive" + Parallel Aware: false + Async Capable: false + Distributed Query: + Job: + Task Count: 2 + Tasks Shown: "One of 2" + Tasks: + - Node: "host=localhost port=xxxxx dbname=regression" + Remote Plan: + - Plan: + Node Type: "Aggregate" + Strategy: "Hashed" + Partial Mode: "Simple" + Parallel Aware: false + Async Capable: false + Group Key: + - "l_quantity" + Plans: + - Node Type: "Seq Scan" + Parent Relationship: "Outer" + Parallel Aware: false + Async Capable: false + Relation Name: "lineitem_360000" + Alias: "lineitem" + +-- Test Text format +EXPLAIN (COSTS FALSE, FORMAT TEXT) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + -> HashAggregate + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_quantity + -> Seq Scan on lineitem_360000 lineitem +-- Test analyze (with TIMING FALSE and SUMMARY FALSE for consistent output) +SELECT public.plan_normalize_memory($Q$ +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +$Q$); +Sort (actual rows=50 loops=1) + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + Sort Method: quicksort Memory: xxx + -> HashAggregate (actual rows=50 loops=1) + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) (actual rows=100 loops=1) + Task Count: 2 + Tuple data received from nodes: 1800 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 900 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=50 loops=1) + Group Key: l_quantity + -> Seq Scan on lineitem_360000 lineitem (actual rows=5894 loops=1) +-- EXPLAIN ANALYZE doesn't show worker tasks for repartition joins yet +SET citus.shard_count TO 3; +CREATE TABLE t1(a int, b int); +CREATE TABLE t2(a int, b int); +SELECT create_distributed_table('t1', 'a'), create_distributed_table('t2', 'a'); +| +BEGIN; +SET LOCAL citus.enable_repartition_joins TO true; +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off) SELECT count(*) FROM t1, t2 WHERE t1.a=t2.b; +Aggregate (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 6 + Tuple data received from nodes: 48 bytes + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 3 + Merge Task Count: 6 + -> MapMergeJob + Map Task Count: 3 + Merge Task Count: 6 +-- Confirm repartiton join in distributed subplan works +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off) +WITH repartition AS (SELECT count(*) FROM t1, t2 WHERE t1.a=t2.b) +SELECT count(*) from repartition; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 14 bytes + Result destination: Write locally + -> Aggregate (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 6 + Tuple data received from nodes: 48 bytes + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 3 + Merge Task Count: 6 + -> MapMergeJob + Map Task Count: 3 + Merge Task Count: 6 + Task Count: 1 + Tuple data received from nodes: 8 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) +END; +DROP TABLE t1, t2; +-- Test query text output, with ANALYZE ON +SELECT public.plan_normalize_memory($Q$ +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE, VERBOSE TRUE) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +$Q$); +Sort (actual rows=50 loops=1) + Output: remote_scan.l_quantity, (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)) + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + Sort Method: quicksort Memory: xxx + -> HashAggregate (actual rows=50 loops=1) + Output: remote_scan.l_quantity, COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint) + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) (actual rows=100 loops=1) + Output: remote_scan.l_quantity, remote_scan.count_quantity + Task Count: 2 + Tuple data received from nodes: 1800 bytes + Tasks Shown: One of 2 + -> Task + Query: SELECT l_quantity, count(*) AS count_quantity FROM public.lineitem_360000 lineitem WHERE true GROUP BY l_quantity + Tuple data received from node: 900 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=50 loops=1) + Output: l_quantity, count(*) + Group Key: lineitem.l_quantity + -> Seq Scan on public.lineitem_360000 lineitem (actual rows=5894 loops=1) + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test query text output, with ANALYZE OFF +EXPLAIN (COSTS FALSE, ANALYZE FALSE, TIMING FALSE, SUMMARY FALSE, VERBOSE TRUE) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Output: remote_scan.l_quantity, (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)) + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + -> HashAggregate + Output: remote_scan.l_quantity, COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint) + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_quantity, remote_scan.count_quantity + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Query: SELECT l_quantity, count(*) AS count_quantity FROM public.lineitem_360000 lineitem WHERE true GROUP BY l_quantity + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_quantity, count(*) + Group Key: lineitem.l_quantity + -> Seq Scan on public.lineitem_360000 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test verbose +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem; +Aggregate + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) + -> Custom Scan (Citus Adaptive) + Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Query: SELECT sum(l_quantity), sum(l_quantity), count(l_quantity) FROM public.lineitem_360000 lineitem WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: sum(l_quantity), sum(l_quantity), count(l_quantity) + -> Seq Scan on public.lineitem_360000 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test join +EXPLAIN (COSTS FALSE) + SELECT * FROM lineitem + JOIN orders ON l_orderkey = o_orderkey AND l_quantity < 5.0 + ORDER BY l_quantity LIMIT 10; +Limit + -> Sort + Sort Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: lineitem.l_quantity + -> Hash Join + Hash Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Seq Scan on lineitem_360000 lineitem + Filter: (l_quantity < 5.0) + -> Hash + -> Seq Scan on orders_360002 orders +-- Test insert +EXPLAIN (COSTS FALSE) + INSERT INTO lineitem VALUES (1,0), (2, 0), (3, 0), (4, 0); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on lineitem_360000 citus_table_alias + -> Values Scan on "*VALUES*" +-- Test update +EXPLAIN (COSTS FALSE) + UPDATE lineitem + SET l_suppkey = 12 + WHERE l_orderkey = 1 AND l_partkey = 0; +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_360000 lineitem + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey = 1) + Filter: (l_partkey = 0) +-- Test analyze (with TIMING FALSE and SUMMARY FALSE for consistent output) +BEGIN; +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) + UPDATE lineitem + SET l_suppkey = 12 + WHERE l_orderkey = 1 AND l_partkey = 0; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_360000 lineitem (actual rows=0 loops=1) + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem (actual rows=0 loops=1) + Index Cond: (l_orderkey = 1) + Filter: (l_partkey = 0) + Rows Removed by Filter: 6 +ROLLBACk; +-- Test delete +EXPLAIN (COSTS FALSE) + DELETE FROM lineitem + WHERE l_orderkey = 1 AND l_partkey = 0; +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_360000 lineitem + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey = 1) + Filter: (l_partkey = 0) +-- Test zero-shard update +EXPLAIN (COSTS FALSE) + UPDATE lineitem + SET l_suppkey = 12 + WHERE l_orderkey = 1 AND l_orderkey = 0; +Custom Scan (Citus Adaptive) + Task Count: 0 + Tasks Shown: All +-- Test zero-shard delete +EXPLAIN (COSTS FALSE) + DELETE FROM lineitem + WHERE l_orderkey = 1 AND l_orderkey = 0; +Custom Scan (Citus Adaptive) + Task Count: 0 + Tasks Shown: All +-- Test single-shard SELECT +EXPLAIN (COSTS FALSE) + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey = 5) +SELECT true AS valid FROM explain_xml($$ + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); +t +SELECT true AS valid FROM explain_json($$ + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); +t +-- Test CREATE TABLE ... AS +EXPLAIN (COSTS FALSE) + CREATE TABLE explain_result AS + SELECT * FROM lineitem; +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on lineitem_360000 lineitem +-- Test having +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem + HAVING sum(l_quantity) > 100; +Aggregate + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) + Filter: (sum(remote_scan.worker_column_4) > '100'::numeric) + -> Custom Scan (Citus Adaptive) + Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2", remote_scan.worker_column_4 + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Query: SELECT sum(l_quantity), sum(l_quantity), count(l_quantity), sum(l_quantity) AS worker_column_4 FROM public.lineitem_360000 lineitem WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + Output: sum(l_quantity), sum(l_quantity), count(l_quantity), sum(l_quantity) + -> Seq Scan on public.lineitem_360000 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test having without aggregate +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT l_quantity FROM lineitem + GROUP BY l_quantity + HAVING l_quantity > (100 * random()); +HashAggregate + Output: remote_scan.l_quantity + Group Key: remote_scan.l_quantity + Filter: ((remote_scan.worker_column_2)::double precision > ('100'::double precision * random())) + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_quantity, remote_scan.worker_column_2 + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Query: SELECT l_quantity, l_quantity AS worker_column_2 FROM public.lineitem_360000 lineitem WHERE true GROUP BY l_quantity + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_quantity, l_quantity + Group Key: lineitem.l_quantity + -> Seq Scan on public.lineitem_360000 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Subquery pushdown tests with explain +EXPLAIN (COSTS OFF) +SELECT + avg(array_length(events, 1)) AS event_average +FROM + (SELECT + tenant_id, + user_id, + array_agg(event_type ORDER BY event_time) AS events + FROM + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + event_type, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type IN ('click', 'submit', 'pay')) AS subquery + GROUP BY + tenant_id, + user_id) AS subquery; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) +-- Union and left join subquery pushdown +EXPLAIN (COSTS OFF) +SELECT + avg(array_length(events, 1)) AS event_average, + hasdone +FROM + (SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(hasdone, 'Has not done paying') AS hasdone + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id) as composite_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id) as composite_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + DISTINCT ON ((composite_id).tenant_id, (composite_id).user_id) composite_id, + (composite_id).tenant_id, + (composite_id).user_id, + 'Has done paying'::TEXT AS hasdone + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay') AS subquery_2 + ON + subquery_1.composite_id = subquery_2.composite_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + hasdone) AS subquery_top +GROUP BY + hasdone; +HashAggregate + Group Key: remote_scan.hasdone + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: subquery_top.hasdone + -> Sort + Sort Key: subquery_top.hasdone + -> Subquery Scan on subquery_top + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.hasdone + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 + -> Unique + -> Sort + Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) +-- Union, left join and having subquery pushdown +EXPLAIN (COSTS OFF) + SELECT + avg(array_length(events, 1)) AS event_average, + count_pay + FROM ( + SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(count_pay, 0) AS count_pay + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id), + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id), + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + composite_id, + COUNT(*) AS count_pay + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay' + GROUP BY + composite_id + HAVING + COUNT(*) > 2) AS subquery_2 + ON + subquery_1.composite_id = subquery_2.composite_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + count_pay) AS subquery_top +WHERE + array_ndims(events) > 0 +GROUP BY + count_pay +ORDER BY + count_pay; +Sort + Sort Key: remote_scan.count_pay + -> HashAggregate + Group Key: remote_scan.count_pay + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: subquery_top.count_pay + -> Sort + Sort Key: subquery_top.count_pay + -> Subquery Scan on subquery_top + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + Filter: (array_ndims(array_agg(('action=>1'::text) ORDER BY events.event_time)) > 0) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), subquery_2.count_pay + -> Hash Left Join + Hash Cond: (users.composite_id = subquery_2.composite_id) + -> HashAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id), users.composite_id, ('action=>1'::text), events.event_time + -> Append + -> Hash Join + Hash Cond: (users.composite_id = events.composite_id) + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events + Filter: ((event_type)::text = 'click'::text) + -> Hash Join + Hash Cond: (users_1.composite_id = events_1.composite_id) + -> Seq Scan on users_1400289 users_1 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Hash + -> Seq Scan on events_1400285 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 + -> GroupAggregate + Group Key: events_2.composite_id + Filter: (count(*) > 2) + -> Sort + Sort Key: events_2.composite_id + -> Seq Scan on events_1400285 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) +-- Lateral join subquery pushdown +-- set subquery_pushdown due to limit in the query +SET citus.subquery_pushdown to ON; +NOTICE: Setting citus.subquery_pushdown flag is discouraged becuase it forces the planner to pushdown certain queries, skipping relevant correctness checks. +DETAIL: When enabled, the planner skips many correctness checks for subqueries and pushes down the queries to shards as-is. It means that the queries are likely to return wrong results unless the user is absolutely sure that pushing down the subquery is safe. This GUC is maintained only for backward compatibility, no new users are supposed to use it. The planner is capable of pushing down as much computation as possible to the shards depending on the query. +EXPLAIN (COSTS OFF) +SELECT + tenant_id, + user_id, + user_lastseen, + event_array +FROM + (SELECT + tenant_id, + user_id, + max(lastseen) as user_lastseen, + array_agg(event_type ORDER BY event_time) AS event_array + FROM + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + composite_id, + lastseen + FROM + users + WHERE + composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + composite_id <= '(1, 9223372036854775807)'::user_composite_type + ORDER BY + lastseen DESC + LIMIT + 10 + ) AS subquery_top + LEFT JOIN LATERAL + (SELECT + event_type, + event_time + FROM + events + WHERE + (composite_id) = subquery_top.composite_id + ORDER BY + event_time DESC + LIMIT + 99) AS subquery_lateral + ON + true + GROUP BY + tenant_id, + user_id + ) AS shard_union +ORDER BY + user_lastseen DESC +LIMIT + 10; +Limit + -> Sort + Sort Key: remote_scan.user_lastseen DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: (max(users.lastseen)) DESC + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Nested Loop Left Join + -> Limit + -> Sort + Sort Key: users.lastseen DESC + -> Seq Scan on users_1400289 users + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type)) + -> Limit + -> Sort + Sort Key: events.event_time DESC + -> Seq Scan on events_1400285 events + Filter: (composite_id = users.composite_id) +RESET citus.subquery_pushdown; +-- Test all tasks output +SET citus.explain_all_tasks TO on; +EXPLAIN (COSTS FALSE) + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Index Only Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey > 9030) + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Index Only Scan using lineitem_pkey_360001 on lineitem_360001 lineitem + Index Cond: (l_orderkey > 9030) +SELECT true AS valid FROM explain_xml($$ + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030$$); +t +SELECT true AS valid FROM explain_json($$ + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030$$); +t +-- Test multi shard update +EXPLAIN (COSTS FALSE) + UPDATE lineitem_hash_part + SET l_suppkey = 12; +Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360041 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360042 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360042 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360043 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360043 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360044 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360044 lineitem_hash_part +EXPLAIN (COSTS FALSE) + UPDATE lineitem_hash_part + SET l_suppkey = 12 + WHERE l_orderkey = 1 OR l_orderkey = 3; +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360041 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: ((l_orderkey = 1) OR (l_orderkey = 3)) + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360042 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360042 lineitem_hash_part + Filter: ((l_orderkey = 1) OR (l_orderkey = 3)) +-- Test multi shard delete +EXPLAIN (COSTS FALSE) + DELETE FROM lineitem_hash_part; +Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360041 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360042 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360042 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360043 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360043 lineitem_hash_part + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360044 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360044 lineitem_hash_part +-- Test analyze (with TIMING FALSE and SUMMARY FALSE for consistent output) +SELECT public.plan_normalize_memory($Q$ +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +$Q$); +Sort (actual rows=50 loops=1) + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity + Sort Method: quicksort Memory: xxx + -> HashAggregate (actual rows=50 loops=1) + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Adaptive) (actual rows=100 loops=1) + Task Count: 2 + Tuple data received from nodes: 1800 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 900 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=50 loops=1) + Group Key: l_quantity + -> Seq Scan on lineitem_360000 lineitem (actual rows=5894 loops=1) + -> Task + Tuple data received from node: 900 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=50 loops=1) + Group Key: l_quantity + -> Seq Scan on lineitem_360001 lineitem (actual rows=6106 loops=1) +SET citus.explain_all_tasks TO off; +-- Test update with subquery +EXPLAIN (COSTS FALSE) + UPDATE lineitem_hash_part + SET l_suppkey = 12 + FROM orders_hash_part + WHERE orders_hash_part.o_orderkey = lineitem_hash_part.l_orderkey; +Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_hash_part_360041 lineitem_hash_part + -> Hash Join + Hash Cond: (lineitem_hash_part.l_orderkey = orders_hash_part.o_orderkey) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + -> Hash + -> Seq Scan on orders_hash_part_360045 orders_hash_part +-- Test delete with subquery +EXPLAIN (COSTS FALSE) + DELETE FROM lineitem_hash_part + USING orders_hash_part + WHERE orders_hash_part.o_orderkey = lineitem_hash_part.l_orderkey; +Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360041 lineitem_hash_part + -> Hash Join + Hash Cond: (lineitem_hash_part.l_orderkey = orders_hash_part.o_orderkey) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + -> Hash + -> Seq Scan on orders_hash_part_360045 orders_hash_part +-- Test track tracker +EXPLAIN (COSTS FALSE) + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Index Only Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey > 9030) +-- Test re-partition join +EXPLAIN (COSTS FALSE) + SELECT count(*) + FROM lineitem, orders, customer_append, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 6 + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 6 + Merge Task Count: 6 + -> MapMergeJob + Map Task Count: 2 + Merge Task Count: 6 + -> MapMergeJob + Map Task Count: 1 + Merge Task Count: 6 + -> MapMergeJob + Map Task Count: 1 + Merge Task Count: 6 +EXPLAIN (COSTS FALSE, FORMAT JSON) + SELECT count(*) + FROM lineitem, orders, customer_append, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +[ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Simple", + "Parallel Aware": false, + "Async Capable": false, + "Plans": [ + { + "Node Type": "Custom Scan", + "Parent Relationship": "Outer", + "Custom Plan Provider": "Citus Adaptive", + "Parallel Aware": false, + "Async Capable": false, + "Distributed Query": { + "Job": { + "Task Count": 6, + "Tasks Shown": "None, not supported for re-partition queries", + "Dependent Jobs": [ + { + "Map Task Count": 6, + "Merge Task Count": 6, + "Dependent Jobs": [ + { + "Map Task Count": 2, + "Merge Task Count": 6 + }, + { + "Map Task Count": 1, + "Merge Task Count": 6 + } + ] + }, + { + "Map Task Count": 1, + "Merge Task Count": 6 + } + ] + } + } + } + ] + } + } +] +SELECT true AS valid FROM explain_json($$ + SELECT count(*) + FROM lineitem, orders, customer_append, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey$$); +t +EXPLAIN (COSTS FALSE, FORMAT XML) + SELECT count(*) + FROM lineitem, orders, customer_append, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; + + + + Aggregate + Plain + Simple + false + false + + + Custom Scan + Outer + Citus Adaptive + false + false + + + 6 + None, not supported for re-partition queries + + + 6 + 6 + + + 2 + 6 + + + 1 + 6 + + + + + 1 + 6 + + + + + + + + + +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM lineitem, orders, customer_append, supplier + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey$$); +t +-- make sure that EXPLAIN works without +-- problems for queries that inlvolves only +-- reference tables +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM nation + WHERE n_name = 'CHINA'$$); +t +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM nation, supplier + WHERE nation.n_nationkey = supplier.s_nationkey$$); +t +EXPLAIN (COSTS FALSE, FORMAT YAML) + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +- Plan: + Node Type: "Aggregate" + Strategy: "Plain" + Partial Mode: "Simple" + Parallel Aware: false + Async Capable: false + Plans: + - Node Type: "Custom Scan" + Parent Relationship: "Outer" + Custom Plan Provider: "Citus Adaptive" + Parallel Aware: false + Async Capable: false + Distributed Query: + Job: + Task Count: 6 + Tasks Shown: "None, not supported for re-partition queries" + Dependent Jobs: + - Map Task Count: 2 + Merge Task Count: 6 + - Map Task Count: 1 + Merge Task Count: 6 +-- ensure local plans display correctly +CREATE TABLE lineitem_clone (LIKE lineitem); +EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem_clone; +Aggregate + -> Seq Scan on lineitem_clone +DROP TABLE lineitem_clone; +-- ensure distributed plans don't break +EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on lineitem_360000 lineitem +-- ensure EXPLAIN EXECUTE doesn't crash +PREPARE task_tracker_query AS + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +EXPLAIN (COSTS FALSE) EXECUTE task_tracker_query; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Index Only Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey > 9030) +PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; +EXPLAIN EXECUTE router_executor_query; +Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem (cost=0.28..13.60 rows=4 width=5) + Index Cond: (l_orderkey = 5) +PREPARE real_time_executor_query AS + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query; +Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Index Only Scan using lineitem_pkey_360000 on lineitem_360000 lineitem + Index Cond: (l_orderkey > 9030) +-- EXPLAIN EXECUTE of parametrized prepared statements is broken, but +-- at least make sure to fail without crashing +PREPARE router_executor_query_param(int) AS SELECT l_quantity FROM lineitem WHERE l_orderkey = $1; +EXPLAIN EXECUTE router_executor_query_param(5); +Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem (cost=0.28..13.60 rows=4 width=5) + Index Cond: (l_orderkey = 5) +EXPLAIN (ANALYZE ON, COSTS OFF, TIMING OFF, SUMMARY OFF) EXECUTE router_executor_query_param(5); +Custom Scan (Citus Adaptive) (actual rows=3 loops=1) + Task Count: 1 + Tuple data received from nodes: 30 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 30 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Index Scan using lineitem_pkey_360000 on lineitem_360000 lineitem (actual rows=3 loops=1) + Index Cond: (l_orderkey = 5) +\set VERBOSITY TERSE +PREPARE multi_shard_query_param(int) AS UPDATE lineitem SET l_quantity = $1; +BEGIN; +EXPLAIN (COSTS OFF) EXECUTE multi_shard_query_param(5); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_360000 lineitem + -> Seq Scan on lineitem_360000 lineitem +ROLLBACK; +BEGIN; +EXPLAIN (ANALYZE ON, COSTS OFF, TIMING OFF, SUMMARY OFF) EXECUTE multi_shard_query_param(5); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on lineitem_360000 lineitem (actual rows=0 loops=1) + -> Seq Scan on lineitem_360000 lineitem (actual rows=5894 loops=1) +ROLLBACK; +\set VERBOSITY DEFAULT +-- test explain in a transaction with alter table to test we use right connections +BEGIN; +CREATE TABLE explain_table(id int); +SELECT create_distributed_table('explain_table', 'id'); + +ALTER TABLE explain_table ADD COLUMN value int; +ROLLBACK; +-- test explain with local INSERT ... SELECT +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part +SELECT o_orderkey FROM orders_hash_part LIMIT 3; +Custom Scan (Citus INSERT ... SELECT) + INSERT/SELECT method: pull to coordinator + -> Limit + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Seq Scan on orders_hash_part_360045 orders_hash_part +SELECT true AS valid FROM explain_json($$ + INSERT INTO lineitem_hash_part (l_orderkey) + SELECT o_orderkey FROM orders_hash_part LIMIT 3; +$$); +t +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part (l_orderkey, l_quantity) +SELECT o_orderkey, 5 FROM orders_hash_part LIMIT 3; +Custom Scan (Citus INSERT ... SELECT) + INSERT/SELECT method: pull to coordinator + -> Limit + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Seq Scan on orders_hash_part_360045 orders_hash_part +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part (l_orderkey) +SELECT s FROM generate_series(1,5) s; +Custom Scan (Citus INSERT ... SELECT) + INSERT/SELECT method: pull to coordinator + -> Function Scan on generate_series s +-- WHERE EXISTS forces pg12 to materialize cte +EXPLAIN (COSTS OFF) +WITH cte1 AS (SELECT s FROM generate_series(1,10) s) +INSERT INTO lineitem_hash_part +WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) +SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); +Custom Scan (Citus INSERT ... SELECT) + INSERT/SELECT method: pull to coordinator + -> Result + One-Time Filter: $3 + CTE cte1 + -> Function Scan on generate_series s + CTE cte1 + -> Limit + InitPlan 2 (returns $1) + -> CTE Scan on cte1 cte1_1 + -> Result + One-Time Filter: $1 + -> CTE Scan on cte1 cte1_2 + InitPlan 4 (returns $3) + -> CTE Scan on cte1 cte1_3 + -> CTE Scan on cte1 +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part +( SELECT s FROM generate_series(1,5) s) UNION +( SELECT s FROM generate_series(5,10) s); +Custom Scan (Citus INSERT ... SELECT) + INSERT/SELECT method: pull to coordinator + -> Subquery Scan on citus_insert_select_subquery + -> HashAggregate + Group Key: s.s + -> Append + -> Function Scan on generate_series s + -> Function Scan on generate_series s_1 +-- explain with recursive planning +EXPLAIN (COSTS OFF, VERBOSE true) +WITH keys AS MATERIALIZED ( + SELECT DISTINCT l_orderkey FROM lineitem_hash_part +), +series AS MATERIALIZED ( + SELECT s FROM generate_series(1,10) s +) +SELECT l_orderkey FROM series JOIN keys ON (s = l_orderkey) +ORDER BY s; +Custom Scan (Citus Adaptive) + Output: remote_scan.l_orderkey + -> Distributed Subplan XXX_1 + -> HashAggregate + Output: remote_scan.l_orderkey + Group Key: remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Output: remote_scan.l_orderkey + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Query: SELECT DISTINCT l_orderkey FROM public.lineitem_hash_part_360041 lineitem_hash_part WHERE true + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_orderkey + Group Key: lineitem_hash_part.l_orderkey + -> Seq Scan on public.lineitem_hash_part_360041 lineitem_hash_part + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Distributed Subplan XXX_2 + -> Function Scan on pg_catalog.generate_series s + Output: s + Function Call: generate_series(1, 10) + Task Count: 1 + Tasks Shown: All + -> Task + Query: SELECT keys.l_orderkey FROM ((SELECT intermediate_result.s FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(s integer)) series JOIN (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) keys ON ((series.s OPERATOR(pg_catalog.=) keys.l_orderkey))) ORDER BY series.s + Node: host=localhost port=xxxxx dbname=regression + -> Merge Join + Output: intermediate_result_1.l_orderkey, intermediate_result.s + Merge Cond: (intermediate_result.s = intermediate_result_1.l_orderkey) + -> Sort + Output: intermediate_result.s + Sort Key: intermediate_result.s + -> Function Scan on pg_catalog.read_intermediate_result intermediate_result + Output: intermediate_result.s + Function Call: read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) + -> Sort + Output: intermediate_result_1.l_orderkey + Sort Key: intermediate_result_1.l_orderkey + -> Function Scan on pg_catalog.read_intermediate_result intermediate_result_1 + Output: intermediate_result_1.l_orderkey + Function Call: read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) +SELECT true AS valid FROM explain_json($$ + WITH result AS ( + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity + ), + series AS ( + SELECT s FROM generate_series(1,10) s + ) + SELECT * FROM result JOIN series ON (s = count_quantity) JOIN orders_hash_part ON (s = o_orderkey) +$$); +t +SELECT true AS valid FROM explain_xml($$ + WITH result AS ( + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity + ), + series AS ( + SELECT s FROM generate_series(1,10) s + ) + SELECT * FROM result JOIN series ON (s = l_quantity) JOIN orders_hash_part ON (s = o_orderkey) +$$); +t +-- +-- Test EXPLAIN ANALYZE udfs +-- +\a\t +\set default_opts '''{"costs": false, "timing": false, "summary": false}'''::jsonb +CREATE TABLE explain_analyze_test(a int, b text); +INSERT INTO explain_analyze_test VALUES (1, 'value 1'), (2, 'value 2'), (3, 'value 3'), (4, 'value 4'); +-- simple select +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', :default_opts) as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Result (actual rows=1 loops=1)+ + +(1 row) + +END; +-- insert into select +BEGIN; +SELECT * FROM worker_save_query_explain_analyze($Q$ + INSERT INTO explain_analyze_test SELECT i, i::text FROM generate_series(1, 5) i $Q$, + :default_opts) as (a int); + a +--------------------------------------------------------------------- +(0 rows) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Insert on explain_analyze_test (actual rows=0 loops=1) + + -> Function Scan on generate_series i (actual rows=5 loops=1)+ + +(1 row) + +ROLLBACK; +-- select from table +BEGIN; +SELECT * FROM worker_save_query_explain_analyze($Q$SELECT * FROM explain_analyze_test$Q$, + :default_opts) as (a int, b text); + a | b +--------------------------------------------------------------------- + 1 | value 1 + 2 | value 2 + 3 | value 3 + 4 | value 4 +(4 rows) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Seq Scan on explain_analyze_test (actual rows=4 loops=1)+ + +(1 row) + +ROLLBACK; +-- insert into with returning +BEGIN; +SELECT * FROM worker_save_query_explain_analyze($Q$ + INSERT INTO explain_analyze_test SELECT i, i::text FROM generate_series(1, 5) i + RETURNING a, b$Q$, + :default_opts) as (a int, b text); + a | b +--------------------------------------------------------------------- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 +(5 rows) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Insert on explain_analyze_test (actual rows=5 loops=1) + + -> Function Scan on generate_series i (actual rows=5 loops=1)+ + +(1 row) + +ROLLBACK; +-- delete with returning +BEGIN; +SELECT * FROM worker_save_query_explain_analyze($Q$ + DELETE FROM explain_analyze_test WHERE a % 2 = 0 + RETURNING a, b$Q$, + :default_opts) as (a int, b text); + a | b +--------------------------------------------------------------------- + 2 | value 2 + 4 | value 4 +(2 rows) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Delete on explain_analyze_test (actual rows=2 loops=1) + + -> Seq Scan on explain_analyze_test (actual rows=2 loops=1)+ + Filter: ((a % 2) = 0) + + Rows Removed by Filter: 2 + + +(1 row) + +ROLLBACK; +-- delete without returning +BEGIN; +SELECT * FROM worker_save_query_explain_analyze($Q$ + DELETE FROM explain_analyze_test WHERE a % 2 = 0$Q$, + :default_opts) as (a int); + a +--------------------------------------------------------------------- +(0 rows) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Delete on explain_analyze_test (actual rows=0 loops=1) + + -> Seq Scan on explain_analyze_test (actual rows=2 loops=1)+ + Filter: ((a % 2) = 0) + + Rows Removed by Filter: 2 + + +(1 row) + +ROLLBACK; +-- multiple queries (should ERROR) +SELECT * FROM worker_save_query_explain_analyze('SELECT 1; SELECT 2', :default_opts) as (a int); +ERROR: cannot EXPLAIN ANALYZE multiple queries +-- error in query +SELECT * FROM worker_save_query_explain_analyze('SELECT x', :default_opts) as (a int); +ERROR: column "x" does not exist +-- error in format string +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"format": "invlaid_format"}') as (a int); +ERROR: Invalid explain analyze format: "invlaid_format" +-- test formats +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"format": "text", "costs": false}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + Result (actual rows=1 loops=1)+ + +(1 row) + +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"format": "json", "costs": false}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + [ + + { + + "Plan": { + + "Node Type": "Result", + + "Parallel Aware": false,+ + "Async Capable": false, + + "Actual Rows": 1, + + "Actual Loops": 1 + + }, + + "Triggers": [ + + ] + + } + + ] +(1 row) + +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"format": "xml", "costs": false}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + + + + + + + Result + + false + + false + + 1 + + 1 + + + + + + + + + + +(1 row) + +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"format": "yaml", "costs": false}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output FROM worker_last_saved_explain_analyze(); + explain_analyze_output +--------------------------------------------------------------------- + - Plan: + + Node Type: "Result" + + Parallel Aware: false+ + Async Capable: false + + Actual Rows: 1 + + Actual Loops: 1 + + Triggers: +(1 row) + +END; +-- costs on, timing off +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT * FROM explain_analyze_test', '{"timing": false, "costs": true}') as (a int); + a +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +SELECT explain_analyze_output ~ 'Seq Scan.*\(cost=0.00.*\) \(actual rows.*\)' FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- costs off, timing on +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT * FROM explain_analyze_test', '{"timing": true, "costs": false}') as (a int); + a +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +SELECT explain_analyze_output ~ 'Seq Scan on explain_analyze_test \(actual time=.* rows=.* loops=1\)' FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- summary on +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{"timing": false, "costs": false, "summary": true}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT explain_analyze_output ~ 'Planning Time:.*Execution Time:.*' FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- buffers on +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT * FROM explain_analyze_test', '{"timing": false, "costs": false, "buffers": true}') as (a int); + a +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +SELECT explain_analyze_output ~ 'Buffers:' FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- verbose on +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('SELECT * FROM explain_analyze_test', '{"timing": false, "costs": false, "verbose": true}') as (a int); + a +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +SELECT explain_analyze_output ~ 'Output: a, b' FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- make sure deleted at transaction end +SELECT * FROM worker_save_query_explain_analyze('SELECT 1', '{}') as (a int); + a +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT count(*) FROM worker_last_saved_explain_analyze(); + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- should be deleted at the end of prepare commit +BEGIN; +SELECT * FROM worker_save_query_explain_analyze('UPDATE explain_analyze_test SET a=6 WHERE a=4', '{}') as (a int); + a +--------------------------------------------------------------------- +(0 rows) + +SELECT count(*) FROM worker_last_saved_explain_analyze(); + count +--------------------------------------------------------------------- + 1 +(1 row) + +PREPARE TRANSACTION 'citus_0_1496350_7_0'; +SELECT count(*) FROM worker_last_saved_explain_analyze(); + count +--------------------------------------------------------------------- + 0 +(1 row) + +COMMIT PREPARED 'citus_0_1496350_7_0'; +-- verify execution time makes sense +BEGIN; +SELECT count(*) FROM worker_save_query_explain_analyze('SELECT pg_sleep(0.05)', :default_opts) as (a int); + count +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT execution_duration BETWEEN 30 AND 200 FROM worker_last_saved_explain_analyze(); + ?column? +--------------------------------------------------------------------- + t +(1 row) + +END; +-- +-- verify we handle parametrized queries properly +-- +CREATE TABLE t(a int); +INSERT INTO t VALUES (1), (2), (3); +-- simple case +PREPARE save_explain AS +SELECT $1, * FROM worker_save_query_explain_analyze('SELECT $1::int', :default_opts) as (a int); +EXECUTE save_explain(1); + ?column? | a +--------------------------------------------------------------------- + 1 | 1 +(1 row) + +deallocate save_explain; +-- Call a UDF first to make sure that we handle stacks of executorBoundParams properly. +-- +-- The prepared statement will first call f() which will force new executor run with new +-- set of parameters. Then it will call worker_save_query_explain_analyze with a +-- parametrized query. If we don't have the correct set of parameters here, it will fail. +CREATE FUNCTION f() RETURNS INT +AS $$ +PREPARE pp1 AS SELECT $1 WHERE $2 = $3; +EXECUTE pp1(4, 5, 5); +deallocate pp1; +SELECT 1$$ LANGUAGE sql volatile; +PREPARE save_explain AS + SELECT $1, CASE WHEN i < 2 THEN + f() = 1 + ELSE + EXISTS(SELECT * FROM worker_save_query_explain_analyze('SELECT $1::int', :default_opts) as (a int) + WHERE a = 1) + END + FROM generate_series(1, 4) i; +EXECUTE save_explain(1); + ?column? | exists +--------------------------------------------------------------------- + 1 | t + 1 | t + 1 | t + 1 | t +(4 rows) + +deallocate save_explain; +DROP FUNCTION f(); +DROP TABLE t; +SELECT * FROM explain_analyze_test ORDER BY a; + a | b +--------------------------------------------------------------------- + 1 | value 1 + 2 | value 2 + 3 | value 3 + 6 | value 4 +(4 rows) + +\a\t +-- +-- Test different cases of EXPLAIN ANALYZE +-- +SET citus.shard_count TO 4; +SET client_min_messages TO WARNING; +SELECT create_distributed_table('explain_analyze_test', 'a'); + +\set default_analyze_flags '(ANALYZE on, COSTS off, TIMING off, SUMMARY off)' +\set default_explain_flags '(ANALYZE off, COSTS off, TIMING off, SUMMARY off)' +-- router SELECT +EXPLAIN :default_analyze_flags SELECT * FROM explain_analyze_test WHERE a = 1; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 11 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 11 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) + Filter: (a = 1) +-- multi-shard SELECT +EXPLAIN :default_analyze_flags SELECT count(*) FROM explain_analyze_test; +Aggregate (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=4 loops=1) + Task Count: 4 + Tuple data received from nodes: 32 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) +-- empty router SELECT +EXPLAIN :default_analyze_flags SELECT * FROM explain_analyze_test WHERE a = 10000; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on explain_analyze_test_570012 explain_analyze_test (actual rows=0 loops=1) + Filter: (a = 10000) + Rows Removed by Filter: 1 +-- empty multi-shard SELECT +EXPLAIN :default_analyze_flags SELECT * FROM explain_analyze_test WHERE b = 'does not exist'; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 4 + Tuple data received from nodes: 0 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + Filter: (b = 'does not exist'::text) + Rows Removed by Filter: 1 +-- router DML +BEGIN; +EXPLAIN :default_analyze_flags DELETE FROM explain_analyze_test WHERE a = 1; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) + Filter: (a = 1) +EXPLAIN :default_analyze_flags UPDATE explain_analyze_test SET b = 'b' WHERE a = 2; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on explain_analyze_test_570012 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570012 explain_analyze_test (actual rows=1 loops=1) + Filter: (a = 2) +SELECT * FROM explain_analyze_test ORDER BY a; +2|b +3|value 3 +6|value 4 +ROLLBACK; +-- multi-shard DML +BEGIN; +EXPLAIN :default_analyze_flags UPDATE explain_analyze_test SET b = 'b' WHERE a IN (1, 2); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) + Filter: (a = ANY ('{1,2}'::integer[])) +EXPLAIN :default_analyze_flags DELETE FROM explain_analyze_test; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) +SELECT * FROM explain_analyze_test ORDER BY a; +ROLLBACK; +-- router DML with RETURNING with empty result +EXPLAIN :default_analyze_flags UPDATE explain_analyze_test SET b = 'something' WHERE a = 10000 RETURNING *; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Update on explain_analyze_test_570012 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570012 explain_analyze_test (actual rows=0 loops=1) + Filter: (a = 10000) + Rows Removed by Filter: 1 +-- multi-shard DML with RETURNING with empty result +EXPLAIN :default_analyze_flags UPDATE explain_analyze_test SET b = 'something' WHERE b = 'does not exist' RETURNING *; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 4 + Tuple data received from nodes: 0 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Update on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=0 loops=1) + Filter: (b = 'does not exist'::text) + Rows Removed by Filter: 1 +-- single-row insert +BEGIN; +EXPLAIN :default_analyze_flags INSERT INTO explain_analyze_test VALUES (5, 'value 5'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on explain_analyze_test_570009 (actual rows=0 loops=1) + -> Result (actual rows=1 loops=1) +ROLLBACK; +-- multi-row insert +BEGIN; +EXPLAIN :default_analyze_flags INSERT INTO explain_analyze_test VALUES (5, 'value 5'), (6, 'value 6'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on explain_analyze_test_570009 citus_table_alias (actual rows=0 loops=1) + -> Result (actual rows=1 loops=1) +ROLLBACK; +-- distributed insert/select +BEGIN; +EXPLAIN :default_analyze_flags INSERT INTO explain_analyze_test SELECT * FROM explain_analyze_test; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on explain_analyze_test_570009 citus_table_alias (actual rows=0 loops=1) + -> Seq Scan on explain_analyze_test_570009 explain_analyze_test (actual rows=1 loops=1) + Filter: (a IS NOT NULL) +ROLLBACK; +DROP TABLE explain_analyze_test; +-- test EXPLAIN ANALYZE works fine with primary keys +CREATE TABLE explain_pk(a int primary key, b int); +SELECT create_distributed_table('explain_pk', 'a'); + +BEGIN; +EXPLAIN :default_analyze_flags INSERT INTO explain_pk VALUES (1, 2), (2, 3); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on explain_pk_570013 citus_table_alias (actual rows=0 loops=1) + -> Result (actual rows=1 loops=1) +SELECT * FROM explain_pk ORDER BY 1; +1|2 +2|3 +ROLLBACK; +-- test EXPLAIN ANALYZE with non-text output formats +BEGIN; +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT JSON) INSERT INTO explain_pk VALUES (1, 2), (2, 3); +[ + { + "Plan": { + "Node Type": "Custom Scan", + "Custom Plan Provider": "Citus Adaptive", + "Parallel Aware": false, + "Async Capable": false, + "Actual Rows": 0, + "Actual Loops": 1, + "Distributed Query": { + "Job": { + "Task Count": 2, + "Tasks Shown": "One of 2", + "Tasks": [ + { + "Node": "host=localhost port=xxxxx dbname=regression", + "Remote Plan": [ + [ + { + "Plan": { + "Node Type": "ModifyTable", + "Operation": "Insert", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "explain_pk_570013", + "Alias": "citus_table_alias", + "Actual Rows": 0, + "Actual Loops": 1, + "Plans": [ + { + "Node Type": "Result", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Actual Rows": 1, + "Actual Loops": 1 + } + ] + }, + "Triggers": [ + ] + } + ] + + ] + } + ] + } + } + }, + "Triggers": [ + ] + } +] +ROLLBACK; +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT JSON) SELECT * FROM explain_pk; +[ + { + "Plan": { + "Node Type": "Custom Scan", + "Custom Plan Provider": "Citus Adaptive", + "Parallel Aware": false, + "Async Capable": false, + "Actual Rows": 0, + "Actual Loops": 1, + "Distributed Query": { + "Job": { + "Task Count": 4, + "Tuple data received from nodes": "0 bytes", + "Tasks Shown": "One of 4", + "Tasks": [ + { + "Tuple data received from node": "0 bytes", + "Node": "host=localhost port=xxxxx dbname=regression", + "Remote Plan": [ + [ + { + "Plan": { + "Node Type": "Seq Scan", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "explain_pk_570013", + "Alias": "explain_pk", + "Actual Rows": 0, + "Actual Loops": 1 + }, + "Triggers": [ + ] + } + ] + + ] + } + ] + } + } + }, + "Triggers": [ + ] + } +] +BEGIN; +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT XML) INSERT INTO explain_pk VALUES (1, 2), (2, 3); + + + + Custom Scan + Citus Adaptive + false + false + 0 + 1 + + + 2 + One of 2 + + + host=localhost port=xxxxx dbname=regression + + + + + ModifyTable + Insert + false + false + explain_pk_570013 + citus_table_alias + 0 + 1 + + + Result + Outer + false + false + 1 + 1 + + + + + + + + + + + + + + + + + +ROLLBACK; +EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT XML) SELECT * FROM explain_pk; + + + + Custom Scan + Citus Adaptive + false + false + 0 + 1 + + + 4 + 0 bytes + One of 4 + + + 0 bytes + host=localhost port=xxxxx dbname=regression + + + + + Seq Scan + false + false + explain_pk_570013 + explain_pk + 0 + 1 + + + + + + + + + + + + + + + +DROP TABLE explain_pk; +-- test EXPLAIN ANALYZE with CTEs and subqueries +CREATE TABLE dist_table(a int, b int); +SELECT create_distributed_table('dist_table', 'a'); + +CREATE TABLE ref_table(a int); +SELECT create_reference_table('ref_table'); + +INSERT INTO dist_table SELECT i, i*i FROM generate_series(1, 10) i; +INSERT INTO ref_table SELECT i FROM generate_series(1, 10) i; +EXPLAIN :default_analyze_flags +WITH r AS ( + SELECT GREATEST(random(), 2) r, a FROM dist_table +) +SELECT count(distinct a) from r NATURAL JOIN ref_table; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 220 bytes + Result destination: Send to 3 nodes + -> Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 4 + Tuple data received from nodes: 120 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 48 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) + Task Count: 1 + Tuple data received from nodes: 8 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + -> Hash Join (actual rows=10 loops=1) + Hash Cond: (ref_table.a = intermediate_result.a) + -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) + -> Hash (actual rows=10 loops=1) + -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) +EXPLAIN :default_analyze_flags +SELECT count(distinct a) FROM (SELECT GREATEST(random(), 2) r, a FROM dist_table) t NATURAL JOIN ref_table; +Aggregate (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=4 loops=1) + Task Count: 4 + Tuple data received from nodes: 32 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + -> Merge Join (actual rows=4 loops=1) + Merge Cond: (t.a = ref_table.a) + -> Sort (actual rows=4 loops=1) + Sort Key: t.a + Sort Method: quicksort Memory: 25kB + -> Subquery Scan on t (actual rows=4 loops=1) + -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) + -> Sort (actual rows=10 loops=1) + Sort Key: ref_table.a + Sort Method: quicksort Memory: 25kB + -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) +EXPLAIN :default_analyze_flags +SELECT count(distinct a) FROM dist_table +WHERE EXISTS(SELECT random() < 2 FROM dist_table NATURAL JOIN ref_table); +Aggregate (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=4 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 70 bytes + Result destination: Send to 2 nodes + -> Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 4 + Tuple data received from nodes: 10 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Merge Join (actual rows=4 loops=1) + Merge Cond: (dist_table.a = ref_table.a) + -> Sort (actual rows=4 loops=1) + Sort Key: dist_table.a + Sort Method: quicksort Memory: 25kB + -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) + -> Sort (actual rows=10 loops=1) + Sort Key: ref_table.a + Sort Method: quicksort Memory: 25kB + -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) + Task Count: 4 + Tuple data received from nodes: 32 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + InitPlan 1 (returns $0) + -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) + -> Result (actual rows=4 loops=1) + One-Time Filter: $0 + -> Seq Scan on dist_table_570017 dist_table (actual rows=4 loops=1) +BEGIN; +EXPLAIN :default_analyze_flags +WITH r AS ( + INSERT INTO dist_table SELECT a, a * a FROM dist_table + RETURNING a +), s AS ( + SELECT random() < 2, a * a a2 FROM r +) +SELECT count(distinct a2) FROM s; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 100 bytes + Result destination: Write locally + -> Custom Scan (Citus Adaptive) (actual rows=20 loops=1) + Task Count: 4 + Tuple data received from nodes: 160 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 64 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Insert on dist_table_570017 citus_table_alias (actual rows=8 loops=1) + -> Seq Scan on dist_table_570017 dist_table (actual rows=8 loops=1) + Filter: (a IS NOT NULL) + -> Distributed Subplan XXX_2 + Intermediate Data Size: 150 bytes + Result destination: Write locally + -> Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 1 + Tuple data received from nodes: 50 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 50 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) + Task Count: 1 + Tuple data received from nodes: 8 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 8 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate (actual rows=1 loops=1) + -> Function Scan on read_intermediate_result intermediate_result (actual rows=10 loops=1) +ROLLBACK; +-- https://github.com/citusdata/citus/issues/4074 +prepare ref_select(int) AS select * from ref_table where 1 = $1; +explain :default_analyze_flags execute ref_select(1); +Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 1 + Tuple data received from nodes: 40 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 40 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Result (actual rows=10 loops=1) + One-Time Filter: (1 = $1) + -> Seq Scan on ref_table_570021 ref_table (actual rows=10 loops=1) +deallocate ref_select; +DROP TABLE ref_table, dist_table; +-- test EXPLAIN ANALYZE with different replication factors +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 1; +CREATE TABLE dist_table_rep1(a int); +SELECT create_distributed_table('dist_table_rep1', 'a'); + +SET citus.shard_replication_factor = 2; +CREATE TABLE dist_table_rep2(a int); +SELECT create_distributed_table('dist_table_rep2', 'a'); + +EXPLAIN :default_analyze_flags INSERT INTO dist_table_rep1 VALUES(1), (2), (3), (4), (10), (100) RETURNING *; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Insert on dist_table_rep1_570022 citus_table_alias (actual rows=4 loops=1) + -> Values Scan on "*VALUES*" (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags SELECT * from dist_table_rep1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags INSERT INTO dist_table_rep2 VALUES(1), (2), (3), (4), (10), (100) RETURNING *; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 48 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 32 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Insert on dist_table_rep2_570024 citus_table_alias (actual rows=4 loops=1) + -> Values Scan on "*VALUES*" (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags SELECT * from dist_table_rep2; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep2_570024 dist_table_rep2 (actual rows=4 loops=1) +prepare p1 as SELECT * FROM dist_table_rep1; +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +EXPLAIN :default_analyze_flags EXECUTE p1; +Custom Scan (Citus Adaptive) (actual rows=6 loops=1) + Task Count: 2 + Tuple data received from nodes: 24 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=4 loops=1) +prepare p2 AS SELECT * FROM dist_table_rep1 WHERE a = $1; +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(1); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(10); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 10) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p2(100); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570023 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 100) + Rows Removed by Filter: 1 +prepare p3 AS SELECT * FROM dist_table_rep1 WHERE a = 1; +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +EXPLAIN :default_analyze_flags EXECUTE p3; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tuple data received from nodes: 4 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on dist_table_rep1_570022 dist_table_rep1 (actual rows=1 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 3 +DROP TABLE dist_table_rep1, dist_table_rep2; +-- https://github.com/citusdata/citus/issues/2009 +CREATE TABLE simple (id integer, name text); +SELECT create_distributed_table('simple', 'id'); + +PREPARE simple_router AS SELECT *, $1 FROM simple WHERE id = 1; +EXPLAIN :default_explain_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +EXPLAIN :default_analyze_flags EXECUTE simple_router(1); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (id = 1) +deallocate simple_router; +-- prepared multi-row insert +PREPARE insert_query AS INSERT INTO simple VALUES ($1, 2), (2, $2); +EXPLAIN :default_explain_flags EXECUTE insert_query(3, 4); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on simple_570026 citus_table_alias + -> Result +EXPLAIN :default_analyze_flags EXECUTE insert_query(3, 4); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on simple_570026 citus_table_alias (actual rows=0 loops=1) + -> Result (actual rows=1 loops=1) +deallocate insert_query; +-- prepared updates +PREPARE update_query AS UPDATE simple SET name=$1 WHERE name=$2; +EXPLAIN :default_explain_flags EXECUTE update_query('x', 'y'); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on simple_570026 simple + -> Seq Scan on simple_570026 simple + Filter: (name = 'y'::text) +EXPLAIN :default_analyze_flags EXECUTE update_query('x', 'y'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Update on simple_570026 simple (actual rows=0 loops=1) + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: (name = $2) + Rows Removed by Filter: 1 +deallocate update_query; +-- prepared deletes +PREPARE delete_query AS DELETE FROM simple WHERE name=$1 OR name=$2; +EXPLAIN (COSTS OFF) EXECUTE delete_query('x', 'y'); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on simple_570026 simple + -> Seq Scan on simple_570026 simple + Filter: ((name = 'x'::text) OR (name = 'y'::text)) +EXPLAIN :default_analyze_flags EXECUTE delete_query('x', 'y'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on simple_570026 simple (actual rows=0 loops=1) + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: ((name = $1) OR (name = $2)) + Rows Removed by Filter: 1 +deallocate delete_query; +-- prepared distributed insert/select +-- we don't support EXPLAIN for prepared insert/selects of other types. +PREPARE distributed_insert_select AS INSERT INTO simple SELECT * FROM simple WHERE name IN ($1, $2); +EXPLAIN :default_explain_flags EXECUTE distributed_insert_select('x', 'y'); +Custom Scan (Citus Adaptive) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on simple_570026 citus_table_alias + -> Seq Scan on simple_570026 simple + Filter: ((id IS NOT NULL) AND (name = ANY ('{x,y}'::text[]))) +EXPLAIN :default_analyze_flags EXECUTE distributed_insert_select('x', 'y'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on simple_570026 citus_table_alias (actual rows=0 loops=1) + -> Seq Scan on simple_570026 simple (actual rows=0 loops=1) + Filter: ((id IS NOT NULL) AND (name = ANY (ARRAY[$1, $2]))) + Rows Removed by Filter: 1 +deallocate distributed_insert_select; +DROP TABLE simple; +-- prepared cte +BEGIN; +PREPARE cte_query AS +WITH keys AS ( + SELECT count(*) FROM + (SELECT DISTINCT l_orderkey, GREATEST(random(), 2) FROM lineitem_hash_part WHERE l_quantity > $1) t +), +series AS ( + SELECT s FROM generate_series(1, $2) s +), +delete_result AS ( + DELETE FROM lineitem_hash_part WHERE l_quantity < $3 RETURNING * +) +SELECT s FROM series; +EXPLAIN :default_explain_flags EXECUTE cte_query(2, 10, -1); +Custom Scan (Citus Adaptive) + -> Distributed Subplan XXX_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360041 lineitem_hash_part + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_quantity < '-1'::numeric) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Function Scan on generate_series s +EXPLAIN :default_analyze_flags EXECUTE cte_query(2, 10, -1); +Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 0 bytes + Result destination: Send to 0 nodes + -> Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 4 + Tuple data received from nodes: 0 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Delete on lineitem_hash_part_360041 lineitem_hash_part (actual rows=0 loops=1) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part (actual rows=0 loops=1) + Filter: (l_quantity < '-1'::numeric) + Rows Removed by Filter: 2885 + Task Count: 1 + Tuple data received from nodes: 40 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 40 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Function Scan on generate_series s (actual rows=10 loops=1) +ROLLBACK; +-- https://github.com/citusdata/citus/issues/2009#issuecomment-653036502 +CREATE TABLE users_table_2 (user_id int primary key, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint); +SELECT create_reference_table('users_table_2'); + +PREPARE p4 (int, int) AS insert into users_table_2 ( value_1, user_id) select value_1, user_id + $2 FROM users_table_2 ON CONFLICT (user_id) DO UPDATE SET value_2 = EXCLUDED.value_1 + $1; +EXPLAIN :default_explain_flags execute p4(20,20); +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on users_table_2_570028 citus_table_alias + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: users_table_2_pkey_570028 + -> Seq Scan on users_table_2_570028 users_table_2 +EXPLAIN :default_analyze_flags execute p4(20,20); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Insert on users_table_2_570028 citus_table_alias (actual rows=0 loops=1) + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: users_table_2_pkey_570028 + Tuples Inserted: 0 + Conflicting Tuples: 0 + -> Seq Scan on users_table_2_570028 users_table_2 (actual rows=0 loops=1) +-- simple test to confirm we can fetch long (>4KB) plans +EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM users_table_2 WHERE value_1::text = '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000X'; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on users_table_2_570028 users_table_2 (actual rows=0 loops=1) + Filter: ((value_1)::text = '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000X'::text) +DROP TABLE users_table_2; +-- sorted explain analyze output +CREATE TABLE explain_analyze_execution_time (a int); +INSERT INTO explain_analyze_execution_time VALUES (2); +SELECT create_distributed_table('explain_analyze_execution_time', 'a'); + +-- show that we can sort the output wrt execution time +-- we do the following hack to make the test outputs +-- be consistent. First, ingest a single row then add +-- pg_sleep() call on the query. Postgres will only +-- sleep for the shard that has the single row, so that +-- will definitely be slower +set citus.explain_analyze_sort_method to "taskId"; +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) select a, CASE WHEN pg_sleep(0.4) IS NULL THEN 'x' END from explain_analyze_execution_time; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 2 + Tuple data received from nodes: 4 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on explain_analyze_execution_time_570029 explain_analyze_execution_time (actual rows=0 loops=1) +set citus.explain_analyze_sort_method to "execution-time"; +EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE) select a, CASE WHEN pg_sleep(0.4) IS NULL THEN 'x' END from explain_analyze_execution_time; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 2 + Tuple data received from nodes: 4 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 4 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on explain_analyze_execution_time_570030 explain_analyze_execution_time (actual rows=1 loops=1) +-- reset back +reset citus.explain_analyze_sort_method; +DROP TABLE explain_analyze_execution_time; +CREATE SCHEMA multi_explain; +SET search_path TO multi_explain; +-- test EXPLAIN ANALYZE when original query returns no columns +CREATE TABLE reference_table(a int); +SELECT create_reference_table('reference_table'); + +INSERT INTO reference_table VALUES (1); +EXPLAIN :default_analyze_flags SELECT FROM reference_table; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on reference_table_570031 reference_table (actual rows=1 loops=1) +CREATE TABLE distributed_table_1(a int, b int); +SELECT create_distributed_table('distributed_table_1','a'); + +INSERT INTO distributed_table_1 values (1,1); +EXPLAIN :default_analyze_flags SELECT row_number() OVER() AS r FROM distributed_table_1; +WindowAgg (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on distributed_table_1_570032 distributed_table_1 (actual rows=1 loops=1) +CREATE TABLE distributed_table_2(a int, b int); +SELECT create_distributed_table('distributed_table_2','a'); + +INSERT INTO distributed_table_2 VALUES (1,1); +EXPLAIN :default_analyze_flags +WITH r AS (SELECT row_number() OVER () AS r FROM distributed_table_1) +SELECT * FROM distributed_table_2 +JOIN r ON (r = distributed_table_2.b) +LIMIT 3; +Limit (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + -> Distributed Subplan XXX_1 + Intermediate Data Size: 14 bytes + Result destination: Send to 2 nodes + -> WindowAgg (actual rows=1 loops=1) + -> Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on distributed_table_1_570032 distributed_table_1 (actual rows=1 loops=1) + Task Count: 2 + Tuple data received from nodes: 16 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 16 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Limit (actual rows=1 loops=1) + -> Nested Loop (actual rows=1 loops=1) + Join Filter: (distributed_table_2.b = intermediate_result.r) + -> Function Scan on read_intermediate_result intermediate_result (actual rows=1 loops=1) + -> Seq Scan on distributed_table_2_570034 distributed_table_2 (actual rows=1 loops=1) +EXPLAIN :default_analyze_flags SELECT FROM (SELECT * FROM reference_table) subquery; +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on reference_table_570031 reference_table (actual rows=1 loops=1) +PREPARE dummy_prep_stmt(int) AS SELECT FROM distributed_table_1; +EXPLAIN :default_analyze_flags EXECUTE dummy_prep_stmt(50); +Custom Scan (Citus Adaptive) (actual rows=1 loops=1) + Task Count: 2 + Tasks Shown: One of 2 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on distributed_table_1_570032 distributed_table_1 (actual rows=1 loops=1) +CREATE TYPE multi_explain.int_wrapper_type AS (int_field int); +CREATE TABLE tbl (a int, b multi_explain.int_wrapper_type); +SELECT create_distributed_table('tbl', 'a'); + +EXPLAIN :default_analyze_flags SELECT * FROM tbl; +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 2 + Tuple data received from nodes: 0 bytes + Tasks Shown: One of 2 + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on tbl_570036 tbl (actual rows=0 loops=1) +PREPARE q1(int_wrapper_type) AS WITH a AS (SELECT * FROM tbl WHERE b = $1 AND a = 1 OFFSET 0) SELECT * FROM a; +EXPLAIN (COSTS false) EXECUTE q1('(1)'); +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on tbl_570036 tbl + Filter: ((b = '(1)'::multi_explain.int_wrapper_type) AND (a = 1)) +EXPLAIN :default_analyze_flags EXECUTE q1('(1)'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on tbl_570036 tbl (actual rows=0 loops=1) + Filter: ((b = $1) AND (a = 1)) +PREPARE q2(int_wrapper_type) AS WITH a AS (UPDATE tbl SET b = $1 WHERE a = 1 RETURNING *) SELECT * FROM a; +EXPLAIN (COSTS false) EXECUTE q2('(1)'); +Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> CTE Scan on a + CTE a + -> Update on tbl_570036 tbl + -> Seq Scan on tbl_570036 tbl + Filter: (a = 1) +EXPLAIN :default_analyze_flags EXECUTE q2('(1)'); +Custom Scan (Citus Adaptive) (actual rows=0 loops=1) + Task Count: 1 + Tuple data received from nodes: 0 bytes + Tasks Shown: All + -> Task + Tuple data received from node: 0 bytes + Node: host=localhost port=xxxxx dbname=regression + -> CTE Scan on a (actual rows=0 loops=1) + CTE a + -> Update on tbl_570036 tbl (actual rows=0 loops=1) + -> Seq Scan on tbl_570036 tbl (actual rows=0 loops=1) + Filter: (a = 1) +-- check when auto explain + analyze is enabled, we do not allow local execution. +CREATE SCHEMA test_auto_explain; +SET search_path TO 'test_auto_explain'; +CREATE TABLE test_ref_table (key int PRIMARY KEY); +SELECT create_reference_table('test_ref_table'); + +LOAD 'auto_explain'; +SET auto_explain.log_min_duration = 0; +set auto_explain.log_analyze to true; +-- the following should not be locally executed since explain analyze is on +select * from test_ref_table; +DROP SCHEMA test_auto_explain CASCADE; +SET client_min_messages TO ERROR; +DROP SCHEMA multi_explain CASCADE; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 72b354966..794568eb4 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -986,7 +986,7 @@ DELETE FROM pg_dist_shard WHERE shardid = 1; CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id); CREATE TABLE orders_2020_07_01 PARTITION OF e_transactions FOR VALUES IN (1,2,3); -INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's'); +INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's'); SELECT (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, (metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null diff --git a/src/test/regress/expected/multi_hash_pruning.out b/src/test/regress/expected/multi_hash_pruning.out index 0a113c5f8..09b1ccd87 100644 --- a/src/test/regress/expected/multi_hash_pruning.out +++ b/src/test/regress/expected/multi_hash_pruning.out @@ -1232,31 +1232,20 @@ WHERE o_orderkey IN (1, 2) -> Seq Scan on lineitem_hash_partitioned_630004 lineitem_hash_partitioned (13 rows) +SELECT public.coordinator_plan($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM orders_hash_partitioned FULL OUTER JOIN lineitem_hash_partitioned ON (o_orderkey = l_orderkey) WHERE o_orderkey IN (1, 2) AND l_orderkey IN (2, 3); - QUERY PLAN +$Q$); + coordinator_plan --------------------------------------------------------------------- Aggregate -> Custom Scan (Citus Adaptive) Task Count: 3 - Tasks Shown: One of 3 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Nested Loop - Join Filter: (orders_hash_partitioned.o_orderkey = lineitem_hash_partitioned.l_orderkey) - -> Seq Scan on orders_hash_partitioned_630000 orders_hash_partitioned - Filter: (o_orderkey = ANY ('{1,2}'::integer[])) - -> Materialize - -> Bitmap Heap Scan on lineitem_hash_partitioned_630004 lineitem_hash_partitioned - Recheck Cond: (l_orderkey = ANY ('{2,3}'::integer[])) - -> Bitmap Index Scan on lineitem_hash_partitioned_pkey_630004 - Index Cond: (l_orderkey = ANY ('{2,3}'::integer[])) -(16 rows) +(3 rows) SET citus.task_executor_type TO DEFAULT; DROP TABLE lineitem_hash_partitioned; diff --git a/src/test/regress/expected/multi_having_pushdown.out b/src/test/regress/expected/multi_having_pushdown.out index d2051a55c..a1ef9f52f 100644 --- a/src/test/regress/expected/multi_having_pushdown.out +++ b/src/test/regress/expected/multi_having_pushdown.out @@ -120,7 +120,7 @@ EXPLAIN (COSTS FALSE) SELECT sum(l_extendedprice * l_discount) as revenue FROM lineitem_hash, orders_hash WHERE o_orderkey = l_orderkey - GROUP BY l_orderkey, o_orderkey, l_shipmode HAVING sum(l_quantity) > 24 + GROUP BY l_orderkey, l_shipmode HAVING sum(l_quantity) > 24 ORDER BY 1 DESC LIMIT 3; QUERY PLAN --------------------------------------------------------------------- @@ -136,7 +136,7 @@ EXPLAIN (COSTS FALSE) -> Sort Sort Key: (sum((lineitem_hash.l_extendedprice * lineitem_hash.l_discount))) DESC -> HashAggregate - Group Key: lineitem_hash.l_orderkey, orders_hash.o_orderkey, lineitem_hash.l_shipmode + Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_shipmode Filter: (sum(lineitem_hash.l_quantity) > '24'::numeric) -> Hash Join Hash Cond: (orders_hash.o_orderkey = lineitem_hash.l_orderkey) diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out index e85253031..cb8f0c0e1 100644 --- a/src/test/regress/expected/multi_metadata_sync.out +++ b/src/test/regress/expected/multi_metadata_sync.out @@ -533,7 +533,7 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f (1 row) SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; @@ -672,7 +672,7 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f (1 row) SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; diff --git a/src/test/regress/expected/multi_metadata_sync_0.out b/src/test/regress/expected/multi_metadata_sync_0.out index 6e1ba6525..c81462e6f 100644 --- a/src/test/regress/expected/multi_metadata_sync_0.out +++ b/src/test/regress/expected/multi_metadata_sync_0.out @@ -533,7 +533,7 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f (1 row) SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; @@ -672,7 +672,7 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY logicalrelid::text; logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 2 | s | f (1 row) SELECT * FROM pg_dist_shard WHERE logicalrelid::text LIKE 'mx_testing_schema%' ORDER BY shardid; diff --git a/src/test/regress/expected/multi_move_mx.out b/src/test/regress/expected/multi_move_mx.out index b6cc5d0d7..b5aeec8ca 100644 --- a/src/test/regress/expected/multi_move_mx.out +++ b/src/test/regress/expected/multi_move_mx.out @@ -148,7 +148,7 @@ SELECT pg_reload_conf(); CREATE SUBSCRIPTION subs_01 CONNECTION 'host=''localhost'' port=57637' PUBLICATION pub_01 WITH (citus_use_authinfo=true); ERROR: could not connect to the publisher: root certificate file "/non/existing/certificate.crt" does not exist -Either provide the file or change sslmode to disable server certificate verification. +Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable server certificate verification. ALTER SYSTEM RESET citus.node_conninfo; SELECT pg_reload_conf(); pg_reload_conf diff --git a/src/test/regress/expected/multi_mx_hide_shard_names.out b/src/test/regress/expected/multi_mx_hide_shard_names.out index 01d9736f2..116269a4e 100644 --- a/src/test/regress/expected/multi_mx_hide_shard_names.out +++ b/src/test/regress/expected/multi_mx_hide_shard_names.out @@ -425,9 +425,25 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name test_table_2_1130000 (4 rows) +-- PG16 added one more backend type B_STANDALONE_BACKEND +-- and also alphabetized the backend types, hence the orders changed +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +SELECT 4 AS client_backend \gset +SELECT 5 AS bgworker \gset +SELECT 12 AS walsender \gset +\else +SELECT 3 AS client_backend \gset +SELECT 4 AS bgworker \gset +SELECT 9 AS walsender \gset +\endif -- say, we set it to bgworker -- the shards and indexes do not show up -SELECT set_backend_type(4); +SELECT set_backend_type(:bgworker); NOTICE: backend type switched to: background worker set_backend_type --------------------------------------------------------------------- @@ -445,7 +461,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- or, we set it to walsender -- the shards and indexes do not show up -SELECT set_backend_type(9); +SELECT set_backend_type(:walsender); NOTICE: backend type switched to: walsender set_backend_type --------------------------------------------------------------------- @@ -480,7 +496,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name RESET application_name; -- but, client backends to see the shards -SELECT set_backend_type(3); +SELECT set_backend_type(:client_backend); NOTICE: backend type switched to: client backend set_backend_type --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_orderby_limit_pushdown.out b/src/test/regress/expected/multi_orderby_limit_pushdown.out index 9d67c9810..0a625e47f 100644 --- a/src/test/regress/expected/multi_orderby_limit_pushdown.out +++ b/src/test/regress/expected/multi_orderby_limit_pushdown.out @@ -374,17 +374,17 @@ LIMIT 2; (2 rows) EXPLAIN (COSTS OFF) -SELECT ut.user_id, count(DISTINCT ut.value_2) +SELECT ut.user_id, avg(ut.value_2) FROM users_table ut, events_table et WHERE ut.user_id = et.user_id and et.value_2 < 5 GROUP BY ut.user_id ORDER BY 2, AVG(ut.value_1), 1 DESC LIMIT 5; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Sort - Sort Key: remote_scan.count, remote_scan.worker_column_3, remote_scan.user_id DESC + Sort Key: remote_scan.avg, remote_scan.worker_column_3, remote_scan.user_id DESC -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 @@ -392,16 +392,14 @@ LIMIT 5; Node: host=localhost port=xxxxx dbname=regression -> Limit -> Sort - Sort Key: (count(DISTINCT ut.value_2)), (avg(ut.value_1)), ut.user_id DESC - -> GroupAggregate + Sort Key: (avg(ut.value_2)), (avg(ut.value_1)), ut.user_id DESC + -> HashAggregate Group Key: ut.user_id - -> Sort - Sort Key: ut.user_id DESC - -> Hash Join - Hash Cond: (ut.user_id = et.user_id) - -> Seq Scan on users_table_1400256 ut - -> Hash - -> Seq Scan on events_table_1400260 et - Filter: (value_2 < 5) -(21 rows) + -> Hash Join + Hash Cond: (ut.user_id = et.user_id) + -> Seq Scan on users_table_1400256 ut + -> Hash + -> Seq Scan on events_table_1400260 et + Filter: (value_2 < 5) +(19 rows) diff --git a/src/test/regress/expected/multi_prune_shard_list.out b/src/test/regress/expected/multi_prune_shard_list.out index 7bbfaeb88..3762bd05d 100644 --- a/src/test/regress/expected/multi_prune_shard_list.out +++ b/src/test/regress/expected/multi_prune_shard_list.out @@ -86,7 +86,7 @@ SELECT prune_using_both_values('pruning', 'tomato', 'rose'); SELECT debug_equality_expression('pruning'); debug_equality_expression --------------------------------------------------------------------- - {OPEXPR :opno 98 :opfuncid 67 :opresulttype 16 :opretset false :opcollid 0 :inputcollid 100 :args ({VAR :varno 1 :varattno 1 :vartype 25 :vartypmod -1 :varcollid 100 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} {CONST :consttype 25 :consttypmod -1 :constcollid 100 :constlen -1 :constbyval false :constisnull true :location -1 :constvalue <>}) :location -1} + {OPEXPR :opno 98 :opfuncid 67 :opresulttype 16 :opretset false :opcollid 0 :inputcollid 100 :args ({VAR :varno 1 :varattno 1 :vartype 25 :vartypmod -1 :varcollid 100 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} {CONST :consttype 25 :consttypmod -1 :constcollid 100 :constlen -1 :constbyval false :constisnull true :location -1 :constvalue <>}) :location -1} (1 row) -- print the initial ordering of shard intervals diff --git a/src/test/regress/expected/multi_select_distinct.out b/src/test/regress/expected/multi_select_distinct.out index 75d47026b..689adcc8a 100644 --- a/src/test/regress/expected/multi_select_distinct.out +++ b/src/test/regress/expected/multi_select_distinct.out @@ -813,7 +813,7 @@ SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) EXPLAIN (COSTS FALSE) SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) FROM lineitem_hash_part - GROUP BY l_orderkey + GROUP BY l_orderkey, l_partkey, l_shipmode ORDER BY 1,2; QUERY PLAN --------------------------------------------------------------------- @@ -827,9 +827,9 @@ EXPLAIN (COSTS FALSE) -> Task Node: host=localhost port=xxxxx dbname=regression -> GroupAggregate - Group Key: l_orderkey + Group Key: l_orderkey, l_partkey, l_shipmode -> Sort - Sort Key: l_orderkey + Sort Key: l_orderkey, l_partkey, l_shipmode -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part (14 rows) @@ -839,7 +839,7 @@ SET enable_hashagg TO off; EXPLAIN (COSTS FALSE) SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) FROM lineitem_hash_part - GROUP BY l_orderkey + GROUP BY l_orderkey, l_partkey, l_shipmode ORDER BY 1,2; QUERY PLAN --------------------------------------------------------------------- @@ -852,9 +852,9 @@ EXPLAIN (COSTS FALSE) -> Task Node: host=localhost port=xxxxx dbname=regression -> GroupAggregate - Group Key: l_orderkey + Group Key: l_orderkey, l_partkey, l_shipmode -> Sort - Sort Key: l_orderkey + Sort Key: l_orderkey, l_partkey, l_shipmode -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part (13 rows) diff --git a/src/test/regress/expected/multi_subquery.out b/src/test/regress/expected/multi_subquery.out index f4c4ccc21..60f978f5e 100644 --- a/src/test/regress/expected/multi_subquery.out +++ b/src/test/regress/expected/multi_subquery.out @@ -1062,7 +1062,7 @@ SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) (26 rows) EXPLAIN (COSTS OFF) -SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 GROUP BY key ORDER BY 1 DESC LIMIT 1); +SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 ORDER BY 1 DESC LIMIT 1); QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) @@ -1070,20 +1070,18 @@ SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 GROUP BY key HAVING sum(value) Tasks Shown: All -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate - Group Key: k1.key + -> Aggregate Filter: (sum(k1.value) > $0) InitPlan 1 (returns $0) -> Limit -> Sort Sort Key: (sum(k2.value)) DESC - -> GroupAggregate - Group Key: k2.key + -> Aggregate -> Seq Scan on keyval2_xxxxxxx k2 Filter: (key = 2) -> Seq Scan on keyval1_xxxxxxx k1 Filter: (key = 2) -(18 rows) +(16 rows) -- Simple join subquery pushdown SELECT diff --git a/src/test/regress/expected/multi_subquery_in_where_reference_clause.out b/src/test/regress/expected/multi_subquery_in_where_reference_clause.out index 52cbe3917..0f656ee0b 100644 --- a/src/test/regress/expected/multi_subquery_in_where_reference_clause.out +++ b/src/test/regress/expected/multi_subquery_in_where_reference_clause.out @@ -152,7 +152,7 @@ SELECT FROM users_table RIGHT JOIN users_reference_table USING (user_id) WHERE - users_table.value_2 IN + users_reference_table.value_2 IN (SELECT value_2 FROM diff --git a/src/test/regress/expected/multi_view.out b/src/test/regress/expected/multi_view.out index 11f78ea34..3445f442a 100644 --- a/src/test/regress/expected/multi_view.out +++ b/src/test/regress/expected/multi_view.out @@ -92,7 +92,7 @@ SELECT l_orderkey, count(*) FROM priority_lineitem GROUP BY 1 ORDER BY 2 DESC, 1 326 | 7 (5 rows) -CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part WHERE l_shipmode = 'AIR'; +CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part table_name_for_view WHERE l_shipmode = 'AIR'; -- join between view and table SELECT count(*) FROM orders_hash_part join air_shipped_lineitems ON (o_orderkey = l_orderkey); count @@ -179,7 +179,7 @@ SELECT o_orderkey, l_linenumber FROM priority_orders left join air_shipped_linei -- it passes planning, fails at execution stage SET client_min_messages TO DEBUG1; SELECT * FROM priority_orders JOIN air_shipped_lineitems ON (o_custkey = l_suppkey) ORDER BY o_orderkey DESC, o_custkey DESC, o_orderpriority DESC LIMIT 5; -DEBUG: generating subplan XXX_1 for subquery SELECT lineitem_hash_part.l_orderkey, lineitem_hash_part.l_partkey, lineitem_hash_part.l_suppkey, lineitem_hash_part.l_linenumber, lineitem_hash_part.l_quantity, lineitem_hash_part.l_extendedprice, lineitem_hash_part.l_discount, lineitem_hash_part.l_tax, lineitem_hash_part.l_returnflag, lineitem_hash_part.l_linestatus, lineitem_hash_part.l_shipdate, lineitem_hash_part.l_commitdate, lineitem_hash_part.l_receiptdate, lineitem_hash_part.l_shipinstruct, lineitem_hash_part.l_shipmode, lineitem_hash_part.l_comment FROM public.lineitem_hash_part WHERE (lineitem_hash_part.l_shipmode OPERATOR(pg_catalog.=) 'AIR'::bpchar) +DEBUG: generating subplan XXX_1 for subquery SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment FROM public.lineitem_hash_part table_name_for_view WHERE (l_shipmode OPERATOR(pg_catalog.=) 'AIR'::bpchar) DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT priority_orders.o_orderkey, priority_orders.o_custkey, priority_orders.o_orderstatus, priority_orders.o_totalprice, priority_orders.o_orderdate, priority_orders.o_orderpriority, priority_orders.o_clerk, priority_orders.o_shippriority, priority_orders.o_comment, air_shipped_lineitems.l_orderkey, air_shipped_lineitems.l_partkey, air_shipped_lineitems.l_suppkey, air_shipped_lineitems.l_linenumber, air_shipped_lineitems.l_quantity, air_shipped_lineitems.l_extendedprice, air_shipped_lineitems.l_discount, air_shipped_lineitems.l_tax, air_shipped_lineitems.l_returnflag, air_shipped_lineitems.l_linestatus, air_shipped_lineitems.l_shipdate, air_shipped_lineitems.l_commitdate, air_shipped_lineitems.l_receiptdate, air_shipped_lineitems.l_shipinstruct, air_shipped_lineitems.l_shipmode, air_shipped_lineitems.l_comment FROM ((SELECT orders_hash_part.o_orderkey, orders_hash_part.o_custkey, orders_hash_part.o_orderstatus, orders_hash_part.o_totalprice, orders_hash_part.o_orderdate, orders_hash_part.o_orderpriority, orders_hash_part.o_clerk, orders_hash_part.o_shippriority, orders_hash_part.o_comment FROM public.orders_hash_part WHERE (orders_hash_part.o_orderpriority OPERATOR(pg_catalog.<) '3-MEDIUM'::bpchar)) priority_orders JOIN (SELECT intermediate_result.l_orderkey, intermediate_result.l_partkey, intermediate_result.l_suppkey, intermediate_result.l_linenumber, intermediate_result.l_quantity, intermediate_result.l_extendedprice, intermediate_result.l_discount, intermediate_result.l_tax, intermediate_result.l_returnflag, intermediate_result.l_linestatus, intermediate_result.l_shipdate, intermediate_result.l_commitdate, intermediate_result.l_receiptdate, intermediate_result.l_shipinstruct, intermediate_result.l_shipmode, intermediate_result.l_comment FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint, l_partkey integer, l_suppkey integer, l_linenumber integer, l_quantity numeric(15,2), l_extendedprice numeric(15,2), l_discount numeric(15,2), l_tax numeric(15,2), l_returnflag character(1), l_linestatus character(1), l_shipdate date, l_commitdate date, l_receiptdate date, l_shipinstruct character(25), l_shipmode character(10), l_comment character varying(44))) air_shipped_lineitems ON ((priority_orders.o_custkey OPERATOR(pg_catalog.=) air_shipped_lineitems.l_suppkey))) ORDER BY priority_orders.o_orderkey DESC, priority_orders.o_custkey DESC, priority_orders.o_orderpriority DESC LIMIT 5 DEBUG: push down of limit count: 5 o_orderkey | o_custkey | o_orderstatus | o_totalprice | o_orderdate | o_orderpriority | o_clerk | o_shippriority | o_comment | l_orderkey | l_partkey | l_suppkey | l_linenumber | l_quantity | l_extendedprice | l_discount | l_tax | l_returnflag | l_linestatus | l_shipdate | l_commitdate | l_receiptdate | l_shipinstruct | l_shipmode | l_comment diff --git a/src/test/regress/expected/non_colocated_subquery_joins.out b/src/test/regress/expected/non_colocated_subquery_joins.out index 1c1a7d935..bcfe06fba 100644 --- a/src/test/regress/expected/non_colocated_subquery_joins.out +++ b/src/test/regress/expected/non_colocated_subquery_joins.out @@ -1079,19 +1079,19 @@ SELECT create_distributed_table('table1','tenant_id'); (1 row) -CREATE VIEW table1_view AS SELECT * from table1 where id < 100; +CREATE VIEW table1_view AS SELECT * from table1 table_name_for_view where id < 100; -- all of the above queries are non-colocated subquery joins -- because the views are replaced with subqueries UPDATE table2 SET id=20 FROM table1_view WHERE table1_view.id=table2.id; DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT table1.id, table1.tenant_id FROM non_colocated_subquery.table1 WHERE (table1.id OPERATOR(pg_catalog.<) 100) +DEBUG: generating subplan XXX_1 for subquery SELECT id, tenant_id FROM non_colocated_subquery.table1 table_name_for_view WHERE (id OPERATOR(pg_catalog.<) 100) DEBUG: Plan XXX query after replacing subqueries and CTEs: UPDATE non_colocated_subquery.table2 SET id = 20 FROM (SELECT intermediate_result.id, intermediate_result.tenant_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, tenant_id integer)) table1_view WHERE (table1_view.id OPERATOR(pg_catalog.=) table2.id) DEBUG: Creating router plan UPDATE table2_p1 SET id=20 FROM table1_view WHERE table1_view.id=table2_p1.id; DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT table1.id, table1.tenant_id FROM non_colocated_subquery.table1 WHERE (table1.id OPERATOR(pg_catalog.<) 100) +DEBUG: generating subplan XXX_1 for subquery SELECT id, tenant_id FROM non_colocated_subquery.table1 table_name_for_view WHERE (id OPERATOR(pg_catalog.<) 100) DEBUG: Plan XXX query after replacing subqueries and CTEs: UPDATE non_colocated_subquery.table2_p1 SET id = 20 FROM (SELECT intermediate_result.id, intermediate_result.tenant_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, tenant_id integer)) table1_view WHERE (table1_view.id OPERATOR(pg_catalog.=) table2_p1.id) DEBUG: Creating router plan RESET client_min_messages; diff --git a/src/test/regress/expected/pg12.out b/src/test/regress/expected/pg12.out index 8999038ec..acc0c3f63 100644 --- a/src/test/regress/expected/pg12.out +++ b/src/test/regress/expected/pg12.out @@ -370,11 +370,13 @@ SELECT DISTINCT y FROM test; (1 row) -- non deterministic collations +SET client_min_messages TO WARNING; CREATE COLLATION test_pg12.case_insensitive ( provider = icu, locale = '@colStrength=secondary', deterministic = false ); +RESET client_min_messages; CREATE TABLE col_test ( id int, val text collate case_insensitive diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 978411571..1b1d80df2 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -18,21 +18,21 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing VACUUM (FULL) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (FULL, PROCESS_TOAST) t1; -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +NOTICE: issuing VACUUM (FULL) pg14.t1_980000 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +NOTICE: issuing VACUUM (FULL) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (FULL, PROCESS_TOAST true) t1; -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +NOTICE: issuing VACUUM (FULL) pg14.t1_980000 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +NOTICE: issuing VACUUM (FULL) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (FULL, PROCESS_TOAST false) t1; ERROR: PROCESS_TOAST required with VACUUM FULL VACUUM (PROCESS_TOAST false) t1; -NOTICE: issuing VACUUM pg14.t1_980000 +NOTICE: issuing VACUUM (PROCESS_TOAST FALSE) pg14.t1_980000 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing VACUUM pg14.t1_980001 +NOTICE: issuing VACUUM (PROCESS_TOAST FALSE) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (INDEX_CLEANUP AUTO) t1; NOTICE: issuing VACUUM (INDEX_CLEANUP auto) pg14.t1_980000 @@ -62,14 +62,14 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (INDEX_CLEANUP "AUTOX") t1; ERROR: index_cleanup requires a Boolean value VACUUM (FULL, FREEZE, VERBOSE false, ANALYZE, SKIP_LOCKED, INDEX_CLEANUP, PROCESS_TOAST, TRUNCATE) t1; -NOTICE: issuing VACUUM (ANALYZE,FREEZE,FULL,SKIP_LOCKED,PROCESS_TOAST,TRUNCATE,INDEX_CLEANUP auto) pg14.t1_980000 +NOTICE: issuing VACUUM (ANALYZE,FREEZE,FULL,SKIP_LOCKED,TRUNCATE,INDEX_CLEANUP auto) pg14.t1_980000 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing VACUUM (ANALYZE,FREEZE,FULL,SKIP_LOCKED,PROCESS_TOAST,TRUNCATE,INDEX_CLEANUP auto) pg14.t1_980001 +NOTICE: issuing VACUUM (ANALYZE,FREEZE,FULL,SKIP_LOCKED,TRUNCATE,INDEX_CLEANUP auto) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx VACUUM (FULL, FREEZE false, VERBOSE false, ANALYZE false, SKIP_LOCKED false, INDEX_CLEANUP "Auto", PROCESS_TOAST true, TRUNCATE false) t1; -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST,TRUNCATE false,INDEX_CLEANUP auto) pg14.t1_980000 +NOTICE: issuing VACUUM (FULL,TRUNCATE false,INDEX_CLEANUP auto) pg14.t1_980000 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing VACUUM (FULL,PROCESS_TOAST,TRUNCATE false,INDEX_CLEANUP auto) pg14.t1_980001 +NOTICE: issuing VACUUM (FULL,TRUNCATE false,INDEX_CLEANUP auto) pg14.t1_980001 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- vacuum (process_toast true) should be vacuuming toast tables (default is true) CREATE TABLE local_vacuum_table(name text); diff --git a/src/test/regress/expected/pg16.out b/src/test/regress/expected/pg16.out new file mode 100644 index 000000000..37580d8a7 --- /dev/null +++ b/src/test/regress/expected/pg16.out @@ -0,0 +1,70 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q +\endif +CREATE SCHEMA pg16; +SET search_path TO pg16; +SET citus.next_shard_id TO 950000; +SET citus.shard_count TO 1; +SET citus.shard_replication_factor TO 1; +-- test the new vacuum and analyze options +-- Relevant PG commits: +-- https://github.com/postgres/postgres/commit/1cbbee03385763b066ae3961fc61f2cd01a0d0d7 +-- https://github.com/postgres/postgres/commit/4211fbd8413b26e0abedbe4338aa7cda2cd469b4 +-- https://github.com/postgres/postgres/commit/a46a7011b27188af526047a111969f257aaf4db8 +CREATE TABLE t1 (a int); +SELECT create_distributed_table('t1','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET citus.log_remote_commands TO ON; +VACUUM (PROCESS_MAIN FALSE) t1; +NOTICE: issuing VACUUM (PROCESS_MAIN FALSE) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (PROCESS_MAIN FALSE, PROCESS_TOAST FALSE) t1; +NOTICE: issuing VACUUM (PROCESS_TOAST FALSE,PROCESS_MAIN FALSE) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (PROCESS_MAIN TRUE) t1; +NOTICE: issuing VACUUM pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (PROCESS_MAIN FALSE, FULL) t1; +NOTICE: issuing VACUUM (FULL,PROCESS_MAIN FALSE) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (SKIP_DATABASE_STATS) t1; +NOTICE: issuing VACUUM (SKIP_DATABASE_STATS) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (ONLY_DATABASE_STATS) t1; +ERROR: ONLY_DATABASE_STATS cannot be specified with a list of tables +VACUUM (BUFFER_USAGE_LIMIT '512 kB') t1; +NOTICE: issuing VACUUM (BUFFER_USAGE_LIMIT 512) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (BUFFER_USAGE_LIMIT 0) t1; +NOTICE: issuing VACUUM (BUFFER_USAGE_LIMIT 0) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (BUFFER_USAGE_LIMIT 16777220) t1; +ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB +VACUUM (BUFFER_USAGE_LIMIT -1) t1; +ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB +VACUUM (BUFFER_USAGE_LIMIT 'test') t1; +ERROR: BUFFER_USAGE_LIMIT option must be 0 or between 128 kB and 16777216 kB +ANALYZE (BUFFER_USAGE_LIMIT '512 kB') t1; +NOTICE: issuing ANALYZE (BUFFER_USAGE_LIMIT 512) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +ANALYZE (BUFFER_USAGE_LIMIT 0) t1; +NOTICE: issuing ANALYZE (BUFFER_USAGE_LIMIT 0) pg16.t1_950000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands TO OFF; +-- only verifying it works and not printing log +-- remote commands because it can be flaky +VACUUM (ONLY_DATABASE_STATS); +\set VERBOSITY terse +SET client_min_messages TO ERROR; +DROP SCHEMA pg16 CASCADE; diff --git a/src/test/regress/expected/pg16_0.out b/src/test/regress/expected/pg16_0.out new file mode 100644 index 000000000..730c916ca --- /dev/null +++ b/src/test/regress/expected/pg16_0.out @@ -0,0 +1,9 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q diff --git a/src/test/regress/expected/recurring_outer_join.out b/src/test/regress/expected/recurring_outer_join.out index 4ff353838..0764f05dc 100644 --- a/src/test/regress/expected/recurring_outer_join.out +++ b/src/test/regress/expected/recurring_outer_join.out @@ -1187,17 +1187,16 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c -- same test using a view, can be recursively planned CREATE VIEW my_view_1 AS -SELECT * FROM dist_1 t2 WHERE EXISTS ( +SELECT * FROM dist_1 table_name_for_view WHERE EXISTS ( SELECT * FROM dist_1 t4 - WHERE t4.a = t2.a -); + WHERE t4.a = table_name_for_view.a); SELECT COUNT(*) FROM ref_1 t1 LEFT JOIN my_view_1 t3 USING (a); DEBUG: recursively planning right side of the left join since the outer side is a recurring rel DEBUG: recursively planning the distributed subquery since it is part of a distributed join node that is outer joined with a recurring rel -DEBUG: generating subplan XXX_1 for subquery SELECT t2.a, t2.b FROM recurring_outer_join.dist_1 t2 WHERE (EXISTS (SELECT t4.a, t4.b FROM recurring_outer_join.dist_1 t4 WHERE (t4.a OPERATOR(pg_catalog.=) t2.a))) +DEBUG: generating subplan XXX_1 for subquery SELECT a, b FROM recurring_outer_join.dist_1 table_name_for_view WHERE (EXISTS (SELECT t4.a, t4.b FROM recurring_outer_join.dist_1 t4 WHERE (t4.a OPERATOR(pg_catalog.=) table_name_for_view.a))) DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (recurring_outer_join.ref_1 t1 LEFT JOIN (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) t3 USING (a)) count --------------------------------------------------------------------- diff --git a/src/test/regress/expected/replicate_reference_tables_to_coordinator.out b/src/test/regress/expected/replicate_reference_tables_to_coordinator.out index 4dbeda307..975f501ef 100644 --- a/src/test/regress/expected/replicate_reference_tables_to_coordinator.out +++ b/src/test/regress/expected/replicate_reference_tables_to_coordinator.out @@ -396,10 +396,10 @@ DROP VIEW numbers_v, local_table_v; -- Joins between reference tables and materialized views are allowed to -- be planned to be executed locally. -- -CREATE MATERIALIZED VIEW numbers_v AS SELECT * FROM numbers WHERE a BETWEEN 1 AND 10; -NOTICE: executing the command locally: SELECT a FROM replicate_ref_to_coordinator.numbers_8000001 numbers WHERE ((a OPERATOR(pg_catalog.>=) 1) AND (a OPERATOR(pg_catalog.<=) 10)) +CREATE MATERIALIZED VIEW numbers_v AS SELECT * FROM numbers table_name_for_view WHERE a BETWEEN 1 AND 10; +NOTICE: executing the command locally: SELECT a FROM replicate_ref_to_coordinator.numbers_8000001 table_name_for_view WHERE ((a OPERATOR(pg_catalog.>=) 1) AND (a OPERATOR(pg_catalog.<=) 10)) REFRESH MATERIALIZED VIEW numbers_v; -NOTICE: executing the command locally: SELECT numbers.a FROM replicate_ref_to_coordinator.numbers_8000001 numbers WHERE ((numbers.a OPERATOR(pg_catalog.>=) 1) AND (numbers.a OPERATOR(pg_catalog.<=) 10)) +NOTICE: executing the command locally: SELECT a FROM replicate_ref_to_coordinator.numbers_8000001 table_name_for_view WHERE ((a OPERATOR(pg_catalog.>=) 1) AND (a OPERATOR(pg_catalog.<=) 10)) SELECT * FROM squares JOIN numbers_v ON squares.a = numbers_v.a ORDER BY 1; NOTICE: executing the command locally: SELECT squares.a, squares.b, numbers_v.a FROM (replicate_ref_to_coordinator.squares_8000000 squares JOIN replicate_ref_to_coordinator.numbers_v ON ((squares.a OPERATOR(pg_catalog.=) numbers_v.a))) ORDER BY squares.a a | b | a diff --git a/src/test/regress/expected/single_shard_table_udfs.out b/src/test/regress/expected/single_shard_table_udfs.out index d49027b60..26b3f82cc 100644 --- a/src/test/regress/expected/single_shard_table_udfs.out +++ b/src/test/regress/expected/single_shard_table_udfs.out @@ -121,7 +121,7 @@ FROM pg_dist_partition WHERE logicalrelid = 'null_dist_key_table'::regclass; SELECT column_name_to_column('null_dist_key_table', 'a'); column_name_to_column --------------------------------------------------------------------- - {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} + {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} (1 row) SELECT master_update_shard_statistics(shardid) diff --git a/src/test/regress/expected/start_stop_metadata_sync.out b/src/test/regress/expected/start_stop_metadata_sync.out index daa23aecd..ec9b0a034 100644 --- a/src/test/regress/expected/start_stop_metadata_sync.out +++ b/src/test/regress/expected/start_stop_metadata_sync.out @@ -158,12 +158,12 @@ SELECT * FROM test_matview; SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'events%' ORDER BY logicalrelid::text; logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - events | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f - events_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f - events_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f - events_replicated | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f - events_replicated_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f - events_replicated_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f + events | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f + events_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f + events_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980000 | s | f + events_replicated | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f + events_replicated_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f + events_replicated_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1} | 980001 | c | f (6 rows) SELECT count(*) > 0 FROM pg_dist_node; diff --git a/src/test/regress/expected/subquery_local_tables.out b/src/test/regress/expected/subquery_local_tables.out index ce5e844aa..42e691c87 100644 --- a/src/test/regress/expected/subquery_local_tables.out +++ b/src/test/regress/expected/subquery_local_tables.out @@ -126,43 +126,6 @@ DEBUG: push down of limit count: 3 4 (3 rows) --- subquery in FROM -> FROM -> WHERE -> WHERE should be replaced if --- it contains onle local tables --- Later the upper level query is also recursively planned due to LIMIT -SELECT user_id, array_length(events_table, 1) -FROM ( - SELECT user_id, array_agg(event ORDER BY time) AS events_table - FROM ( - SELECT - u.user_id, e.event_type::text AS event, e.time - FROM - users_table AS u, - events_table AS e - WHERE u.user_id = e.user_id AND - u.user_id IN - ( - SELECT - user_id - FROM - users_table - WHERE value_2 >= 5 - AND EXISTS (SELECT user_id FROM events_table_local WHERE event_type > 1 AND event_type <= 3 AND value_3 > 1) - AND NOT EXISTS (SELECT user_id FROM events_table WHERE event_type > 3 AND event_type <= 4 AND value_3 > 1 AND user_id = users_table.user_id) - LIMIT 5 - ) - ) t - GROUP BY user_id -) q -ORDER BY 2 DESC, 1; -DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM subquery_local_tables.events_table_local WHERE ((event_type OPERATOR(pg_catalog.>) 1) AND (event_type OPERATOR(pg_catalog.<=) 3) AND (value_3 OPERATOR(pg_catalog.>) (1)::double precision)) -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_2 for subquery SELECT user_id FROM public.users_table WHERE ((value_2 OPERATOR(pg_catalog.>=) 5) AND (EXISTS (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))) AND (NOT (EXISTS (SELECT events_table.user_id FROM public.events_table WHERE ((events_table.event_type OPERATOR(pg_catalog.>) 3) AND (events_table.event_type OPERATOR(pg_catalog.<=) 4) AND (events_table.value_3 OPERATOR(pg_catalog.>) (1)::double precision) AND (events_table.user_id OPERATOR(pg_catalog.=) users_table.user_id)))))) LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id, array_length(events_table, 1) AS array_length FROM (SELECT t.user_id, array_agg(t.event ORDER BY t."time") AS events_table FROM (SELECT u.user_id, (e.event_type)::text AS event, e."time" FROM public.users_table u, public.events_table e WHERE ((u.user_id OPERATOR(pg_catalog.=) e.user_id) AND (u.user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))))) t GROUP BY t.user_id) q ORDER BY (array_length(events_table, 1)) DESC, user_id - user_id | array_length ---------------------------------------------------------------------- - 5 | 364 -(1 row) - -- subquery (i.e., subquery_2) in WHERE->FROM should be replaced due to local tables SELECT user_id diff --git a/src/test/regress/expected/undistribute_table.out b/src/test/regress/expected/undistribute_table.out index 98b1d98f1..15d0e2695 100644 --- a/src/test/regress/expected/undistribute_table.out +++ b/src/test/regress/expected/undistribute_table.out @@ -304,18 +304,18 @@ SELECT create_distributed_table('view_table', 'a'); INSERT INTO view_table VALUES (1, 2, 3), (2, 4, 6), (3, 6, 9); CREATE SCHEMA another_schema; -CREATE VIEW undis_view1 AS SELECT a, b FROM view_table; -CREATE VIEW undis_view2 AS SELECT a, c FROM view_table; +CREATE VIEW undis_view1 AS SELECT a, b FROM view_table table_name_for_view; +CREATE VIEW undis_view2 AS SELECT a, c FROM view_table table_name_for_view; CREATE VIEW another_schema.undis_view3 AS SELECT b, c FROM undis_view1 JOIN undis_view2 ON undis_view1.a = undis_view2.a; SELECT schemaname, viewname, viewowner, definition FROM pg_views WHERE viewname LIKE 'undis\_view%' ORDER BY viewname; schemaname | viewname | viewowner | definition --------------------------------------------------------------------- - undistribute_table | undis_view1 | postgres | SELECT view_table.a, + - | | | view_table.b + - | | | FROM view_table; - undistribute_table | undis_view2 | postgres | SELECT view_table.a, + - | | | view_table.c + - | | | FROM view_table; + undistribute_table | undis_view1 | postgres | SELECT a, + + | | | b + + | | | FROM view_table table_name_for_view; + undistribute_table | undis_view2 | postgres | SELECT a, + + | | | c + + | | | FROM view_table table_name_for_view; another_schema | undis_view3 | postgres | SELECT undis_view1.b, + | | | undis_view2.c + | | | FROM (undis_view1 + @@ -348,12 +348,12 @@ NOTICE: renaming the new table to undistribute_table.view_table SELECT schemaname, viewname, viewowner, definition FROM pg_views WHERE viewname LIKE 'undis\_view%' ORDER BY viewname; schemaname | viewname | viewowner | definition --------------------------------------------------------------------- - undistribute_table | undis_view1 | postgres | SELECT view_table.a, + - | | | view_table.b + - | | | FROM view_table; - undistribute_table | undis_view2 | postgres | SELECT view_table.a, + - | | | view_table.c + - | | | FROM view_table; + undistribute_table | undis_view1 | postgres | SELECT a, + + | | | b + + | | | FROM view_table table_name_for_view; + undistribute_table | undis_view2 | postgres | SELECT a, + + | | | c + + | | | FROM view_table table_name_for_view; another_schema | undis_view3 | postgres | SELECT undis_view1.b, + | | | undis_view2.c + | | | FROM (undis_view1 + @@ -400,22 +400,6 @@ NOTICE: renaming the new table to undistribute_table.dist_type_table (1 row) --- test CREATE RULE with ON SELECT -CREATE TABLE rule_table_1 (a INT); -CREATE TABLE rule_table_2 (a INT); -SELECT create_distributed_table('rule_table_2', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -CREATE RULE "_RETURN" AS ON SELECT TO rule_table_1 DO INSTEAD SELECT * FROM rule_table_2; --- the CREATE RULE turns rule_table_1 into a view -ALTER EXTENSION plpgsql ADD VIEW rule_table_1; -NOTICE: Citus does not propagate adding/dropping member objects -HINT: You can add/drop the member objects on the workers as well. -SELECT undistribute_table('rule_table_2'); -ERROR: cannot alter table because an extension depends on it -- test CREATE RULE without ON SELECT CREATE TABLE rule_table_3 (a INT); CREATE TABLE rule_table_4 (a INT); @@ -444,9 +428,6 @@ NOTICE: renaming the new table to undistribute_table.rule_table_4 ALTER EXTENSION plpgsql DROP VIEW extension_view; NOTICE: Citus does not propagate adding/dropping member objects HINT: You can add/drop the member objects on the workers as well. -ALTER EXTENSION plpgsql DROP VIEW rule_table_1; -NOTICE: Citus does not propagate adding/dropping member objects -HINT: You can add/drop the member objects on the workers as well. ALTER EXTENSION plpgsql DROP TABLE rule_table_3; NOTICE: Citus does not propagate adding/dropping member objects HINT: You can add/drop the member objects on the workers as well. @@ -456,11 +437,9 @@ DETAIL: drop cascades to view undis_view1 drop cascades to view undis_view2 drop cascades to view another_schema.undis_view3 DROP SCHEMA undistribute_table, another_schema CASCADE; -NOTICE: drop cascades to 7 other objects +NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to table extension_table drop cascades to view extension_view drop cascades to table dist_type_table -drop cascades to table rule_table_2 -drop cascades to view rule_table_1 drop cascades to table rule_table_3 drop cascades to table rule_table_4 diff --git a/src/test/regress/expected/upgrade_basic_after.out b/src/test/regress/expected/upgrade_basic_after.out index e724b81d3..1bfbfc989 100644 --- a/src/test/regress/expected/upgrade_basic_after.out +++ b/src/test/regress/expected/upgrade_basic_after.out @@ -400,3 +400,12 @@ SELECT * FROM t_range ORDER BY id; (9 rows) ROLLBACK; +-- There is a difference in partkey Var representation between PG16 and older versions +-- Sanity check here that we can properly do column_to_column_name +SELECT column_to_column_name(logicalrelid, partkey) +FROM pg_dist_partition WHERE partkey IS NOT NULL ORDER BY 1 LIMIT 1; + column_to_column_name +--------------------------------------------------------------------- + a +(1 row) + diff --git a/src/test/regress/expected/upgrade_basic_before.out b/src/test/regress/expected/upgrade_basic_before.out index 880747a38..9abb6c806 100644 --- a/src/test/regress/expected/upgrade_basic_before.out +++ b/src/test/regress/expected/upgrade_basic_before.out @@ -65,3 +65,12 @@ UPDATE pg_dist_shard SET shardminvalue = '1', shardmaxvalue = '3' WHERE shardid UPDATE pg_dist_shard SET shardminvalue = '5', shardmaxvalue = '7' WHERE shardid = :shardid2; \copy t_range FROM STDIN with (DELIMITER ',') \copy t_range FROM STDIN with (DELIMITER ',') +-- There is a difference in partkey Var representation between PG16 and older versions +-- Sanity check here that we can properly do column_to_column_name +SELECT column_to_column_name(logicalrelid, partkey) +FROM pg_dist_partition WHERE partkey IS NOT NULL ORDER BY 1 LIMIT 1; + column_to_column_name +--------------------------------------------------------------------- + a +(1 row) + diff --git a/src/test/regress/expected/view_propagation.out b/src/test/regress/expected/view_propagation.out index d3d5bdb7b..5591a962f 100644 --- a/src/test/regress/expected/view_propagation.out +++ b/src/test/regress/expected/view_propagation.out @@ -316,13 +316,13 @@ UNION ALL employees e INNER JOIN reporting_line rl ON e.manager_id = rl.employee_id; -- Aliases are supported -CREATE VIEW aliased_opt_prop_view(alias_1, alias_2) AS SELECT * FROM view_table_6; +CREATE VIEW aliased_opt_prop_view(alias_1, alias_2) AS SELECT * FROM view_table_6 table_name_for_view; -- View options are supported CREATE VIEW opt_prop_view WITH(check_option=CASCADED, security_barrier=true) - AS SELECT * FROM view_table_6; + AS SELECT * FROM view_table_6 table_name_for_view; CREATE VIEW sep_opt_prop_view - AS SELECT * FROM view_table_6 + AS SELECT * FROM view_table_6 table_name_for_view WITH LOCAL CHECK OPTION; SELECT * FROM (SELECT pg_identify_object_as_address(classid, objid, objsubid) as obj_identifier from pg_catalog.pg_dist_object) as obj_identifiers where obj_identifier::text like '%opt_prop_view%' ORDER BY 1; obj_identifier @@ -335,27 +335,27 @@ SELECT * FROM (SELECT pg_identify_object_as_address(classid, objid, objsubid) as -- Check definitions and reltoptions of views are correct on workers \c - - - :worker_1_port SELECT definition FROM pg_views WHERE viewname = 'aliased_opt_prop_view'; - definition + definition --------------------------------------------------------------------- - SELECT view_table_6.id AS alias_1, + - view_table_6.val_1 AS alias_2 + - FROM view_prop_schema.view_table_6; + SELECT id AS alias_1, + + val_1 AS alias_2 + + FROM view_prop_schema.view_table_6 table_name_for_view; (1 row) SELECT definition FROM pg_views WHERE viewname = 'opt_prop_view'; - definition + definition --------------------------------------------------------------------- - SELECT view_table_6.id, + - view_table_6.val_1 + - FROM view_prop_schema.view_table_6; + SELECT id, + + val_1 + + FROM view_prop_schema.view_table_6 table_name_for_view; (1 row) SELECT definition FROM pg_views WHERE viewname = 'sep_opt_prop_view'; - definition + definition --------------------------------------------------------------------- - SELECT view_table_6.id, + - view_table_6.val_1 + - FROM view_prop_schema.view_table_6; + SELECT id, + + val_1 + + FROM view_prop_schema.view_table_6 table_name_for_view; (1 row) SELECT relname, reloptions @@ -444,7 +444,7 @@ SELECT create_distributed_table('alter_view_table','id'); (1 row) -CREATE VIEW alter_view_1 AS SELECT * FROM alter_view_table; +CREATE VIEW alter_view_1 AS SELECT * FROM alter_view_table table_name_for_view; -- Set/drop default value is not supported by Citus ALTER VIEW alter_view_1 ALTER COLUMN val1 SET DEFAULT random()::text; ERROR: Citus doesn't support setting or resetting default values for a column of view @@ -465,11 +465,11 @@ ALTER TABLE alter_view_1 SET (check_option=cascaded, security_barrier); ALTER TABLE alter_view_1 SET (check_option=cascaded, security_barrier = true); -- Check the definition on both coordinator and worker node SELECT definition FROM pg_views WHERE viewname = 'alter_view_1'; - definition + definition --------------------------------------------------------------------- - SELECT alter_view_table.id,+ - alter_view_table.val1 + - FROM alter_view_table; + SELECT id, + + val1 + + FROM alter_view_table table_name_for_view; (1 row) SELECT relname, reloptions @@ -482,11 +482,11 @@ WHERE oid = 'view_prop_schema.alter_view_1'::regclass::oid; \c - - - :worker_1_port SELECT definition FROM pg_views WHERE viewname = 'alter_view_1'; - definition + definition --------------------------------------------------------------------- - SELECT alter_view_table.id, + - alter_view_table.val1 + - FROM view_prop_schema.alter_view_table; + SELECT id, + + val1 + + FROM view_prop_schema.alter_view_table table_name_for_view; (1 row) SELECT relname, reloptions diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 19040f1e0..65a272566 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -65,6 +65,7 @@ test: pg13 pg12 test: pg14 test: pg15 test: pg15_jsonpath detect_conn_close +test: pg16 test: drop_column_partitioned_table test: tableam diff --git a/src/test/regress/spec/isolation_add_remove_node.spec b/src/test/regress/spec/isolation_add_remove_node.spec index 014592d25..58c5c1517 100644 --- a/src/test/regress/spec/isolation_add_remove_node.spec +++ b/src/test/regress/spec/isolation_add_remove_node.spec @@ -45,6 +45,12 @@ step "s1-disable-node-1" SELECT public.wait_until_metadata_sync(); } +step "s1-disable-node-2" +{ + SELECT 1 FROM master_disable_node('localhost', 57638); + SELECT public.wait_until_metadata_sync(); +} + step "s1-remove-node-1" { SELECT * FROM master_remove_node('localhost', 57637); @@ -88,6 +94,16 @@ step "s2-disable-node-1" SELECT public.wait_until_metadata_sync(); } +step "s2-disable-node-2" +{ + SELECT 1 FROM master_disable_node('localhost', 57638); +} + +step "s2-wait-metadata-sync" +{ + SELECT public.wait_until_metadata_sync(); +} + step "s2-remove-node-1" { SELECT * FROM master_remove_node('localhost', 57637); @@ -135,4 +151,4 @@ permutation "s1-add-inactive-1" "s1-begin" "s1-disable-node-1" "s2-activate-node permutation "s1-add-inactive-1" "s1-begin" "s1-activate-node-1" "s2-disable-node-1" "s1-commit" "s1-show-nodes" // disable an active node from 2 transactions, one aborts -permutation "s1-add-node-1" "s1-begin" "s1-disable-node-1" "s2-disable-node-1" "s1-abort" "s1-show-nodes" +permutation "s1-add-node-1" "s1-add-node-2" "s1-begin" "s1-disable-node-2" "s2-disable-node-2" "s1-abort" "s2-wait-metadata-sync" "s1-show-nodes" diff --git a/src/test/regress/sql/citus_local_tables_mx.sql b/src/test/regress/sql/citus_local_tables_mx.sql index 2bb79a802..2f7c76d6e 100644 --- a/src/test/regress/sql/citus_local_tables_mx.sql +++ b/src/test/regress/sql/citus_local_tables_mx.sql @@ -473,7 +473,7 @@ select run_command_on_workers($$SELECT count(*)=0 from citus_local_tables_mx.v10 select run_command_on_workers($$SELECT count(*)=0 from citus_local_tables_mx.v102$$); CREATE TABLE loc_tb_2 (a int); -CREATE VIEW v104 AS SELECT * from loc_tb_2; +CREATE VIEW v104 AS SELECT * from loc_tb_2 table_name_for_view; SET client_min_messages TO DEBUG1; -- verify the CREATE command for the view is generated correctly diff --git a/src/test/regress/sql/columnar_chunk_filtering.sql b/src/test/regress/sql/columnar_chunk_filtering.sql index b8b2b411d..d37a8d8b6 100644 --- a/src/test/regress/sql/columnar_chunk_filtering.sql +++ b/src/test/regress/sql/columnar_chunk_filtering.sql @@ -1,6 +1,10 @@ -- -- Test chunk filtering in columnar using min/max values in stripe skip lists. -- +-- It has an alternative test output file +-- because PG16 changed the order of some Filters in EXPLAIN +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/2489d76c4906f4461a364ca8ad7e0751ead8aa0d -- diff --git a/src/test/regress/sql/columnar_memory.sql b/src/test/regress/sql/columnar_memory.sql index 21bab57f5..5f29eb1e3 100644 --- a/src/test/regress/sql/columnar_memory.sql +++ b/src/test/regress/sql/columnar_memory.sql @@ -77,10 +77,10 @@ SELECT CASE WHEN 1.0 * TopMemoryContext / :top_post BETWEEN 0.98 AND 1.03 THEN 1 FROM columnar_test_helpers.columnar_store_memory_stats(); -- before this change, max mem usage while executing inserts was 28MB and --- with this change it's less than 8MB. +-- with this change it's less than 9MB. SELECT - (SELECT max(memusage) < 8 * 1024 * 1024 FROM t WHERE tag='large batch') AS large_batch_ok, - (SELECT max(memusage) < 8 * 1024 * 1024 FROM t WHERE tag='first batch') AS first_batch_ok; + (SELECT max(memusage) < 9 * 1024 * 1024 FROM t WHERE tag='large batch') AS large_batch_ok, + (SELECT max(memusage) < 9 * 1024 * 1024 FROM t WHERE tag='first batch') AS first_batch_ok; \x diff --git a/src/test/regress/sql/columnar_paths.sql b/src/test/regress/sql/columnar_paths.sql index 92ffa7d66..3c92d4a21 100644 --- a/src/test/regress/sql/columnar_paths.sql +++ b/src/test/regress/sql/columnar_paths.sql @@ -193,7 +193,7 @@ WHERE w2.a = 123; EXPLAIN (COSTS OFF) SELECT sub_1.b, sub_2.a, sub_3.avg FROM - (SELECT b FROM full_correlated WHERE (a > 2) GROUP BY b HAVING count(DISTINCT a) > 0 ORDER BY 1 DESC LIMIT 5) AS sub_1, + (SELECT b FROM full_correlated WHERE (a > 2) GROUP BY b ORDER BY 1 DESC LIMIT 5) AS sub_1, (SELECT a FROM full_correlated WHERE (a > 10) GROUP BY a HAVING count(DISTINCT a) >= 1 ORDER BY 1 DESC LIMIT 3) AS sub_2, (SELECT avg(a) AS AVG FROM full_correlated WHERE (a > 2) GROUP BY a HAVING sum(a) > 10 ORDER BY (sum(d) - avg(a) - COALESCE(array_upper(ARRAY[max(a)],1) * 5, 0)) DESC LIMIT 3) AS sub_3 WHERE sub_2.a < sub_1.b::integer diff --git a/src/test/regress/sql/create_role_propagation.sql b/src/test/regress/sql/create_role_propagation.sql index ceda9f10c..027e4f72e 100644 --- a/src/test/regress/sql/create_role_propagation.sql +++ b/src/test/regress/sql/create_role_propagation.sql @@ -117,7 +117,7 @@ GRANT non_dist_role_4 TO dist_role_4; SELECT 1 FROM master_add_node('localhost', :worker_2_port); -SELECT roleid::regrole::text AS role, member::regrole::text, grantor::regrole::text, admin_option FROM pg_auth_members WHERE roleid::regrole::text LIKE '%dist\_%' ORDER BY 1, 2; +SELECT roleid::regrole::text AS role, member::regrole::text, (grantor::regrole::text IN ('postgres', 'non_dist_role_1', 'dist_role_1')) AS grantor, admin_option FROM pg_auth_members WHERE roleid::regrole::text LIKE '%dist\_%' ORDER BY 1, 2; SELECT objid::regrole FROM pg_catalog.pg_dist_object WHERE classid='pg_authid'::regclass::oid AND objid::regrole::text LIKE '%dist\_%' ORDER BY 1; \c - - - :worker_1_port diff --git a/src/test/regress/sql/global_cancel.sql b/src/test/regress/sql/global_cancel.sql index 4a6157489..848c3b01a 100644 --- a/src/test/regress/sql/global_cancel.sql +++ b/src/test/regress/sql/global_cancel.sql @@ -47,9 +47,12 @@ RESET client_min_messages; SELECT pg_typeof(:maintenance_daemon_gpid); +\set VERBOSITY terse + SELECT pg_cancel_backend(:maintenance_daemon_gpid); SELECT pg_terminate_backend(:maintenance_daemon_gpid); +\set VERBOSITY default -- we can cancel our own backend SELECT pg_cancel_backend(citus_backend_gpid()); diff --git a/src/test/regress/sql/insert_select_repartition.sql b/src/test/regress/sql/insert_select_repartition.sql index 4d13a83f4..30d77f5b8 100644 --- a/src/test/regress/sql/insert_select_repartition.sql +++ b/src/test/regress/sql/insert_select_repartition.sql @@ -611,7 +611,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, @@ -625,7 +625,7 @@ SELECT c1, c2, c3, c4, -1::float AS c5, sum(cardinality), sum(sum) FROM source_table -GROUP BY c1, c2, c3, c4, c5, c6 +GROUP BY c1, c2, c3, c4, c6 ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, diff --git a/src/test/regress/sql/local_dist_join_mixed.sql b/src/test/regress/sql/local_dist_join_mixed.sql index b07da2fc8..c6eb53d4e 100644 --- a/src/test/regress/sql/local_dist_join_mixed.sql +++ b/src/test/regress/sql/local_dist_join_mixed.sql @@ -78,14 +78,13 @@ SELECT count(*) FROM distributed JOIN unlogged_local USING (id); CREATE MATERIALIZED VIEW mat_view AS SELECT * FROM local; SELECT count(*) FROM distributed JOIN mat_view USING (id); -CREATE VIEW local_regular_view AS SELECT * FROM local; +CREATE VIEW local_regular_view AS SELECT * FROM local table_name_for_view; CREATE VIEW dist_regular_view AS SELECT * FROM distributed; SELECT count(*) FROM distributed JOIN local_regular_view USING (id); SELECT count(*) FROM local JOIN dist_regular_view USING (id); SELECT count(*) FROM dist_regular_view JOIN local_regular_view USING (id); - -- join alias/table alias SELECT COUNT(*) FROM (distributed JOIN local USING (id)) AS t(a,b,c,d) ORDER BY d,c,a,b LIMIT 3; SELECT COUNT(*) FROM (distributed d1(x,y,y1) JOIN local l1(x,t) USING (x)) AS t(a,b,c,d) ORDER BY d,c,a,b LIMIT 3; diff --git a/src/test/regress/sql/local_table_join.sql b/src/test/regress/sql/local_table_join.sql index 8d0d7d332..393b15378 100644 --- a/src/test/regress/sql/local_table_join.sql +++ b/src/test/regress/sql/local_table_join.sql @@ -362,9 +362,6 @@ select typdefault from ( select a from tbl where typdefault > 'a' limit 1) as subq_0 - where ( - select true as bool from pg_catalog.pg_am limit 1 - ) ) as subq_1 ) as subq_2; @@ -379,9 +376,6 @@ select typdefault from ( select a from tbl where typdefault > 'a' limit 1) as subq_0 - where ( - select true as bool from pg_catalog.pg_am limit 1 - ) ) as subq_1 ) as subq_2; diff --git a/src/test/regress/sql/metadata_sync_helpers.sql b/src/test/regress/sql/metadata_sync_helpers.sql index 1c5d5b15d..a4044bab3 100644 --- a/src/test/regress/sql/metadata_sync_helpers.sql +++ b/src/test/regress/sql/metadata_sync_helpers.sql @@ -798,8 +798,19 @@ BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED; SET application_name to 'citus_internal gpid=10000000001'; -- with an ugly trick, update the vartype of table from int to bigint -- so that making two tables colocated fails - UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1}' + + -- include varnullingrels for PG16 + SHOW server_version \gset + SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 + \gset + \if :server_version_ge_16 + UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varnullingrels (b) :varlevelsup 1 :varnoold 1 :varoattno 1 :location -1}' WHERE logicalrelid = 'test_2'::regclass; + \else + UPDATE pg_dist_partition SET partkey = '{VAR :varno 1 :varattno 1 :vartype 20 :vartypmod -1 :varcollid 0 :varlevelsup 1 :varnoold 1 :varoattno 1 :location -1}' + WHERE logicalrelid = 'test_2'::regclass; + \endif + SELECT citus_internal_update_relation_colocation('test_2'::regclass, 251); ROLLBACK; diff --git a/src/test/regress/sql/multi_complex_count_distinct.sql b/src/test/regress/sql/multi_complex_count_distinct.sql index 9957d0959..0e06fc0c8 100644 --- a/src/test/regress/sql/multi_complex_count_distinct.sql +++ b/src/test/regress/sql/multi_complex_count_distinct.sql @@ -1,7 +1,13 @@ -- -- COMPLEX_COUNT_DISTINCT -- - +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; SET citus.next_shard_id TO 240000; SET citus.shard_count TO 8; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index dd4615434..7fa75c8be 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -1,6 +1,13 @@ -- -- MULTI_EXPLAIN -- +-- This test file has an alternative output because of the following in PG16: +-- https://github.com/postgres/postgres/commit/1349d2790bf48a4de072931c722f39337e72055e +-- https://github.com/postgres/postgres/commit/f4c7c410ee4a7baa06f51ebb8d5333c169691dd3 +-- The alternative output can be deleted when we drop support for PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16; SET citus.next_shard_id TO 570000; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index e5cf29833..1726c260f 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -458,7 +458,7 @@ DELETE FROM pg_dist_shard WHERE shardid = 1; CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id); CREATE TABLE orders_2020_07_01 PARTITION OF e_transactions FOR VALUES IN (1,2,3); -INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's'); +INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's'); SELECT (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, diff --git a/src/test/regress/sql/multi_hash_pruning.sql b/src/test/regress/sql/multi_hash_pruning.sql index df432ca90..ef6da8638 100644 --- a/src/test/regress/sql/multi_hash_pruning.sql +++ b/src/test/regress/sql/multi_hash_pruning.sql @@ -336,12 +336,14 @@ FULL OUTER JOIN lineitem_hash_partitioned ON (o_orderkey = l_orderkey) WHERE o_orderkey IN (1, 2) OR l_orderkey IN (2, 3); +SELECT public.coordinator_plan($Q$ EXPLAIN (COSTS OFF) SELECT count(*) FROM orders_hash_partitioned FULL OUTER JOIN lineitem_hash_partitioned ON (o_orderkey = l_orderkey) WHERE o_orderkey IN (1, 2) AND l_orderkey IN (2, 3); +$Q$); SET citus.task_executor_type TO DEFAULT; diff --git a/src/test/regress/sql/multi_having_pushdown.sql b/src/test/regress/sql/multi_having_pushdown.sql index 497fd8cc3..48475099d 100644 --- a/src/test/regress/sql/multi_having_pushdown.sql +++ b/src/test/regress/sql/multi_having_pushdown.sql @@ -43,7 +43,7 @@ EXPLAIN (COSTS FALSE) SELECT sum(l_extendedprice * l_discount) as revenue FROM lineitem_hash, orders_hash WHERE o_orderkey = l_orderkey - GROUP BY l_orderkey, o_orderkey, l_shipmode HAVING sum(l_quantity) > 24 + GROUP BY l_orderkey, l_shipmode HAVING sum(l_quantity) > 24 ORDER BY 1 DESC LIMIT 3; EXPLAIN (COSTS FALSE) diff --git a/src/test/regress/sql/multi_mx_hide_shard_names.sql b/src/test/regress/sql/multi_mx_hide_shard_names.sql index 9d2536973..e5213a41b 100644 --- a/src/test/regress/sql/multi_mx_hide_shard_names.sql +++ b/src/test/regress/sql/multi_mx_hide_shard_names.sql @@ -226,14 +226,32 @@ RESET citus.enable_metadata_sync; -- the shards and indexes do not show up SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; +-- PG16 added one more backend type B_STANDALONE_BACKEND +-- and also alphabetized the backend types, hence the orders changed +-- Relevant PG commit: +-- https://github.com/postgres/postgres/commit/0c679464a837079acc75ff1d45eaa83f79e05690 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset + +\if :server_version_ge_16 +SELECT 4 AS client_backend \gset +SELECT 5 AS bgworker \gset +SELECT 12 AS walsender \gset +\else +SELECT 3 AS client_backend \gset +SELECT 4 AS bgworker \gset +SELECT 9 AS walsender \gset +\endif + -- say, we set it to bgworker -- the shards and indexes do not show up -SELECT set_backend_type(4); +SELECT set_backend_type(:bgworker); SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; -- or, we set it to walsender -- the shards and indexes do not show up -SELECT set_backend_type(9); +SELECT set_backend_type(:walsender); SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; -- unless the application name starts with citus_shard @@ -242,7 +260,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name RESET application_name; -- but, client backends to see the shards -SELECT set_backend_type(3); +SELECT set_backend_type(:client_backend); SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; diff --git a/src/test/regress/sql/multi_orderby_limit_pushdown.sql b/src/test/regress/sql/multi_orderby_limit_pushdown.sql index 821c0130a..7b35d82eb 100644 --- a/src/test/regress/sql/multi_orderby_limit_pushdown.sql +++ b/src/test/regress/sql/multi_orderby_limit_pushdown.sql @@ -177,7 +177,7 @@ ORDER BY 2, AVG(ut.value_1), 1 DESC LIMIT 2; EXPLAIN (COSTS OFF) -SELECT ut.user_id, count(DISTINCT ut.value_2) +SELECT ut.user_id, avg(ut.value_2) FROM users_table ut, events_table et WHERE ut.user_id = et.user_id and et.value_2 < 5 GROUP BY ut.user_id diff --git a/src/test/regress/sql/multi_select_distinct.sql b/src/test/regress/sql/multi_select_distinct.sql index c3ba20cf1..597076199 100644 --- a/src/test/regress/sql/multi_select_distinct.sql +++ b/src/test/regress/sql/multi_select_distinct.sql @@ -303,7 +303,7 @@ SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) EXPLAIN (COSTS FALSE) SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) FROM lineitem_hash_part - GROUP BY l_orderkey + GROUP BY l_orderkey, l_partkey, l_shipmode ORDER BY 1,2; -- check the plan if the hash aggreate is disabled. We expect to see sort + unique @@ -312,7 +312,7 @@ SET enable_hashagg TO off; EXPLAIN (COSTS FALSE) SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) FROM lineitem_hash_part - GROUP BY l_orderkey + GROUP BY l_orderkey, l_partkey, l_shipmode ORDER BY 1,2; SET enable_hashagg TO on; diff --git a/src/test/regress/sql/multi_subquery.sql b/src/test/regress/sql/multi_subquery.sql index 68265606a..e5d8aa17c 100644 --- a/src/test/regress/sql/multi_subquery.sql +++ b/src/test/regress/sql/multi_subquery.sql @@ -676,7 +676,7 @@ EXPLAIN (COSTS OFF) SELECT count(*) FROM keyval1 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyval2 GROUP BY key ORDER BY 1 DESC LIMIT 1); EXPLAIN (COSTS OFF) -SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 GROUP BY key HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 GROUP BY key ORDER BY 1 DESC LIMIT 1); +SELECT count(*) FROM keyval1 k1 WHERE k1.key = 2 HAVING sum(value) > (SELECT sum(value) FROM keyval2 k2 WHERE k2.key = 2 ORDER BY 1 DESC LIMIT 1); -- Simple join subquery pushdown SELECT diff --git a/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql b/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql index a3dd9c06e..fc1bb5c17 100644 --- a/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql +++ b/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql @@ -132,7 +132,7 @@ SELECT FROM users_table RIGHT JOIN users_reference_table USING (user_id) WHERE - users_table.value_2 IN + users_reference_table.value_2 IN (SELECT value_2 FROM diff --git a/src/test/regress/sql/multi_view.sql b/src/test/regress/sql/multi_view.sql index d80ed5c97..889dde818 100644 --- a/src/test/regress/sql/multi_view.sql +++ b/src/test/regress/sql/multi_view.sql @@ -37,7 +37,7 @@ CREATE VIEW priority_lineitem AS SELECT li.* FROM lineitem_hash_part li JOIN pri SELECT l_orderkey, count(*) FROM priority_lineitem GROUP BY 1 ORDER BY 2 DESC, 1 LIMIT 5; -CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part WHERE l_shipmode = 'AIR'; +CREATE VIEW air_shipped_lineitems AS SELECT * FROM lineitem_hash_part table_name_for_view WHERE l_shipmode = 'AIR'; -- join between view and table SELECT count(*) FROM orders_hash_part join air_shipped_lineitems ON (o_orderkey = l_orderkey); diff --git a/src/test/regress/sql/non_colocated_subquery_joins.sql b/src/test/regress/sql/non_colocated_subquery_joins.sql index bde8f5b0a..a74b8e16e 100644 --- a/src/test/regress/sql/non_colocated_subquery_joins.sql +++ b/src/test/regress/sql/non_colocated_subquery_joins.sql @@ -792,7 +792,7 @@ SET citus.shard_replication_factor TO 1; SELECT create_distributed_table('table2','tenant_id'); SELECT create_distributed_table('table1','tenant_id'); -CREATE VIEW table1_view AS SELECT * from table1 where id < 100; +CREATE VIEW table1_view AS SELECT * from table1 table_name_for_view where id < 100; -- all of the above queries are non-colocated subquery joins -- because the views are replaced with subqueries diff --git a/src/test/regress/sql/pg12.sql b/src/test/regress/sql/pg12.sql index a86dbbb42..831ce40bb 100644 --- a/src/test/regress/sql/pg12.sql +++ b/src/test/regress/sql/pg12.sql @@ -242,11 +242,13 @@ COMMIT; SELECT DISTINCT y FROM test; -- non deterministic collations +SET client_min_messages TO WARNING; CREATE COLLATION test_pg12.case_insensitive ( provider = icu, locale = '@colStrength=secondary', deterministic = false ); +RESET client_min_messages; CREATE TABLE col_test ( id int, diff --git a/src/test/regress/sql/pg16.sql b/src/test/regress/sql/pg16.sql new file mode 100644 index 000000000..29638ac1c --- /dev/null +++ b/src/test/regress/sql/pg16.sql @@ -0,0 +1,50 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q +\endif + +CREATE SCHEMA pg16; +SET search_path TO pg16; +SET citus.next_shard_id TO 950000; +SET citus.shard_count TO 1; +SET citus.shard_replication_factor TO 1; + +-- test the new vacuum and analyze options +-- Relevant PG commits: +-- https://github.com/postgres/postgres/commit/1cbbee03385763b066ae3961fc61f2cd01a0d0d7 +-- https://github.com/postgres/postgres/commit/4211fbd8413b26e0abedbe4338aa7cda2cd469b4 +-- https://github.com/postgres/postgres/commit/a46a7011b27188af526047a111969f257aaf4db8 + +CREATE TABLE t1 (a int); +SELECT create_distributed_table('t1','a'); +SET citus.log_remote_commands TO ON; + +VACUUM (PROCESS_MAIN FALSE) t1; +VACUUM (PROCESS_MAIN FALSE, PROCESS_TOAST FALSE) t1; +VACUUM (PROCESS_MAIN TRUE) t1; +VACUUM (PROCESS_MAIN FALSE, FULL) t1; +VACUUM (SKIP_DATABASE_STATS) t1; +VACUUM (ONLY_DATABASE_STATS) t1; +VACUUM (BUFFER_USAGE_LIMIT '512 kB') t1; +VACUUM (BUFFER_USAGE_LIMIT 0) t1; +VACUUM (BUFFER_USAGE_LIMIT 16777220) t1; +VACUUM (BUFFER_USAGE_LIMIT -1) t1; +VACUUM (BUFFER_USAGE_LIMIT 'test') t1; +ANALYZE (BUFFER_USAGE_LIMIT '512 kB') t1; +ANALYZE (BUFFER_USAGE_LIMIT 0) t1; + +SET citus.log_remote_commands TO OFF; + +-- only verifying it works and not printing log +-- remote commands because it can be flaky +VACUUM (ONLY_DATABASE_STATS); + +\set VERBOSITY terse +SET client_min_messages TO ERROR; +DROP SCHEMA pg16 CASCADE; diff --git a/src/test/regress/sql/recurring_outer_join.sql b/src/test/regress/sql/recurring_outer_join.sql index 595d734ec..d33309817 100644 --- a/src/test/regress/sql/recurring_outer_join.sql +++ b/src/test/regress/sql/recurring_outer_join.sql @@ -612,10 +612,9 @@ USING (a); -- same test using a view, can be recursively planned CREATE VIEW my_view_1 AS -SELECT * FROM dist_1 t2 WHERE EXISTS ( +SELECT * FROM dist_1 table_name_for_view WHERE EXISTS ( SELECT * FROM dist_1 t4 - WHERE t4.a = t2.a -); + WHERE t4.a = table_name_for_view.a); SELECT COUNT(*) FROM ref_1 t1 LEFT JOIN diff --git a/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql b/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql index cc5e74cd9..284fdfc66 100644 --- a/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql +++ b/src/test/regress/sql/replicate_reference_tables_to_coordinator.sql @@ -192,7 +192,7 @@ DROP VIEW numbers_v, local_table_v; -- Joins between reference tables and materialized views are allowed to -- be planned to be executed locally. -- -CREATE MATERIALIZED VIEW numbers_v AS SELECT * FROM numbers WHERE a BETWEEN 1 AND 10; +CREATE MATERIALIZED VIEW numbers_v AS SELECT * FROM numbers table_name_for_view WHERE a BETWEEN 1 AND 10; REFRESH MATERIALIZED VIEW numbers_v; SELECT * FROM squares JOIN numbers_v ON squares.a = numbers_v.a ORDER BY 1; diff --git a/src/test/regress/sql/subquery_local_tables.sql b/src/test/regress/sql/subquery_local_tables.sql index 7f52120f1..65fc6a5b8 100644 --- a/src/test/regress/sql/subquery_local_tables.sql +++ b/src/test/regress/sql/subquery_local_tables.sql @@ -95,37 +95,6 @@ FROM LIMIT 3; --- subquery in FROM -> FROM -> WHERE -> WHERE should be replaced if --- it contains onle local tables --- Later the upper level query is also recursively planned due to LIMIT -SELECT user_id, array_length(events_table, 1) -FROM ( - SELECT user_id, array_agg(event ORDER BY time) AS events_table - FROM ( - SELECT - u.user_id, e.event_type::text AS event, e.time - FROM - users_table AS u, - events_table AS e - WHERE u.user_id = e.user_id AND - u.user_id IN - ( - SELECT - user_id - FROM - users_table - WHERE value_2 >= 5 - AND EXISTS (SELECT user_id FROM events_table_local WHERE event_type > 1 AND event_type <= 3 AND value_3 > 1) - AND NOT EXISTS (SELECT user_id FROM events_table WHERE event_type > 3 AND event_type <= 4 AND value_3 > 1 AND user_id = users_table.user_id) - LIMIT 5 - ) - ) t - GROUP BY user_id -) q -ORDER BY 2 DESC, 1; - - - -- subquery (i.e., subquery_2) in WHERE->FROM should be replaced due to local tables SELECT user_id diff --git a/src/test/regress/sql/undistribute_table.sql b/src/test/regress/sql/undistribute_table.sql index 1703440c0..737f5a0f9 100644 --- a/src/test/regress/sql/undistribute_table.sql +++ b/src/test/regress/sql/undistribute_table.sql @@ -105,8 +105,8 @@ INSERT INTO view_table VALUES (1, 2, 3), (2, 4, 6), (3, 6, 9); CREATE SCHEMA another_schema; -CREATE VIEW undis_view1 AS SELECT a, b FROM view_table; -CREATE VIEW undis_view2 AS SELECT a, c FROM view_table; +CREATE VIEW undis_view1 AS SELECT a, b FROM view_table table_name_for_view; +CREATE VIEW undis_view2 AS SELECT a, c FROM view_table table_name_for_view; CREATE VIEW another_schema.undis_view3 AS SELECT b, c FROM undis_view1 JOIN undis_view2 ON undis_view1.a = undis_view2.a; SELECT schemaname, viewname, viewowner, definition FROM pg_views WHERE viewname LIKE 'undis\_view%' ORDER BY viewname; @@ -131,18 +131,6 @@ SELECT create_distributed_table('dist_type_table', 'a'); SELECT undistribute_table('dist_type_table'); --- test CREATE RULE with ON SELECT -CREATE TABLE rule_table_1 (a INT); -CREATE TABLE rule_table_2 (a INT); -SELECT create_distributed_table('rule_table_2', 'a'); - -CREATE RULE "_RETURN" AS ON SELECT TO rule_table_1 DO INSTEAD SELECT * FROM rule_table_2; - --- the CREATE RULE turns rule_table_1 into a view -ALTER EXTENSION plpgsql ADD VIEW rule_table_1; - -SELECT undistribute_table('rule_table_2'); - -- test CREATE RULE without ON SELECT CREATE TABLE rule_table_3 (a INT); CREATE TABLE rule_table_4 (a INT); @@ -155,7 +143,6 @@ ALTER EXTENSION plpgsql ADD TABLE rule_table_3; SELECT undistribute_table('rule_table_4'); ALTER EXTENSION plpgsql DROP VIEW extension_view; -ALTER EXTENSION plpgsql DROP VIEW rule_table_1; ALTER EXTENSION plpgsql DROP TABLE rule_table_3; DROP TABLE view_table CASCADE; diff --git a/src/test/regress/sql/upgrade_basic_after.sql b/src/test/regress/sql/upgrade_basic_after.sql index f2fc61769..b40501a1e 100644 --- a/src/test/regress/sql/upgrade_basic_after.sql +++ b/src/test/regress/sql/upgrade_basic_after.sql @@ -143,3 +143,8 @@ SELECT * FROM t_range ORDER BY id; ROLLBACK; + +-- There is a difference in partkey Var representation between PG16 and older versions +-- Sanity check here that we can properly do column_to_column_name +SELECT column_to_column_name(logicalrelid, partkey) +FROM pg_dist_partition WHERE partkey IS NOT NULL ORDER BY 1 LIMIT 1; diff --git a/src/test/regress/sql/upgrade_basic_before.sql b/src/test/regress/sql/upgrade_basic_before.sql index e223ac965..868483264 100644 --- a/src/test/regress/sql/upgrade_basic_before.sql +++ b/src/test/regress/sql/upgrade_basic_before.sql @@ -51,3 +51,8 @@ UPDATE pg_dist_shard SET shardminvalue = '5', shardmaxvalue = '7' WHERE shardid 6,3 7,4 \. + +-- There is a difference in partkey Var representation between PG16 and older versions +-- Sanity check here that we can properly do column_to_column_name +SELECT column_to_column_name(logicalrelid, partkey) +FROM pg_dist_partition WHERE partkey IS NOT NULL ORDER BY 1 LIMIT 1; diff --git a/src/test/regress/sql/view_propagation.sql b/src/test/regress/sql/view_propagation.sql index 44bbbf7b0..f0d63da85 100644 --- a/src/test/regress/sql/view_propagation.sql +++ b/src/test/regress/sql/view_propagation.sql @@ -207,15 +207,15 @@ UNION ALL INNER JOIN reporting_line rl ON e.manager_id = rl.employee_id; -- Aliases are supported -CREATE VIEW aliased_opt_prop_view(alias_1, alias_2) AS SELECT * FROM view_table_6; +CREATE VIEW aliased_opt_prop_view(alias_1, alias_2) AS SELECT * FROM view_table_6 table_name_for_view; -- View options are supported CREATE VIEW opt_prop_view WITH(check_option=CASCADED, security_barrier=true) - AS SELECT * FROM view_table_6; + AS SELECT * FROM view_table_6 table_name_for_view; CREATE VIEW sep_opt_prop_view - AS SELECT * FROM view_table_6 + AS SELECT * FROM view_table_6 table_name_for_view WITH LOCAL CHECK OPTION; SELECT * FROM (SELECT pg_identify_object_as_address(classid, objid, objsubid) as obj_identifier from pg_catalog.pg_dist_object) as obj_identifiers where obj_identifier::text like '%opt_prop_view%' ORDER BY 1; @@ -273,7 +273,7 @@ CREATE OR REPLACE VIEW view_for_unsup_commands AS SELECT id FROM table_to_test_u CREATE TABLE alter_view_table(id int, val1 text); SELECT create_distributed_table('alter_view_table','id'); -CREATE VIEW alter_view_1 AS SELECT * FROM alter_view_table; +CREATE VIEW alter_view_1 AS SELECT * FROM alter_view_table table_name_for_view; -- Set/drop default value is not supported by Citus ALTER VIEW alter_view_1 ALTER COLUMN val1 SET DEFAULT random()::text;