diff --git a/.gitattributes b/.gitattributes index 9f2989b09..01924741d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -30,4 +30,5 @@ src/backend/distributed/utils/citus_read.c -citus-style src/backend/distributed/utils/citus_readfuncs_95.c -citus-style src/backend/distributed/utils/ruleutils_95.c -citus-style src/backend/distributed/utils/ruleutils_96.c -citus-style +src/backend/distributed/utils/ruleutils_10.c -citus-style src/include/distributed/citus_nodes.h -citus-style diff --git a/.travis.yml b/.travis.yml index 91a9d18e6..7a0b35bba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ env: matrix: - PGVERSION=9.5 - PGVERSION=9.6 + - PGVERSION=10 before_install: - git clone -b v0.6.2 --depth 1 https://github.com/citusdata/tools.git - sudo make -C tools install @@ -27,7 +28,7 @@ install: - install_pg - install_custom_pg # download and install HLL manually, as custom builds won't satisfy deps - - apt-get download "postgresql-${PGVERSION}-hll=2.10.1.citus-1" && sudo dpkg --force-confold --force-confdef --force-all -i *hll*.deb + - apt-get download "postgresql-${PGVERSION}-hll=2.10.2.citus-1" && sudo dpkg --force-confold --force-confdef --force-all -i *hll*.deb before_script: citus_indent --quiet --check script: CFLAGS=-Werror pg_travis_multi_test check after_success: diff --git a/configure b/configure index c75bbe0ae..27f434469 100755 --- a/configure +++ b/configure @@ -1996,13 +1996,16 @@ fi # check we're building against a supported version of PostgreSQL citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) version_num=$(echo "$citusac_pg_config_version"| - $SED -e 's/^PostgreSQL \([0-9]*\)\.\([0-9]*\)\([a-zA-Z0-9.]*\)$/\1.\2/') + $SED -e 's/^PostgreSQL \([0-9]*\)\(\.[0-9]*\)\{0,1\}\(.*\)$/\1\2/') + +# if PostgreSQL version starts with two digits, the major version is those digits +version_num=$(echo "$version_num"| $SED -e 's/^\([0-9]\{2\}\)\(.*\)$/\1/') if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 fi -if test "$version_num" != '9.5' -a "$version_num" != '9.6'; then +if test "$version_num" != '9.5' -a "$version_num" != '9.6' -a "$version_num" != '10'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.in b/configure.in index 8e9ab3f0a..f546e7dee 100644 --- a/configure.in +++ b/configure.in @@ -50,13 +50,16 @@ fi # check we're building against a supported version of PostgreSQL citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) version_num=$(echo "$citusac_pg_config_version"| - $SED -e 's/^PostgreSQL \([[0-9]]*\)\.\([[0-9]]*\)\([[a-zA-Z0-9.]]*\)$/\1.\2/') + $SED -e 's/^PostgreSQL \([[0-9]]*\)\(\.[[0-9]]*\)\{0,1\}\(.*\)$/\1\2/') + +# if PostgreSQL version starts with two digits, the major version is those digits +version_num=$(echo "$version_num"| $SED -e 's/^\([[0-9]]\{2\}\)\(.*\)$/\1/') if test -z "$version_num"; then AC_MSG_ERROR([Could not detect PostgreSQL version from pg_config.]) fi -if test "$version_num" != '9.5' -a "$version_num" != '9.6'; then +if test "$version_num" != '9.5' -a "$version_num" != '9.6' -a "$version_num" != '10'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 103fedc4f..fdd1c99db 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -84,6 +84,9 @@ static void CreateHashDistributedTable(Oid relationId, char *distributionColumnN static Oid ColumnType(Oid relationId, char *columnName); static void CopyLocalDataIntoShards(Oid relationId); static List * TupleDescColumnNameList(TupleDesc tupleDescriptor); +#if (PG_VERSION_NUM >= 100000) +static bool RelationUsesIdentityColumns(TupleDesc relationDesc); +#endif /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_create_distributed_table); @@ -349,6 +352,23 @@ ConvertToDistributedTable(Oid relationId, char *distributionColumnName, "foreign tables."))); } +#if (PG_VERSION_NUM >= 100000) + if (relation->rd_rel->relispartition) + { + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot distribute relation: %s", relationName), + errdetail("Distributing partition tables is unsupported."))); + } + + if (RelationUsesIdentityColumns(relationDesc)) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot distribute relation: %s", relationName), + errdetail("Distributed relations must not use GENERATED " + "... AS IDENTITY."))); + } +#endif + /* check that table is empty if that is required */ if (requireEmpty && !LocalTableEmpty(relationId)) { @@ -874,3 +894,30 @@ TupleDescColumnNameList(TupleDesc tupleDescriptor) return columnNameList; } + + +/* + * RelationUsesIdentityColumns returns whether a given relation uses the SQL + * GENERATED ... AS IDENTITY features supported as of PostgreSQL 10. + */ +#if (PG_VERSION_NUM >= 100000) +static bool +RelationUsesIdentityColumns(TupleDesc relationDesc) +{ + int attributeIndex = 0; + + for (attributeIndex = 0; attributeIndex < relationDesc->natts; attributeIndex++) + { + Form_pg_attribute attributeForm = relationDesc->attrs[attributeIndex]; + + if (attributeForm->attidentity != '\0') + { + return true; + } + } + + return false; +} + + +#endif diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 4e2db236d..0a526af53 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -356,11 +356,21 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) dest->rStartup(dest, 0, tupleDescriptor); /* initialize copy state to read from COPY data source */ +#if (PG_VERSION_NUM >= 100000) + copyState = BeginCopyFrom(NULL, + distributedRelation, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); +#else copyState = BeginCopyFrom(distributedRelation, copyStatement->filename, copyStatement->is_program, copyStatement->attlist, copyStatement->options); +#endif /* set up callback to identify error line number */ errorCallback.callback = CopyFromErrorCallback; @@ -455,11 +465,21 @@ CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) (ShardConnections *) palloc0(sizeof(ShardConnections)); /* initialize copy state to read from COPY data source */ +#if (PG_VERSION_NUM >= 100000) + CopyState copyState = BeginCopyFrom(NULL, + distributedRelation, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); +#else CopyState copyState = BeginCopyFrom(distributedRelation, copyStatement->filename, copyStatement->is_program, copyStatement->attlist, copyStatement->options); +#endif CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index af0fcf5e7..e3e674a9d 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -9,6 +9,7 @@ #include "postgres.h" #include "miscadmin.h" +#include "pgstat.h" #include #include @@ -54,7 +55,13 @@ RedirectCopyDataToRegularFile(const char *filename) /* if received data has contents, append to regular file */ if (copyData->len > 0) { +#if (PG_VERSION_NUM >= 100000) + int appended = FileWrite(fileDesc, copyData->data, copyData->len, + PG_WAIT_IO); +#else int appended = FileWrite(fileDesc, copyData->data, copyData->len); +#endif + if (appended != copyData->len) { ereport(ERROR, (errcode_for_file_access(), @@ -98,7 +105,12 @@ SendRegularFile(const char *filename) SendCopyOutStart(); +#if (PG_VERSION_NUM >= 100000) + readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize, PG_WAIT_IO); +#else readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize); +#endif + while (readBytes > 0) { fileBuffer->len = readBytes; @@ -106,7 +118,12 @@ SendRegularFile(const char *filename) SendCopyData(fileBuffer); resetStringInfo(fileBuffer); +#if (PG_VERSION_NUM >= 100000) + readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize, + PG_WAIT_IO); +#else readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize); +#endif } SendCopyDone(); diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index 050b8eee2..ecfc54517 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -9,6 +9,7 @@ */ #include "postgres.h" +#include "pgstat.h" #include "libpq-fe.h" @@ -434,7 +435,12 @@ GetRemoteCommandResult(MultiConnection *connection, bool raiseInterrupts) /* this means we have to wait for data to go out */ Assert(rc == 1); +#if (PG_VERSION_NUM >= 100000) + rc = WaitLatchOrSocket(MyLatch, waitFlags | WL_SOCKET_WRITEABLE, socket, 0, + PG_WAIT_EXTENSION); +#else rc = WaitLatchOrSocket(MyLatch, waitFlags | WL_SOCKET_WRITEABLE, socket, 0); +#endif if (rc & WL_POSTMASTER_DEATH) { @@ -484,7 +490,12 @@ GetRemoteCommandResult(MultiConnection *connection, bool raiseInterrupts) break; } +#if (PG_VERSION_NUM >= 100000) + rc = WaitLatchOrSocket(MyLatch, waitFlags | WL_SOCKET_READABLE, socket, 0, + PG_WAIT_EXTENSION); +#else rc = WaitLatchOrSocket(MyLatch, waitFlags | WL_SOCKET_READABLE, socket, 0); +#endif if (rc & WL_POSTMASTER_DEATH) { diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index e6e144930..91dc72119 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -152,6 +152,10 @@ ExecuteIntoDestReceiver(Query *query, ParamListInfo params, DestReceiver *dest) NULL); PortalStart(portal, params, eflags, GetActiveSnapshot()); +#if (PG_VERSION_NUM >= 100000) + PortalRun(portal, count, false, true, dest, dest, NULL); +#else PortalRun(portal, count, false, dest, dest, NULL); +#endif PortalDrop(portal, false); } diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 40abbff78..4c848c560 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -320,7 +320,15 @@ LoadTuplesIntoTupleStore(CitusScanState *citusScanState, Job *workerJob) if (BinaryMasterCopyFormat) { - DefElem *copyOption = makeDefElem("format", (Node *) makeString("binary")); + DefElem *copyOption = NULL; + +#if (PG_VERSION_NUM >= 100000) + int location = -1; /* "unknown" token location */ + copyOption = makeDefElem("format", (Node *) makeString("binary"), location); +#else + copyOption = makeDefElem("format", (Node *) makeString("binary")); +#endif + copyOptions = lappend(copyOptions, copyOption); } @@ -334,8 +342,13 @@ LoadTuplesIntoTupleStore(CitusScanState *citusScanState, Job *workerJob) jobDirectoryName = MasterJobDirectoryName(workerTask->jobId); taskFilename = TaskFilename(jobDirectoryName, workerTask->taskId); +#if (PG_VERSION_NUM >= 100000) + copyState = BeginCopyFrom(NULL, stubRelation, taskFilename->data, false, NULL, + NULL, copyOptions); +#else copyState = BeginCopyFrom(stubRelation, taskFilename->data, false, NULL, copyOptions); +#endif while (true) { diff --git a/src/backend/distributed/executor/multi_utility.c b/src/backend/distributed/executor/multi_utility.c index fb9601115..9be4e2df8 100644 --- a/src/backend/distributed/executor/multi_utility.c +++ b/src/backend/distributed/executor/multi_utility.c @@ -163,6 +163,49 @@ static void PostProcessUtility(Node *parsetree); static bool warnedUserAbout2PC = false; +/* + * multi_ProcessUtility9x is the 9.x-compatible wrapper for Citus' main utility + * hook. It simply adapts the old-style hook to call into the new-style (10+) + * hook, which is what now houses all actual logic. + */ +void +multi_ProcessUtility9x(Node *parsetree, + const char *queryString, + ProcessUtilityContext context, + ParamListInfo params, + DestReceiver *dest, + char *completionTag) +{ + PlannedStmt *plannedStmt = makeNode(PlannedStmt); + plannedStmt->commandType = CMD_UTILITY; + plannedStmt->utilityStmt = parsetree; + + multi_ProcessUtility(plannedStmt, queryString, context, params, NULL, dest, + completionTag); +} + + +/* + * CitusProcessUtility is a version-aware wrapper of ProcessUtility to account + * for argument differences between the 9.x and 10+ PostgreSQL versions. + */ +void +CitusProcessUtility(Node *node, const char *queryString, ProcessUtilityContext context, + ParamListInfo params, DestReceiver *dest, char *completionTag) +{ +#if (PG_VERSION_NUM >= 100000) + PlannedStmt *plannedStmt = makeNode(PlannedStmt); + plannedStmt->commandType = CMD_UTILITY; + plannedStmt->utilityStmt = node; + + ProcessUtility(plannedStmt, queryString, context, params, NULL, dest, + completionTag); +#else + ProcessUtility(node, queryString, context, params, dest, completionTag); +#endif +} + + /* * multi_ProcessUtility is the main entry hook for implementing Citus-specific * utility behavior. Its primary responsibilities are intercepting COPY and DDL @@ -173,13 +216,15 @@ static bool warnedUserAbout2PC = false; * TRUNCATE and VACUUM are also supported. */ void -multi_ProcessUtility(Node *parsetree, +multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + struct QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { + Node *parsetree = pstmt->utilityStmt; bool commandMustRunAsOwner = false; Oid savedUserId = InvalidOid; int savedSecurityContext = 0; @@ -194,8 +239,13 @@ multi_ProcessUtility(Node *parsetree, * that state. Since we never need to intercept transaction statements, * skip our checks and immediately fall into standard_ProcessUtility. */ +#if (PG_VERSION_NUM >= 100000) + standard_ProcessUtility(pstmt, queryString, context, + params, queryEnv, dest, completionTag); +#else standard_ProcessUtility(parsetree, queryString, context, params, dest, completionTag); +#endif return; } @@ -213,8 +263,13 @@ multi_ProcessUtility(Node *parsetree, * Ensure that utility commands do not behave any differently until CREATE * EXTENSION is invoked. */ +#if (PG_VERSION_NUM >= 100000) + standard_ProcessUtility(pstmt, queryString, context, + params, queryEnv, dest, completionTag); +#else standard_ProcessUtility(parsetree, queryString, context, params, dest, completionTag); +#endif return; } @@ -280,9 +335,14 @@ multi_ProcessUtility(Node *parsetree, { if (IsA(parsetree, IndexStmt)) { + MemoryContext oldContext = MemoryContextSwitchTo(GetMemoryChunkContext( + parsetree)); + /* copy parse tree since we might scribble on it to fix the schema name */ parsetree = copyObject(parsetree); + MemoryContextSwitchTo(oldContext); + ddlJobs = PlanIndexStmt((IndexStmt *) parsetree, queryString); } @@ -392,8 +452,14 @@ multi_ProcessUtility(Node *parsetree, SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); } +#if (PG_VERSION_NUM >= 100000) + pstmt->utilityStmt = parsetree; + standard_ProcessUtility(pstmt, queryString, context, + params, queryEnv, dest, completionTag); +#else standard_ProcessUtility(parsetree, queryString, context, params, dest, completionTag); +#endif /* don't run post-process code for local commands */ if (ddlJobs != NIL) @@ -593,6 +659,8 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, bool *commandMustR { bool isFrom = copyStatement->is_from; Relation copiedRelation = NULL; + char *schemaName = NULL; + MemoryContext relationContext = NULL; /* consider using RangeVarGetRelidExtended to check perms before locking */ copiedRelation = heap_openrv(copyStatement->relation, @@ -601,8 +669,12 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, bool *commandMustR isDistributedRelation = IsDistributedTable(RelationGetRelid(copiedRelation)); /* ensure future lookups hit the same relation */ - copyStatement->relation->schemaname = get_namespace_name( - RelationGetNamespace(copiedRelation)); + schemaName = get_namespace_name(RelationGetNamespace(copiedRelation)); + + /* ensure we copy string into proper context */ + relationContext = GetMemoryChunkContext(copyStatement->relation); + schemaName = MemoryContextStrdup(relationContext, schemaName); + copyStatement->relation->schemaname = schemaName; heap_close(copiedRelation, NoLock); } @@ -723,6 +795,7 @@ PlanIndexStmt(IndexStmt *createIndexStatement, const char *createIndexCommand) bool isDistributedRelation = false; char *namespaceName = NULL; LOCKMODE lockmode = ShareLock; + MemoryContext relationContext = NULL; /* * We don't support concurrently creating indexes for distributed @@ -753,6 +826,10 @@ PlanIndexStmt(IndexStmt *createIndexStatement, const char *createIndexCommand) * search path by the time postgres starts processing this statement. */ namespaceName = get_namespace_name(RelationGetNamespace(relation)); + + /* ensure we copy string into proper context */ + relationContext = GetMemoryChunkContext(createIndexStatement->relation); + namespaceName = MemoryContextStrdup(relationContext, namespaceName); createIndexStatement->relation->schemaname = namespaceName; heap_close(relation, NoLock); @@ -1506,7 +1583,7 @@ ErrorIfUnsupportedIndexStmt(IndexStmt *createIndexStatement) /* caller uses ShareLock for non-concurrent indexes, use the same lock here */ LOCKMODE lockMode = ShareLock; Oid relationId = RangeVarGetRelid(relation, lockMode, missingOk); - Var *partitionKey = PartitionKey(relationId); + Var *partitionKey = DistPartitionKey(relationId); char partitionMethod = PartitionMethod(relationId); List *indexParameterList = NIL; ListCell *indexParameterCell = NULL; @@ -1653,7 +1730,7 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) continue; } - partitionColumn = PartitionKey(relationId); + partitionColumn = DistPartitionKey(relationId); tuple = SearchSysCacheAttName(relationId, alterColumnName); if (HeapTupleIsValid(tuple)) @@ -1737,7 +1814,7 @@ ErrorIfUnsupportedAlterAddConstraintStmt(AlterTableStmt *alterTableStatement) LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); char distributionMethod = PartitionMethod(relationId); - Var *distributionColumn = PartitionKey(relationId); + Var *distributionColumn = DistPartitionKey(relationId); uint32 colocationId = TableColocationId(relationId); Relation relation = relation_open(relationId, ExclusiveLock); @@ -2010,7 +2087,7 @@ ErrorIfUnsupportedForeignConstraint(Relation relation, char distributionMethod, * Partition column must exist in both referencing and referenced side of the * foreign key constraint. They also must be in same ordinal. */ - referencedTablePartitionColumn = PartitionKey(referencedTableId); + referencedTablePartitionColumn = DistPartitionKey(referencedTableId); } else { @@ -2141,6 +2218,7 @@ ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt) { Oid sequenceId = RangeVarGetRelid(alterSeqStmt->sequence, AccessShareLock, alterSeqStmt->missing_ok); + bool sequenceOwned = false; Oid ownedByTableId = InvalidOid; Oid newOwnedByTableId = InvalidOid; int32 ownedByColumnId = 0; @@ -2152,8 +2230,20 @@ ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt) return; } - /* see whether the sequences is already owned by a distributed table */ - if (sequenceIsOwned(sequenceId, &ownedByTableId, &ownedByColumnId)) +#if (PG_VERSION_NUM >= 100000) + sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_AUTO, &ownedByTableId, + &ownedByColumnId); + if (!sequenceOwned) + { + sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_INTERNAL, &ownedByTableId, + &ownedByColumnId); + } +#else + sequenceOwned = sequenceIsOwned(sequenceId, &ownedByTableId, &ownedByColumnId); +#endif + + /* see whether the sequence is already owned by a distributed table */ + if (sequenceOwned) { hasDistributedOwner = IsDistributedTable(ownedByTableId); } @@ -2350,8 +2440,9 @@ CreateLocalTable(RangeVar *relation, char *nodeName, int32 nodePort) /* run only a selected set of DDL commands */ if (applyDDLCommand) { - ProcessUtility(ddlCommandNode, CreateCommandTag(ddlCommandNode), - PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); + CitusProcessUtility(ddlCommandNode, CreateCommandTag(ddlCommandNode), + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); + CommandCounterIncrement(); } } @@ -2984,8 +3075,7 @@ PostProcessUtility(Node *parsetree) indexForm = (Form_pg_index) GETSTRUCT(indexTuple); indexForm->indisvalid = true; - simple_heap_update(pg_index, &indexTuple->t_self, indexTuple); - CatalogUpdateIndexes(pg_index, indexTuple); + CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple); /* clean up; index now marked valid, but ROLLBACK will mark invalid */ heap_freetuple(indexTuple); diff --git a/src/backend/distributed/master/master_delete_protocol.c b/src/backend/distributed/master/master_delete_protocol.c index 99239518e..89754e7e6 100644 --- a/src/backend/distributed/master/master_delete_protocol.c +++ b/src/backend/distributed/master/master_delete_protocol.c @@ -109,11 +109,16 @@ master_apply_delete_command(PG_FUNCTION_ARGS) LOCKMODE lockMode = 0; char partitionMethod = 0; bool failOK = false; +#if (PG_VERSION_NUM >= 100000) + RawStmt *rawStmt = (RawStmt *) ParseTreeRawStmt(queryString); + queryTreeNode = rawStmt->stmt; +#else + queryTreeNode = ParseTreeNode(queryString); +#endif EnsureCoordinator(); CheckCitusVersion(ERROR); - queryTreeNode = ParseTreeNode(queryString); if (!IsA(queryTreeNode, DeleteStmt)) { ereport(ERROR, (errmsg("query \"%s\" is not a delete statement", @@ -144,7 +149,11 @@ master_apply_delete_command(PG_FUNCTION_ARGS) CheckDistributedTable(relationId); EnsureTablePermissions(relationId, ACL_DELETE); +#if (PG_VERSION_NUM >= 100000) + queryTreeList = pg_analyze_and_rewrite(rawStmt, queryString, NULL, 0, NULL); +#else queryTreeList = pg_analyze_and_rewrite(queryTreeNode, queryString, NULL, 0); +#endif deleteQuery = (Query *) linitial(queryTreeList); CheckTableCount(deleteQuery); @@ -490,7 +499,7 @@ CheckDeleteCriteria(Node *deleteCriteria) static void CheckPartitionColumn(Oid relationId, Node *whereClause) { - Var *partitionColumn = PartitionKey(relationId); + Var *partitionColumn = DistPartitionKey(relationId); ListCell *columnCell = NULL; List *columnList = pull_var_clause_default(whereClause); @@ -558,7 +567,11 @@ ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, restrictInfoList = lappend(restrictInfoList, lessThanRestrictInfo); restrictInfoList = lappend(restrictInfoList, greaterThanRestrictInfo); +#if (PG_VERSION_NUM >= 100000) + dropShard = predicate_implied_by(deleteCriteriaList, restrictInfoList, false); +#else dropShard = predicate_implied_by(deleteCriteriaList, restrictInfoList); +#endif if (dropShard) { dropShardIntervalList = lappend(dropShardIntervalList, shardInterval); diff --git a/src/backend/distributed/master/master_metadata_utility.c b/src/backend/distributed/master/master_metadata_utility.c index ad7e293c0..cc1453435 100644 --- a/src/backend/distributed/master/master_metadata_utility.c +++ b/src/backend/distributed/master/master_metadata_utility.c @@ -800,8 +800,7 @@ InsertShardRow(Oid relationId, uint64 shardId, char storageType, tupleDescriptor = RelationGetDescr(pgDistShard); heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - simple_heap_insert(pgDistShard, heapTuple); - CatalogUpdateIndexes(pgDistShard, heapTuple); + CatalogTupleInsert(pgDistShard, heapTuple); /* invalidate previous cache entry and close relation */ CitusInvalidateRelcacheByRelid(relationId); @@ -848,8 +847,7 @@ InsertShardPlacementRow(uint64 shardId, uint64 placementId, tupleDescriptor = RelationGetDescr(pgDistShardPlacement); heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - simple_heap_insert(pgDistShardPlacement, heapTuple); - CatalogUpdateIndexes(pgDistShardPlacement, heapTuple); + CatalogTupleInsert(pgDistShardPlacement, heapTuple); CitusInvalidateRelcacheByShardId(shardId); @@ -904,8 +902,8 @@ InsertIntoPgDistPartition(Oid relationId, char distributionMethod, newTuple = heap_form_tuple(RelationGetDescr(pgDistPartition), newValues, newNulls); /* finally insert tuple, build index entries & register cache invalidation */ - simple_heap_insert(pgDistPartition, newTuple); - CatalogUpdateIndexes(pgDistPartition, newTuple); + CatalogTupleInsert(pgDistPartition, newTuple); + CitusInvalidateRelcacheByRelid(relationId); RecordDistributedRelationDependencies(relationId, (Node *) distributionColumn); @@ -946,8 +944,13 @@ RecordDistributedRelationDependencies(Oid distributedRelationId, Node *distribut recordDependencyOn(&relationAddr, &citusExtensionAddr, DEPENDENCY_NORMAL); /* make sure the distribution key column/expression does not just go away */ +#if (PG_VERSION_NUM >= 100000) + recordDependencyOnSingleRelExpr(&relationAddr, distributionKey, distributedRelationId, + DEPENDENCY_NORMAL, DEPENDENCY_NORMAL, false); +#else recordDependencyOnSingleRelExpr(&relationAddr, distributionKey, distributedRelationId, DEPENDENCY_NORMAL, DEPENDENCY_NORMAL); +#endif } @@ -1156,9 +1159,8 @@ UpdateShardPlacementState(uint64 placementId, char shardState) replace[Anum_pg_dist_shard_placement_shardstate - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); - simple_heap_update(pgDistShardPlacement, &heapTuple->t_self, heapTuple); - CatalogUpdateIndexes(pgDistShardPlacement, heapTuple); + CatalogTupleUpdate(pgDistShardPlacement, &heapTuple->t_self, heapTuple); shardId = DatumGetInt64(heap_getattr(heapTuple, Anum_pg_dist_shard_placement_shardid, @@ -1223,9 +1225,8 @@ UpdateColocationGroupReplicationFactor(uint32 colocationId, int replicationFacto replace[Anum_pg_dist_colocation_replicationfactor - 1] = true; newHeapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); - simple_heap_update(pgDistColocation, &newHeapTuple->t_self, newHeapTuple); - CatalogUpdateIndexes(pgDistColocation, newHeapTuple); + CatalogTupleUpdate(pgDistColocation, &newHeapTuple->t_self, newHeapTuple); CommandCounterIncrement(); diff --git a/src/backend/distributed/master/master_modify_multiple_shards.c b/src/backend/distributed/master/master_modify_multiple_shards.c index f7838b08c..08d9bc291 100644 --- a/src/backend/distributed/master/master_modify_multiple_shards.c +++ b/src/backend/distributed/master/master_modify_multiple_shards.c @@ -85,12 +85,17 @@ master_modify_multiple_shards(PG_FUNCTION_ARGS) List *prunedShardIntervalList = NIL; List *taskList = NIL; int32 affectedTupleCount = 0; +#if (PG_VERSION_NUM >= 100000) + RawStmt *rawStmt = (RawStmt *) ParseTreeRawStmt(queryString); + queryTreeNode = rawStmt->stmt; +#else + queryTreeNode = ParseTreeNode(queryString); +#endif EnsureCoordinator(); CheckCitusVersion(ERROR); - queryTreeNode = ParseTreeNode(queryString); if (IsA(queryTreeNode, DeleteStmt)) { DeleteStmt *deleteStatement = (DeleteStmt *) queryTreeNode; @@ -136,7 +141,11 @@ master_modify_multiple_shards(PG_FUNCTION_ARGS) CheckDistributedTable(relationId); +#if (PG_VERSION_NUM >= 100000) + queryTreeList = pg_analyze_and_rewrite(rawStmt, queryString, NULL, 0, NULL); +#else queryTreeList = pg_analyze_and_rewrite(queryTreeNode, queryString, NULL, 0); +#endif modifyQuery = (Query *) linitial(queryTreeList); if (modifyQuery->commandType != CMD_UTILITY) diff --git a/src/backend/distributed/master/master_node_protocol.c b/src/backend/distributed/master/master_node_protocol.c index d7214cf04..5a82fa8bd 100644 --- a/src/backend/distributed/master/master_node_protocol.c +++ b/src/backend/distributed/master/master_node_protocol.c @@ -60,6 +60,9 @@ #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/tqual.h" +#if (PG_VERSION_NUM >= 100000) +#include "utils/varlena.h" +#endif /* Shard related configuration */ @@ -495,7 +498,11 @@ GetTableCreationCommands(Oid relationId, bool includeSequenceDefaults) { List *tableDDLEventList = NIL; char tableType = 0; +#if (PG_VERSION_NUM >= 100000) + List *sequenceIdlist = getOwnedSequences(relationId, InvalidAttrNumber); +#else List *sequenceIdlist = getOwnedSequences(relationId); +#endif ListCell *sequenceIdCell; char *tableSchemaDef = NULL; char *tableColumnOptionsDef = NULL; diff --git a/src/backend/distributed/master/worker_node_manager.c b/src/backend/distributed/master/worker_node_manager.c index 0953bf7d2..64e1cc891 100644 --- a/src/backend/distributed/master/worker_node_manager.c +++ b/src/backend/distributed/master/worker_node_manager.c @@ -19,7 +19,11 @@ #include "distributed/metadata_cache.h" #include "distributed/multi_client_executor.h" #include "libpq/hba.h" +#if (PG_VERSION_NUM >= 100000) +#include "common/ip.h" +#else #include "libpq/ip.h" +#endif #include "libpq/libpq-be.h" #include "postmaster/postmaster.h" #include "storage/fd.h" diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index e7a659c2b..088038f8b 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -812,9 +812,8 @@ MarkNodeHasMetadata(char *nodeName, int32 nodePort, bool hasMetadata) replace[Anum_pg_dist_node_hasmetadata - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); - simple_heap_update(pgDistNode, &heapTuple->t_self, heapTuple); - CatalogUpdateIndexes(pgDistNode, heapTuple); + CatalogTupleUpdate(pgDistNode, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistNodeRelationId()); @@ -837,7 +836,11 @@ List * SequenceDDLCommandsForTable(Oid relationId) { List *sequenceDDLList = NIL; +#if (PG_VERSION_NUM >= 100000) + List *ownedSequences = getOwnedSequences(relationId, InvalidAttrNumber); +#else List *ownedSequences = getOwnedSequences(relationId); +#endif ListCell *listCell; char *ownerName = TableOwner(relationId); @@ -921,7 +924,19 @@ EnsureSupportedSequenceColumnType(Oid sequenceOid) bool hasMetadataWorkers = HasMetadataWorkers(); /* call sequenceIsOwned in order to get the tableId and columnId */ +#if (PG_VERSION_NUM >= 100000) + bool sequenceOwned = sequenceIsOwned(sequenceOid, DEPENDENCY_AUTO, &tableId, + &columnId); + if (!sequenceOwned) + { + sequenceOwned = sequenceIsOwned(sequenceOid, DEPENDENCY_INTERNAL, &tableId, + &columnId); + } + + Assert(sequenceOwned); +#else sequenceIsOwned(sequenceOid, &tableId, &columnId); +#endif shouldSyncMetadata = ShouldSyncTableMetadata(tableId); diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index e6a5b3e94..d0ff84d93 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -72,8 +72,6 @@ typedef struct RemoteExplainPlan /* Explain functions for distributed queries */ -static void CitusExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params); static void ExplainJob(Job *job, ExplainState *es); static void ExplainMapMergeJob(MapMergeJob *mapMergeJob, ExplainState *es); static void ExplainTaskList(List *taskList, ExplainState *es); @@ -85,6 +83,15 @@ static void ExplainTaskPlacement(ShardPlacement *taskPlacement, List *explainOut static StringInfo BuildRemoteExplainQuery(char *queryString, ExplainState *es); /* Static Explain functions copied from explain.c */ +#if (PG_VERSION_NUM >= 100000) +static void ExplainOneQuery(Query *query, int cursorOptions, + IntoClause *into, ExplainState *es, + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv); +#else +static void ExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, + const char *queryString, ParamListInfo params); +#endif static void ExplainOpenGroup(const char *objtype, const char *labelname, bool labeled, ExplainState *es); static void ExplainCloseGroup(const char *objtype, const char *labelname, @@ -121,6 +128,42 @@ CitusExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es } +/* + * CoordinatorInsertSelectExplainScan is a custom scan explain callback function + * which is used to print explain information of a Citus plan for an INSERT INTO + * distributed_table SELECT ... query that is evaluated on the coordinator. + */ +void +CoordinatorInsertSelectExplainScan(CustomScanState *node, List *ancestors, + struct ExplainState *es) +{ + CitusScanState *scanState = (CitusScanState *) node; + MultiPlan *multiPlan = scanState->multiPlan; + Query *query = multiPlan->insertSelectSubquery; + IntoClause *into = NULL; + ParamListInfo params = NULL; + char *queryString = NULL; + + if (es->analyze) + { + /* avoiding double execution here is tricky, error out for now */ + ereport(ERROR, (errmsg("EXPLAIN ANALYZE is currently not supported for INSERT " + "... SELECT commands via the coordinator"))); + } + + ExplainOpenGroup("Select Query", "Select Query", false, es); + + /* explain the inner SELECT query */ +#if (PG_VERSION_NUM >= 100000) + ExplainOneQuery(query, 0, into, es, queryString, params, NULL); +#else + ExplainOneQuery(query, into, es, queryString, params); +#endif + + ExplainCloseGroup("Select Query", "Select Query", false, es); +} + + /* * ExplainJob shows the EXPLAIN output for a Job in the physical plan of * a distributed query by showing the remote EXPLAIN for the first task, @@ -539,6 +582,61 @@ BuildRemoteExplainQuery(char *queryString, ExplainState *es) /* *INDENT-OFF* */ +/* + * ExplainOneQuery - + * print out the execution plan for one Query + * + * "into" is NULL unless we are explaining the contents of a CreateTableAsStmt. + */ +static void +#if (PG_VERSION_NUM >= 100000) +ExplainOneQuery(Query *query, int cursorOptions, + IntoClause *into, ExplainState *es, + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv) +#else +ExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, + const char *queryString, ParamListInfo params) +#endif +{ + /* if an advisor plugin is present, let it manage things */ + if (ExplainOneQuery_hook) +#if (PG_VERSION_NUM >= 100000) + (*ExplainOneQuery_hook) (query, cursorOptions, into, es, + queryString, params); +#else + (*ExplainOneQuery_hook) (query, into, es, queryString, params); +#endif + else + { + PlannedStmt *plan; + instr_time planstart, + planduration; + + INSTR_TIME_SET_CURRENT(planstart); + + /* plan the query */ +#if (PG_VERSION_NUM >= 100000) + plan = pg_plan_query(query, cursorOptions, params); +#elif (PG_VERSION_NUM >= 90600) + plan = pg_plan_query(query, into ? 0 : CURSOR_OPT_PARALLEL_OK, params); +#else + plan = pg_plan_query(query, 0, params); +#endif + + INSTR_TIME_SET_CURRENT(planduration); + INSTR_TIME_SUBTRACT(planduration, planstart); + + /* run it (if needed) and produce output */ +#if (PG_VERSION_NUM >= 100000) + ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + &planduration); +#else + ExplainOnePlan(plan, into, es, queryString, params, &planduration); +#endif + } +} + /* * Open a group of related objects. * @@ -715,73 +813,3 @@ ExplainYAMLLineStarting(ExplainState *es) appendStringInfoSpaces(es->str, es->indent * 2); } } - - -/* - * CoordinatorInsertSelectExplainScan is a custom scan explain callback function - * which is used to print explain information of a Citus plan for an INSERT INTO - * distributed_table SELECT ... query that is evaluated on the coordinator. - */ -void -CoordinatorInsertSelectExplainScan(CustomScanState *node, List *ancestors, - struct ExplainState *es) -{ - CitusScanState *scanState = (CitusScanState *) node; - MultiPlan *multiPlan = scanState->multiPlan; - Query *query = multiPlan->insertSelectSubquery; - IntoClause *into = NULL; - ParamListInfo params = NULL; - char *queryString = NULL; - - if (es->analyze) - { - /* avoiding double execution here is tricky, error out for now */ - ereport(ERROR, (errmsg("EXPLAIN ANALYZE is currently not supported for INSERT " - "... SELECT commands via the coordinator"))); - } - - ExplainOpenGroup("Select Query", "Select Query", false, es); - - /* explain the inner SELECT query */ - CitusExplainOneQuery(query, into, es, queryString, params); - - ExplainCloseGroup("Select Query", "Select Query", false, es); -} - - -/* - * CitusExplainOneQuery is simply a duplicate of ExplainOneQuery in explain.c, which - * is static. - */ -static void -CitusExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params) -{ - /* copied from ExplainOneQuery in explain.c */ - if (ExplainOneQuery_hook) - { - (*ExplainOneQuery_hook) (query, into, es, queryString, params); - } - else - { - PlannedStmt *plan; - instr_time planstart, - planduration; - int cursorOptions = 0; - - INSTR_TIME_SET_CURRENT(planstart); - - #if (PG_VERSION_NUM >= 90600) - cursorOptions = into ? 0 : CURSOR_OPT_PARALLEL_OK; - #endif - - /* plan the query */ - plan = pg_plan_query(query, cursorOptions, params); - - INSTR_TIME_SET_CURRENT(planduration); - INSTR_TIME_SUBTRACT(planduration, planstart); - - /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, &planduration); - } -} diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index ce0894e4e..a5242694c 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -1526,7 +1526,7 @@ RightColumn(OpExpr *joinClause) Var * PartitionColumn(Oid relationId, uint32 rangeTableId) { - Var *partitionKey = PartitionKey(relationId); + Var *partitionKey = DistPartitionKey(relationId); Var *partitionColumn = NULL; /* short circuit for reference tables */ @@ -1544,7 +1544,7 @@ PartitionColumn(Oid relationId, uint32 rangeTableId) /* - * PartitionKey returns the partition key column for the given relation. Note + * DistPartitionKey returns the partition key column for the given relation. Note * that in the context of distributed join and query planning, the callers of * this function *must* set the partition key column's range table reference * (varno) to match the table's location in the query range table list. @@ -1553,7 +1553,7 @@ PartitionColumn(Oid relationId, uint32 rangeTableId) * returns NULL when called for reference tables. */ Var * -PartitionKey(Oid relationId) +DistPartitionKey(Oid relationId) { DistTableCacheEntry *partitionEntry = DistributedTableCacheEntry(relationId); Node *variableNode = NULL; diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 7bfa0efec..6189dee78 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -47,6 +47,9 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#if (PG_VERSION_NUM >= 100000) +#include "utils/regproc.h" +#endif #include "utils/rel.h" #include "utils/syscache.h" #include "utils/tqual.h" @@ -1718,7 +1721,12 @@ MasterAverageExpression(Oid sumAggregateType, Oid countAggregateType, * will convert the types of the aggregates if necessary. */ operatorNameList = list_make1(makeString(DIVISION_OPER_NAME)); +#if (PG_VERSION_NUM >= 100000) + opExpr = make_op(NULL, operatorNameList, (Node *) firstSum, (Node *) secondSum, NULL, + -1); +#else opExpr = make_op(NULL, operatorNameList, (Node *) firstSum, (Node *) secondSum, -1); +#endif return opExpr; } @@ -2845,7 +2853,7 @@ IsPartitionColumn(Expr *columnExpression, Query *query) if (relationId != InvalidOid && column != NULL) { - Var *partitionColumn = PartitionKey(relationId); + Var *partitionColumn = DistPartitionKey(relationId); /* not all distributed tables have partition column */ if (partitionColumn != NULL && column->varattno == partitionColumn->varattno) @@ -3119,7 +3127,7 @@ PartitionColumnOpExpressionList(Query *query) Assert(rangeTableEntry->rtekind == RTE_RELATION); relationId = rangeTableEntry->relid; - partitionColumn = PartitionKey(relationId); + partitionColumn = DistPartitionKey(relationId); if (partitionColumn != NULL && candidatePartitionColumn->varattno == partitionColumn->varattno) diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 64a13de63..0bd255de1 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -131,6 +131,9 @@ static DeferredErrorMessage * InsertPartitionColumnMatchesSelect(Query *query, Oid * selectPartitionColumnTableId); static DeferredErrorMessage * ErrorIfQueryHasModifyingCTE(Query *queryTree); +#if (PG_VERSION_NUM >= 100000) +static List * get_all_actual_clauses(List *restrictinfo_list); +#endif /* @@ -1638,13 +1641,14 @@ MasterIrreducibleExpressionWalker(Node *expression, WalkerState *state) * should be checked in this function. * * Look through contain_mutable_functions_walker or future PG's equivalent for new - * node types before bumping this version number to fix compilation. + * node types before bumping this version number to fix compilation; e.g. for any + * PostgreSQL after 9.5, see check_functions_in_node. * * Once you've added them to this check, make sure you also evaluate them in the * executor! */ - StaticAssertStmt(PG_VERSION_NUM < 90700, "When porting to a newer PG this section" - " needs to be reviewed."); + StaticAssertStmt(PG_VERSION_NUM < 100100, "When porting to a newer PG this section" + " needs to be reviewed."); if (IsA(expression, Aggref)) { Aggref *expr = (Aggref *) expression; @@ -1836,12 +1840,20 @@ TargetEntryChangesValue(TargetEntry *targetEntry, Var *column, FromExpr *joinTre List *restrictClauseList = WhereClauseList(joinTree); OpExpr *equalityExpr = MakeOpExpression(column, BTEqualStrategyNumber); Const *rightConst = (Const *) get_rightop((Expr *) equalityExpr); + bool predicateIsImplied = false; rightConst->constvalue = newValue->constvalue; rightConst->constisnull = newValue->constisnull; rightConst->constbyval = newValue->constbyval; - if (predicate_implied_by(list_make1(equalityExpr), restrictClauseList)) +#if (PG_VERSION_NUM >= 100000) + predicateIsImplied = predicate_implied_by(list_make1(equalityExpr), + restrictClauseList, false); +#else + predicateIsImplied = predicate_implied_by(list_make1(equalityExpr), + restrictClauseList); +#endif + if (predicateIsImplied) { /* target entry of the form SET col = WHERE col = AND ... */ isColumnValueChanged = false; @@ -3036,3 +3048,34 @@ ErrorIfQueryHasModifyingCTE(Query *queryTree) /* everything OK */ return NULL; } + + +#if (PG_VERSION_NUM >= 100000) + +/* + * get_all_actual_clauses + * + * Returns a list containing the bare clauses from 'restrictinfo_list'. + * + * This loses the distinction between regular and pseudoconstant clauses, + * so be careful what you use it for. + */ +static List * +get_all_actual_clauses(List *restrictinfo_list) +{ + List *result = NIL; + ListCell *l; + + foreach(l, restrictinfo_list) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(l); + + Assert(IsA(rinfo, RestrictInfo)); + + result = lappend(result, rinfo->clause); + } + return result; +} + + +#endif diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 5fb340e17..8b45da19e 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -278,7 +278,7 @@ FindTranslatedVar(List *appendRelList, Oid relationOid, Index relationRteIndex, return NULL; } - relationPartitionKey = PartitionKey(relationOid); + relationPartitionKey = DistPartitionKey(relationOid); translaterVars = targetAppendRelInfo->translated_vars; foreach(translatedVarCell, translaterVars) @@ -429,7 +429,7 @@ EquivalenceListContainsRelationsEquality(List *attributeEquivalenceList, (RelationRestriction *) lfirst(relationRestrictionCell); int rteIdentity = GetRTEIdentity(relationRestriction->rte); - if (PartitionKey(relationRestriction->relationId) && + if (DistPartitionKey(relationRestriction->relationId) && !bms_is_member(rteIdentity, commonRteIdentities)) { return false; @@ -1114,7 +1114,7 @@ AddRteRelationToAttributeEquivalenceClass(AttributeEquivalenceClass ** return; } - relationPartitionKey = PartitionKey(relationId); + relationPartitionKey = DistPartitionKey(relationId); if (relationPartitionKey->varattno != varToBeAdded->varattno) { return; diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 0c09eb6f6..c4931d0cb 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -156,7 +156,11 @@ _PG_init(void) planner_hook = multi_planner; /* register utility hook */ +#if (PG_VERSION_NUM >= 100000) ProcessUtility_hook = multi_ProcessUtility; +#else + ProcessUtility_hook = multi_ProcessUtility9x; +#endif /* register for planner hook */ set_rel_pathlist_hook = multi_relation_restriction_hook; diff --git a/src/backend/distributed/test/deparse_shard_query.c b/src/backend/distributed/test/deparse_shard_query.c index fe70cb13a..76be876a1 100644 --- a/src/backend/distributed/test/deparse_shard_query.c +++ b/src/backend/distributed/test/deparse_shard_query.c @@ -50,9 +50,14 @@ deparse_shard_query_test(PG_FUNCTION_ARGS) { Node *parsetree = (Node *) lfirst(parseTreeCell); ListCell *queryTreeCell = NULL; + List *queryTreeList = NIL; - List *queryTreeList = pg_analyze_and_rewrite(parsetree, queryStringChar, - NULL, 0); +#if (PG_VERSION_NUM >= 100000) + queryTreeList = pg_analyze_and_rewrite((RawStmt *) parsetree, queryStringChar, + NULL, 0, NULL); +#else + queryTreeList = pg_analyze_and_rewrite(parsetree, queryStringChar, NULL, 0); +#endif foreach(queryTreeCell, queryTreeList) { diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index edff64850..52c800c3b 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -100,8 +100,8 @@ LogTransactionRecord(int groupId, char *transactionName) tupleDescriptor = RelationGetDescr(pgDistTransaction); heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - simple_heap_insert(pgDistTransaction, heapTuple); - CatalogUpdateIndexes(pgDistTransaction, heapTuple); + CatalogTupleInsert(pgDistTransaction, heapTuple); + CommandCounterIncrement(); /* close relation and invalidate previous cache entry */ diff --git a/src/backend/distributed/utils/citus_clauses.c b/src/backend/distributed/utils/citus_clauses.c index d357e953c..8514073a4 100644 --- a/src/backend/distributed/utils/citus_clauses.c +++ b/src/backend/distributed/utils/citus_clauses.c @@ -364,7 +364,11 @@ citus_evaluate_expr(Expr *expr, Oid result_type, int32 result_typmod, /* * And evaluate it. */ +#if (PG_VERSION_NUM >= 100000) + const_val = ExecEvalExprSwitchContext(exprstate, econtext, &const_is_null); +#else const_val = ExecEvalExprSwitchContext(exprstate, econtext, &const_is_null, NULL); +#endif /* Get info needed about result datatype */ get_typlenbyval(result_type, &resultTypLen, &resultTypByVal); diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index 42c7a4ece..50e25c86b 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -294,6 +294,10 @@ GetRangeTblKind(RangeTblEntry *rte) switch (rte->rtekind) { /* directly rtekind if it's not possibly an extended RTE */ +#if (PG_VERSION_NUM >= 100000) + case RTE_TABLEFUNC: + case RTE_NAMEDTUPLESTORE: +#endif case RTE_RELATION: case RTE_SUBQUERY: case RTE_JOIN: diff --git a/src/backend/distributed/utils/citus_ruleutils.c b/src/backend/distributed/utils/citus_ruleutils.c index baf7248d8..c1e2fd05d 100644 --- a/src/backend/distributed/utils/citus_ruleutils.c +++ b/src/backend/distributed/utils/citus_ruleutils.c @@ -193,10 +193,18 @@ pg_get_sequencedef_string(Oid sequenceRelationId) /* build our DDL command */ qualifiedSequenceName = generate_relation_name(sequenceRelationId, NIL); + +#if (PG_VERSION_NUM >= 100000) + sequenceDef = psprintf(CREATE_SEQUENCE_COMMAND, qualifiedSequenceName, + pgSequenceForm->seqincrement, pgSequenceForm->seqmin, + pgSequenceForm->seqmax, pgSequenceForm->seqstart, + pgSequenceForm->seqcycle ? "" : "NO "); +#else sequenceDef = psprintf(CREATE_SEQUENCE_COMMAND, qualifiedSequenceName, pgSequenceForm->increment_by, pgSequenceForm->min_value, pgSequenceForm->max_value, pgSequenceForm->start_value, pgSequenceForm->is_cycled ? "" : "NO "); +#endif return sequenceDef; } @@ -210,8 +218,20 @@ Form_pg_sequence pg_get_sequencedef(Oid sequenceRelationId) { Form_pg_sequence pgSequenceForm = NULL; - SysScanDesc scanDescriptor = NULL; HeapTuple heapTuple = NULL; + +#if (PG_VERSION_NUM >= 100000) + heapTuple = SearchSysCache1(SEQRELID, sequenceRelationId); + if (!HeapTupleIsValid(heapTuple)) + { + elog(ERROR, "cache lookup failed for sequence %u", sequenceRelationId); + } + + pgSequenceForm = (Form_pg_sequence) GETSTRUCT(heapTuple); + + ReleaseSysCache(heapTuple); +#else + SysScanDesc scanDescriptor = NULL; Relation sequenceRel = NULL; AclResult permissionCheck = ACLCHECK_NO_PRIV; @@ -241,6 +261,7 @@ pg_get_sequencedef(Oid sequenceRelationId) systable_endscan(scanDescriptor); heap_close(sequenceRel, AccessShareLock); +#endif return pgSequenceForm; } diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index a7357f20d..53e59b8fc 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -20,6 +20,7 @@ #include "commands/sequence.h" #include "distributed/colocation_utils.h" #include "distributed/listutils.h" +#include "distributed/master_metadata_utility.h" #include "distributed/master_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" @@ -126,7 +127,7 @@ MarkTablesColocated(Oid sourceRelationId, Oid targetRelationId) uint32 shardCount = ShardIntervalCount(sourceRelationId); uint32 shardReplicationFactor = TableShardReplicationFactor(sourceRelationId); - Var *sourceDistributionColumn = PartitionKey(sourceRelationId); + Var *sourceDistributionColumn = DistPartitionKey(sourceRelationId); Oid sourceDistributionColumnType = InvalidOid; /* reference tables has NULL distribution column */ @@ -477,8 +478,7 @@ CreateColocationGroup(int shardCount, int replicationFactor, Oid distributionCol tupleDescriptor = RelationGetDescr(pgDistColocation); heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - simple_heap_insert(pgDistColocation, heapTuple); - CatalogUpdateIndexes(pgDistColocation, heapTuple); + CatalogTupleInsert(pgDistColocation, heapTuple); /* increment the counter so that next command can see the row */ CommandCounterIncrement(); @@ -567,7 +567,7 @@ CheckDistributionColumnType(Oid sourceRelationId, Oid targetRelationId) Oid targetDistributionColumnType = InvalidOid; /* reference tables have NULL distribution column */ - sourceDistributionColumn = PartitionKey(sourceRelationId); + sourceDistributionColumn = DistPartitionKey(sourceRelationId); if (sourceDistributionColumn == NULL) { sourceDistributionColumnType = InvalidOid; @@ -578,7 +578,7 @@ CheckDistributionColumnType(Oid sourceRelationId, Oid targetRelationId) } /* reference tables have NULL distribution column */ - targetDistributionColumn = PartitionKey(targetRelationId); + targetDistributionColumn = DistPartitionKey(targetRelationId); if (targetDistributionColumn == NULL) { targetDistributionColumnType = InvalidOid; @@ -648,9 +648,10 @@ UpdateRelationColocationGroup(Oid distributedRelationId, uint32 colocationId) replace[Anum_pg_dist_partition_colocationid - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isNull, replace); - simple_heap_update(pgDistPartition, &heapTuple->t_self, heapTuple); - CatalogUpdateIndexes(pgDistPartition, heapTuple); + + CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); + CitusInvalidateRelcacheByRelid(distributedRelationId); CommandCounterIncrement(); diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index bf90c6df7..da5652157 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -43,7 +43,11 @@ typedef struct MaintenanceDaemonControlData * data in dbHash. */ int trancheId; +#if (PG_VERSION_NUM >= 100000) + char *lockTrancheName; +#else LWLockTranche lockTranche; +#endif LWLock lock; /* @@ -257,7 +261,11 @@ CitusMaintenanceDaemonMain(Datum main_arg) /* * Wait until timeout, or until somebody wakes us up. */ +#if (PG_VERSION_NUM >= 100000) + rc = WaitLatch(MyLatch, latchFlags, timeout, PG_WAIT_EXTENSION); +#else rc = WaitLatch(MyLatch, latchFlags, timeout); +#endif /* emergency bailout if postmaster has died */ if (rc & WL_POSTMASTER_DEATH) @@ -343,6 +351,13 @@ MaintenanceDaemonShmemInit(void) */ if (!alreadyInitialized) { +#if (PG_VERSION_NUM >= 100000) + MaintenanceDaemonControl->trancheId = LWLockNewTrancheId(); + MaintenanceDaemonControl->lockTrancheName = "Citus Maintenance Daemon"; + LWLockRegisterTranche(MaintenanceDaemonControl->trancheId, + MaintenanceDaemonControl->lockTrancheName); +#else + /* initialize lwlock */ LWLockTranche *tranche = &MaintenanceDaemonControl->lockTranche; @@ -355,6 +370,8 @@ MaintenanceDaemonShmemInit(void) tranche->array_stride = sizeof(LWLock); tranche->name = "Citus Maintenance Daemon"; LWLockRegisterTranche(MaintenanceDaemonControl->trancheId, tranche); +#endif + LWLockInitialize(&MaintenanceDaemonControl->lock, MaintenanceDaemonControl->trancheId); } diff --git a/src/backend/distributed/utils/node_metadata.c b/src/backend/distributed/utils/node_metadata.c index ecac82267..9121be569 100644 --- a/src/backend/distributed/utils/node_metadata.c +++ b/src/backend/distributed/utils/node_metadata.c @@ -361,7 +361,7 @@ get_shard_id_for_distribution_column(PG_FUNCTION_ARGS) inputDataType = get_fn_expr_argtype(fcinfo->flinfo, 1); distributionValueString = DatumToString(inputDatum, inputDataType); - distributionColumn = PartitionKey(relationId); + distributionColumn = DistPartitionKey(relationId); distributionDataType = distributionColumn->vartype; distributionValueDatum = StringToDatum(distributionValueString, @@ -625,9 +625,9 @@ SetNodeState(char *nodeName, int32 nodePort, bool isActive) replace[Anum_pg_dist_node_isactive - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); - simple_heap_update(pgDistNode, &heapTuple->t_self, heapTuple); - CatalogUpdateIndexes(pgDistNode, heapTuple); + CatalogTupleUpdate(pgDistNode, &heapTuple->t_self, heapTuple); + CitusInvalidateRelcacheByRelid(DistNodeRelationId()); CommandCounterIncrement(); @@ -868,8 +868,7 @@ InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, uint32 groupId, char * tupleDescriptor = RelationGetDescr(pgDistNode); heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - simple_heap_insert(pgDistNode, heapTuple); - CatalogUpdateIndexes(pgDistNode, heapTuple); + CatalogTupleInsert(pgDistNode, heapTuple); /* close relation and invalidate previous cache entry */ heap_close(pgDistNode, AccessExclusiveLock); diff --git a/src/backend/distributed/utils/ruleutils_10.c b/src/backend/distributed/utils/ruleutils_10.c new file mode 100644 index 000000000..159c38745 --- /dev/null +++ b/src/backend/distributed/utils/ruleutils_10.c @@ -0,0 +1,7745 @@ +/*------------------------------------------------------------------------- + * + * ruleutils_10.c + * Functions to convert stored expressions/querytrees back to + * source text + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/distributed/utils/ruleutils_10.c + * + * This needs to be closely in sync with the core code. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#if (PG_VERSION_NUM >= 100000) + +#include +#include +#include + +#include "access/amapi.h" +#include "access/htup_details.h" +#include "access/sysattr.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_aggregate.h" +#include "catalog/pg_am.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_language.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_partitioned_table.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/extension.h" +#include "commands/tablespace.h" +#include "common/keywords.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_ruleutils.h" +#include "executor/spi.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/tlist.h" +#include "parser/parse_node.h" +#include "parser/parse_agg.h" +#include "parser/parse_func.h" +#include "parser/parse_node.h" +#include "parser/parse_oper.h" +#include "parser/parser.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rewriteSupport.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/tqual.h" +#include "utils/typcache.h" +#include "utils/varlena.h" +#include "utils/xml.h" + + +/* ---------- + * Pretty formatting constants + * ---------- + */ + +/* Indent counts */ +#define PRETTYINDENT_STD 8 +#define PRETTYINDENT_JOIN 4 +#define PRETTYINDENT_VAR 4 + +#define PRETTYINDENT_LIMIT 40 /* wrap limit */ + +/* Pretty flags */ +#define PRETTYFLAG_PAREN 1 +#define PRETTYFLAG_INDENT 2 + +/* Default line length for pretty-print wrapping: 0 means wrap always */ +#define WRAP_COLUMN_DEFAULT 0 + +/* macro to test if pretty action needed */ +#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) +#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) + + +/* ---------- + * Local data types + * ---------- + */ + +/* Context info needed for invoking a recursive querytree display routine */ +typedef struct +{ + StringInfo buf; /* output buffer to append to */ + List *namespaces; /* List of deparse_namespace nodes */ + List *windowClause; /* Current query level's WINDOW clause */ + List *windowTList; /* targetlist for resolving WINDOW clause */ + int prettyFlags; /* enabling of pretty-print functions */ + int wrapColumn; /* max line length, or -1 for no limit */ + int indentLevel; /* current indent level for prettyprint */ + bool varprefix; /* TRUE to print prefixes on Vars */ + Oid distrelid; /* the distributed table being modified, if valid */ + int64 shardid; /* a distributed table's shardid, if positive */ + ParseExprKind special_exprkind; /* set only for exprkinds needing + * special handling */ +} deparse_context; + +/* + * Each level of query context around a subtree needs a level of Var namespace. + * A Var having varlevelsup=N refers to the N'th item (counting from 0) in + * the current context's namespaces list. + * + * The rangetable is the list of actual RTEs from the query tree, and the + * cte list is the list of actual CTEs. + * + * rtable_names holds the alias name to be used for each RTE (either a C + * string, or NULL for nameless RTEs such as unnamed joins). + * rtable_columns holds the column alias names to be used for each RTE. + * + * In some cases we need to make names of merged JOIN USING columns unique + * across the whole query, not only per-RTE. If so, unique_using is TRUE + * and using_names is a list of C strings representing names already assigned + * to USING columns. + * + * When deparsing plan trees, there is always just a single item in the + * deparse_namespace list (since a plan tree never contains Vars with + * varlevelsup > 0). We store the PlanState node that is the immediate + * parent of the expression to be deparsed, as well as a list of that + * PlanState's ancestors. In addition, we store its outer and inner subplan + * state nodes, as well as their plan nodes' targetlists, and the index tlist + * if the current plan node might contain INDEX_VAR Vars. (These fields could + * be derived on-the-fly from the current PlanState, but it seems notationally + * clearer to set them up as separate fields.) + */ +typedef struct +{ + List *rtable; /* List of RangeTblEntry nodes */ + List *rtable_names; /* Parallel list of names for RTEs */ + List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *ctes; /* List of CommonTableExpr nodes */ + /* Workspace for column alias assignment: */ + bool unique_using; /* Are we making USING names globally unique */ + List *using_names; /* List of assigned names for USING columns */ + /* Remaining fields are used only when deparsing a Plan tree: */ + PlanState *planstate; /* immediate parent of current expression */ + List *ancestors; /* ancestors of planstate */ + PlanState *outer_planstate; /* outer subplan state, or NULL if none */ + PlanState *inner_planstate; /* inner subplan state, or NULL if none */ + List *outer_tlist; /* referent for OUTER_VAR Vars */ + List *inner_tlist; /* referent for INNER_VAR Vars */ + List *index_tlist; /* referent for INDEX_VAR Vars */ +} deparse_namespace; + +/* + * Per-relation data about column alias names. + * + * Selecting aliases is unreasonably complicated because of the need to dump + * rules/views whose underlying tables may have had columns added, deleted, or + * renamed since the query was parsed. We must nonetheless print the rule/view + * in a form that can be reloaded and will produce the same results as before. + * + * For each RTE used in the query, we must assign column aliases that are + * unique within that RTE. SQL does not require this of the original query, + * but due to factors such as *-expansion we need to be able to uniquely + * reference every column in a decompiled query. As long as we qualify all + * column references, per-RTE uniqueness is sufficient for that. + * + * However, we can't ensure per-column name uniqueness for unnamed join RTEs, + * since they just inherit column names from their input RTEs, and we can't + * rename the columns at the join level. Most of the time this isn't an issue + * because we don't need to reference the join's output columns as such; we + * can reference the input columns instead. That approach can fail for merged + * JOIN USING columns, however, so when we have one of those in an unnamed + * join, we have to make that column's alias globally unique across the whole + * query to ensure it can be referenced unambiguously. + * + * Another problem is that a JOIN USING clause requires the columns to be + * merged to have the same aliases in both input RTEs, and that no other + * columns in those RTEs or their children conflict with the USING names. + * To handle that, we do USING-column alias assignment in a recursive + * traversal of the query's jointree. When descending through a JOIN with + * USING, we preassign the USING column names to the child columns, overriding + * other rules for column alias assignment. We also mark each RTE with a list + * of all USING column names selected for joins containing that RTE, so that + * when we assign other columns' aliases later, we can avoid conflicts. + * + * Another problem is that if a JOIN's input tables have had columns added or + * deleted since the query was parsed, we must generate a column alias list + * for the join that matches the current set of input columns --- otherwise, a + * change in the number of columns in the left input would throw off matching + * of aliases to columns of the right input. Thus, positions in the printable + * column alias list are not necessarily one-for-one with varattnos of the + * JOIN, so we need a separate new_colnames[] array for printing purposes. + */ +typedef struct +{ + /* + * colnames is an array containing column aliases to use for columns that + * existed when the query was parsed. Dropped columns have NULL entries. + * This array can be directly indexed by varattno to get a Var's name. + * + * Non-NULL entries are guaranteed unique within the RTE, *except* when + * this is for an unnamed JOIN RTE. In that case we merely copy up names + * from the two input RTEs. + * + * During the recursive descent in set_using_names(), forcible assignment + * of a child RTE's column name is represented by pre-setting that element + * of the child's colnames array. So at that stage, NULL entries in this + * array just mean that no name has been preassigned, not necessarily that + * the column is dropped. + */ + int num_cols; /* length of colnames[] array */ + char **colnames; /* array of C strings and NULLs */ + + /* + * new_colnames is an array containing column aliases to use for columns + * that would exist if the query was re-parsed against the current + * definitions of its base tables. This is what to print as the column + * alias list for the RTE. This array does not include dropped columns, + * but it will include columns added since original parsing. Indexes in + * it therefore have little to do with current varattno values. As above, + * entries are unique unless this is for an unnamed JOIN RTE. (In such an + * RTE, we never actually print this array, but we must compute it anyway + * for possible use in computing column names of upper joins.) The + * parallel array is_new_col marks which of these columns are new since + * original parsing. Entries with is_new_col false must match the + * non-NULL colnames entries one-for-one. + */ + int num_new_cols; /* length of new_colnames[] array */ + char **new_colnames; /* array of C strings */ + bool *is_new_col; /* array of bool flags */ + + /* This flag tells whether we should actually print a column alias list */ + bool printaliases; + + /* This list has all names used as USING names in joins above this RTE */ + List *parentUsing; /* names assigned to parent merged columns */ + + /* + * If this struct is for a JOIN RTE, we fill these fields during the + * set_using_names() pass to describe its relationship to its child RTEs. + * + * leftattnos and rightattnos are arrays with one entry per existing + * output column of the join (hence, indexable by join varattno). For a + * simple reference to a column of the left child, leftattnos[i] is the + * child RTE's attno and rightattnos[i] is zero; and conversely for a + * column of the right child. But for merged columns produced by JOIN + * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. + * Also, if the column has been dropped, both are zero. + * + * If it's a JOIN USING, usingNames holds the alias names selected for the + * merged columns (these might be different from the original USING list, + * if we had to modify names to achieve uniqueness). + */ + int leftrti; /* rangetable index of left child */ + int rightrti; /* rangetable index of right child */ + int *leftattnos; /* left-child varattnos of join cols, or 0 */ + int *rightattnos; /* right-child varattnos of join cols, or 0 */ + List *usingNames; /* names assigned to merged columns */ +} deparse_columns; + +/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ +#define deparse_columns_fetch(rangetable_index, dpns) \ + ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) + +/* + * Entry in set_rtable_names' hash table + */ +typedef struct +{ + char name[NAMEDATALEN]; /* Hash key --- must be first */ + int counter; /* Largest addition used so far for name */ +} NameHashEntry; + + +/* ---------- + * Local functions + * + * Most of these functions used to use fixed-size buffers to build their + * results. Now, they take an (already initialized) StringInfo object + * as a parameter, and append their text output to its contents. + * ---------- + */ +static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used); +static void set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces); +static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); +static void set_using_names(deparse_namespace *dpns, Node *jtnode, + List *parentUsing); +static void set_relation_column_names(deparse_namespace *dpns, + RangeTblEntry *rte, + deparse_columns *colinfo); +static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo); +static bool colname_is_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static char *make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static void expand_colnames_array_to(deparse_columns *colinfo, int n); +static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo); +static void flatten_join_using_qual(Node *qual, + List **leftvars, List **rightvars); +static char *get_rtable_name(int rtindex, deparse_context *context); +static void set_deparse_planstate(deparse_namespace *dpns, PlanState *ps); +static void push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns); +static void pop_child_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns); +static void pop_ancestor_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent); +static void get_query_def_extended(Query *query, StringInfo buf, + List *parentnamespace, Oid distrelid, int64 shardid, + TupleDesc resultDesc, int prettyFlags, int wrapColumn, + int startIndent); +static void get_values_def(List *values_lists, deparse_context *context); +static void get_with_clause(Query *query, deparse_context *context); +static void get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_insert_query_def(Query *query, deparse_context *context); +static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); +static void get_delete_query_def(Query *query, deparse_context *context); +static void get_utility_query_def(Query *query, deparse_context *context); +static void get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc); +static void get_setop_query(Node *setOp, Query *query, + deparse_context *context, + TupleDesc resultDesc); +static Node *get_rule_sortgroupclause(Index ref, List *tlist, + bool force_colno, + deparse_context *context); +static void get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context); +static void get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context); +static void get_rule_windowclause(Query *query, deparse_context *context); +static void get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context); +static char *get_variable(Var *var, int levelsup, bool istoplevel, + deparse_context *context); +static void get_special_variable(Node *node, deparse_context *context, + void *private); +static void resolve_special_varno(Node *node, deparse_context *context, + void *private, + void (*callback) (Node *, deparse_context *, void *)); +static Node *find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p); +static void get_parameter(Param *param, deparse_context *context); +static const char *get_simple_binary_op_name(OpExpr *expr); +static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); +static void appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus); +static void removeStringInfoSpaces(StringInfo str); +static void get_rule_expr(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit); +static void get_oper_expr(OpExpr *expr, deparse_context *context); +static void get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit); +static void get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref); +static void get_agg_combine_expr(Node *node, deparse_context *context, + void *private); +static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static void get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode); +static void get_const_expr(Const *constval, deparse_context *context, + int showtype); +static void get_const_collation(Const *constval, deparse_context *context); +static void simple_quote_literal(StringInfo buf, const char *val); +static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); +static void get_from_clause(Query *query, const char *prefix, + deparse_context *context); +static void get_from_clause_item(Node *jtnode, Query *query, + deparse_context *context); +static void get_column_alias_list(deparse_columns *colinfo, + deparse_context *context); +static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context); +static void get_tablesample_def(TableSampleClause *tablesample, + deparse_context *context); +static void get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf); +static Node *processIndirection(Node *node, deparse_context *context); +static void printSubscripts(ArrayRef *aref, deparse_context *context); +static char *get_relation_name(Oid relid); +static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, + int64 shardid, List *namespaces); +static char *generate_fragment_name(char *schemaName, char *tableName); +static char *generate_function_name(Oid funcid, int nargs, + List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind); +static char *generate_operator_name(Oid operid, Oid arg1, Oid arg2); + +#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") + + + +/* + * pg_get_query_def parses back one query tree, and outputs the resulting query + * string into given buffer. + */ +void +pg_get_query_def(Query *query, StringInfo buffer) +{ + get_query_def(query, buffer, NIL, NULL, 0, WRAP_COLUMN_DEFAULT, 0); +} + + +/* + * set_rtable_names: select RTE aliases to be used in printing a query + * + * We fill in dpns->rtable_names with a list of names that is one-for-one with + * the already-filled dpns->rtable list. Each RTE name is unique among those + * in the new namespace plus any ancestor namespaces listed in + * parent_namespaces. + * + * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. + * + * Note that this function is only concerned with relation names, not column + * names. + */ +static void +set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used) +{ + HASHCTL hash_ctl; + HTAB *names_hash; + NameHashEntry *hentry; + bool found; + int rtindex; + ListCell *lc; + + dpns->rtable_names = NIL; + /* nothing more to do if empty rtable */ + if (dpns->rtable == NIL) + return; + + /* + * We use a hash table to hold known names, so that this process is O(N) + * not O(N^2) for N names. + */ + MemSet(&hash_ctl, 0, sizeof(hash_ctl)); + hash_ctl.keysize = NAMEDATALEN; + hash_ctl.entrysize = sizeof(NameHashEntry); + hash_ctl.hcxt = CurrentMemoryContext; + names_hash = hash_create("set_rtable_names names", + list_length(dpns->rtable), + &hash_ctl, + HASH_ELEM | HASH_CONTEXT); + /* Preload the hash table with names appearing in parent_namespaces */ + foreach(lc, parent_namespaces) + { + deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, olddpns->rtable_names) + { + char *oldname = (char *) lfirst(lc2); + + if (oldname == NULL) + continue; + hentry = (NameHashEntry *) hash_search(names_hash, + oldname, + HASH_ENTER, + &found); + /* we do not complain about duplicate names in parent namespaces */ + hentry->counter = 0; + } + } + + /* Now we can scan the rtable */ + rtindex = 1; + foreach(lc, dpns->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + char *refname; + + /* Just in case this takes an unreasonable amount of time ... */ + CHECK_FOR_INTERRUPTS(); + + if (rels_used && !bms_is_member(rtindex, rels_used)) + { + /* Ignore unreferenced RTE */ + refname = NULL; + } + else if (rte->alias) + { + /* If RTE has a user-defined alias, prefer that */ + refname = rte->alias->aliasname; + } + else if (rte->rtekind == RTE_RELATION) + { + /* Use the current actual name of the relation */ + refname = get_rel_name(rte->relid); + } + else if (rte->rtekind == RTE_JOIN) + { + /* Unnamed join has no refname */ + refname = NULL; + } + else + { + /* Otherwise use whatever the parser assigned */ + refname = rte->eref->aliasname; + } + + /* + * If the selected name isn't unique, append digits to make it so, and + * make a new hash entry for it once we've got a unique name. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (refname) + { + hentry = (NameHashEntry *) hash_search(names_hash, + refname, + HASH_ENTER, + &found); + if (found) + { + /* Name already in use, must choose a new one */ + int refnamelen = strlen(refname); + char *modname = (char *) palloc(refnamelen + 16); + NameHashEntry *hentry2; + + do + { + hentry->counter++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave + * if the data is not valid in what libc thinks is the + * prevailing encoding. + */ + memcpy(modname, refname, refnamelen); + sprintf(modname + refnamelen, "_%d", hentry->counter); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from refname to keep all the digits */ + refnamelen = pg_mbcliplen(refname, refnamelen, + refnamelen - 1); + } + hentry2 = (NameHashEntry *) hash_search(names_hash, + modname, + HASH_ENTER, + &found); + } while (found); + hentry2->counter = 0; /* init new hash entry */ + refname = modname; + } + else + { + /* Name not previously used, need only initialize hentry */ + hentry->counter = 0; + } + } + + dpns->rtable_names = lappend(dpns->rtable_names, refname); + rtindex++; + } + + hash_destroy(names_hash); +} + +/* + * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree + * + * For convenience, this is defined to initialize the deparse_namespace struct + * from scratch. + */ +static void +set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize *dpns and fill rtable/ctes links */ + memset(dpns, 0, sizeof(deparse_namespace)); + dpns->rtable = query->rtable; + dpns->ctes = query->cteList; + + /* Assign a unique relation alias to each RTE */ + set_rtable_names(dpns, parent_namespaces, NULL); + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* If it's a utility query, it won't have a jointree */ + if (query->jointree) + { + /* Detect whether global uniqueness of USING names is needed */ + dpns->unique_using = + has_dangerous_join_using(dpns, (Node *) query->jointree); + + /* + * Select names for columns merged by USING, via a recursive pass over + * the query jointree. + */ + set_using_names(dpns, (Node *) query->jointree, NIL); + } + + /* + * Now assign remaining column aliases for each RTE. We do this in a + * linear scan of the rtable, so as to process RTEs whether or not they + * are in the jointree (we mustn't miss NEW.*, INSERT target relations, + * etc). JOIN RTEs must be processed after their children, but this is + * okay because they appear later in the rtable list than their children + * (cf Asserts in identify_join_columns()). + */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + if (rte->rtekind == RTE_JOIN) + set_join_column_names(dpns, rte, colinfo); + else + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * has_dangerous_join_using: search jointree for unnamed JOIN USING + * + * Merged columns of a JOIN USING may act differently from either of the input + * columns, either because they are merged with COALESCE (in a FULL JOIN) or + * because an implicit coercion of the underlying input column is required. + * In such a case the column must be referenced as a column of the JOIN not as + * a column of either input. And this is problematic if the join is unnamed + * (alias-less): we cannot qualify the column's name with an RTE name, since + * there is none. (Forcibly assigning an alias to the join is not a solution, + * since that will prevent legal references to tables below the join.) + * To ensure that every column in the query is unambiguously referenceable, + * we must assign such merged columns names that are globally unique across + * the whole query, aliasing other columns out of the way as necessary. + * + * Because the ensuing re-aliasing is fairly damaging to the readability of + * the query, we don't do this unless we have to. So, we must pre-scan + * the join tree to see if we have to, before starting set_using_names(). + */ +static bool +has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do here */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + { + if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) + return true; + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + /* Is it an unnamed JOIN with USING? */ + if (j->alias == NULL && j->usingClause) + { + /* + * Yes, so check each join alias var to see if any of them are not + * simple references to underlying columns. If so, we have a + * dangerous situation and must pick unique aliases. + */ + RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); + ListCell *lc; + + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + if (aliasvar != NULL && !IsA(aliasvar, Var)) + return true; + } + } + + /* Nope, but inspect children */ + if (has_dangerous_join_using(dpns, j->larg)) + return true; + if (has_dangerous_join_using(dpns, j->rarg)) + return true; + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + return false; +} + +/* + * set_using_names: select column aliases to be used for merged USING columns + * + * We do this during a recursive descent of the query jointree. + * dpns->unique_using must already be set to determine the global strategy. + * + * Column alias info is saved in the dpns->rtable_columns list, which is + * assumed to be filled with pre-zeroed deparse_columns structs. + * + * parentUsing is a list of all USING aliases assigned in parent joins of + * the current jointree node. (The passed-in list must not be modified.) + */ +static void +set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do now */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + set_using_names(dpns, (Node *) lfirst(lc), parentUsing); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + int *leftattnos; + int *rightattnos; + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + int i; + ListCell *lc; + + /* Get info about the shape of the join */ + identify_join_columns(j, rte, colinfo); + leftattnos = colinfo->leftattnos; + rightattnos = colinfo->rightattnos; + + /* Look up the not-yet-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * If this join is unnamed, then we cannot substitute new aliases at + * this level, so any name requirements pushed down to here must be + * pushed down again to the children. + */ + if (rte->alias == NULL) + { + for (i = 0; i < colinfo->num_cols; i++) + { + char *colname = colinfo->colnames[i]; + + if (colname == NULL) + continue; + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + } + } + + /* + * If there's a USING clause, select the USING column names and push + * those names down to the children. We have two strategies: + * + * If dpns->unique_using is TRUE, we force all USING names to be + * unique across the whole query level. In principle we'd only need + * the names of dangerous USING columns to be globally unique, but to + * safely assign all USING names in a single pass, we have to enforce + * the same uniqueness rule for all of them. However, if a USING + * column's name has been pushed down from the parent, we should use + * it as-is rather than making a uniqueness adjustment. This is + * necessary when we're at an unnamed join, and it creates no risk of + * ambiguity. Also, if there's a user-written output alias for a + * merged column, we prefer to use that rather than the input name; + * this simplifies the logic and seems likely to lead to less aliasing + * overall. + * + * If dpns->unique_using is FALSE, we only need USING names to be + * unique within their own join RTE. We still need to honor + * pushed-down names, though. + * + * Though significantly different in results, these two strategies are + * implemented by the same code, with only the difference of whether + * to put assigned names into dpns->using_names. + */ + if (j->usingClause) + { + /* Copy the input parentUsing list so we don't modify it */ + parentUsing = list_copy(parentUsing); + + /* USING names must correspond to the first join output columns */ + expand_colnames_array_to(colinfo, list_length(j->usingClause)); + i = 0; + foreach(lc, j->usingClause) + { + char *colname = strVal(lfirst(lc)); + + /* Assert it's a merged column */ + Assert(leftattnos[i] != 0 && rightattnos[i] != 0); + + /* Adopt passed-down name if any, else select unique name */ + if (colinfo->colnames[i] != NULL) + colname = colinfo->colnames[i]; + else + { + /* Prefer user-written output alias if any */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + /* Make it appropriately unique */ + colname = make_colname_unique(colname, dpns, colinfo); + if (dpns->unique_using) + dpns->using_names = lappend(dpns->using_names, + colname); + /* Save it as output column name, too */ + colinfo->colnames[i] = colname; + } + + /* Remember selected names for use later */ + colinfo->usingNames = lappend(colinfo->usingNames, colname); + parentUsing = lappend(parentUsing, colname); + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + + i++; + } + } + + /* Mark child deparse_columns structs with correct parentUsing info */ + leftcolinfo->parentUsing = parentUsing; + rightcolinfo->parentUsing = parentUsing; + + /* Now recursively assign USING column names in children */ + set_using_names(dpns, j->larg, parentUsing); + set_using_names(dpns, j->rarg, parentUsing); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * set_relation_column_names: select column aliases for a non-join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. + */ +static void +set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + int ncolumns; + char **real_colnames; + bool changed_any; + int noldcolumns; + int i; + int j; + + /* + * Extract the RTE's "real" column names. This is comparable to + * get_rte_attribute_name, except that it's important to disregard dropped + * columns. We put NULL into the array for a dropped column. + */ + if (rte->rtekind == RTE_RELATION) + { + /* Relation --- look to the system catalogs for up-to-date info */ + Relation rel; + TupleDesc tupdesc; + + rel = relation_open(rte->relid, AccessShareLock); + tupdesc = RelationGetDescr(rel); + + ncolumns = tupdesc->natts; + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + for (i = 0; i < ncolumns; i++) + { + if (tupdesc->attrs[i]->attisdropped) + real_colnames[i] = NULL; + else + real_colnames[i] = pstrdup(NameStr(tupdesc->attrs[i]->attname)); + } + relation_close(rel, AccessShareLock); + } + else + { + /* Otherwise use the column names from eref */ + ListCell *lc; + + ncolumns = list_length(rte->eref->colnames); + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + i = 0; + foreach(lc, rte->eref->colnames) + { + /* + * If the column name shown in eref is an empty string, then it's + * a column that was dropped at the time of parsing the query, so + * treat it as dropped. + */ + char *cname = strVal(lfirst(lc)); + + if (cname[0] == '\0') + cname = NULL; + real_colnames[i] = cname; + i++; + } + } + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that there are now more columns than there were when the + * query was parsed, ie colnames could be longer than rte->eref->colnames. + * We must assign unique aliases to the new columns too, else there could + * be unresolved conflicts when the view/rule is reloaded. + */ + expand_colnames_array_to(colinfo, ncolumns); + Assert(colinfo->num_cols == ncolumns); + + /* + * Make sufficiently large new_colnames and is_new_col arrays, too. + * + * Note: because we leave colinfo->num_new_cols zero until after the loop, + * colname_is_unique will not consult that array, which is fine because it + * would only be duplicate effort. + */ + colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); + + /* + * Scan the columns, select a unique alias for each one, and store it in + * colinfo->colnames and colinfo->new_colnames. The former array has NULL + * entries for dropped columns, the latter omits them. Also mark + * new_colnames entries as to whether they are new since parse time; this + * is the case for entries beyond the length of rte->eref->colnames. + */ + noldcolumns = list_length(rte->eref->colnames); + changed_any = false; + j = 0; + for (i = 0; i < ncolumns; i++) + { + char *real_colname = real_colnames[i]; + char *colname = colinfo->colnames[i]; + + /* Skip dropped columns */ + if (real_colname == NULL) + { + Assert(colname == NULL); /* colnames[i] is already NULL */ + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Put names of non-dropped columns in new_colnames[] too */ + colinfo->new_colnames[j] = colname; + /* And mark them as new or not */ + colinfo->is_new_col[j] = (i >= noldcolumns); + j++; + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Set correct length for new_colnames[] array. (Note: if columns have + * been added, colinfo->num_cols includes them, which is not really quite + * right but is harmless, since any new columns must be at the end where + * they won't affect varattnos of pre-existing columns.) + */ + colinfo->num_new_cols = j; + + /* + * For a relation RTE, we need only print the alias column names if any + * are different from the underlying "real" names. For a function RTE, + * always emit a complete column alias list; this is to protect against + * possible instability of the default column names (eg, from altering + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). + */ + if (rte->rtekind == RTE_RELATION) + colinfo->printaliases = changed_any; + else if (rte->rtekind == RTE_FUNCTION) + colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; + else if (rte->alias && rte->alias->colnames != NIL) + colinfo->printaliases = true; + else + colinfo->printaliases = changed_any; +} + +/* + * set_join_column_names: select column aliases for a join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. Also, names for USING columns were already chosen by + * set_using_names(). We further expect that column alias selection has been + * completed for both input RTEs. + */ +static void +set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + bool changed_any; + int noldcolumns; + int nnewcolumns; + Bitmapset *leftmerged = NULL; + Bitmapset *rightmerged = NULL; + int i; + int j; + int ic; + int jc; + + /* Look up the previously-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that one or both inputs now have more columns than there + * were when the query was parsed, but we'll deal with that below. We + * only need entries in colnames for pre-existing columns. + */ + noldcolumns = list_length(rte->eref->colnames); + expand_colnames_array_to(colinfo, noldcolumns); + Assert(colinfo->num_cols == noldcolumns); + + /* + * Scan the join output columns, select an alias for each one, and store + * it in colinfo->colnames. If there are USING columns, set_using_names() + * already selected their names, so we can start the loop at the first + * non-merged column. + */ + changed_any = false; + for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) + { + char *colname = colinfo->colnames[i]; + char *real_colname; + + /* Ignore dropped column (only possible for non-merged column) */ + if (colinfo->leftattnos[i] == 0 && colinfo->rightattnos[i] == 0) + { + Assert(colname == NULL); + continue; + } + + /* Get the child column name */ + if (colinfo->leftattnos[i] > 0) + real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; + else if (colinfo->rightattnos[i] > 0) + real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; + else + { + /* We're joining system columns --- use eref name */ + real_colname = strVal(list_nth(rte->eref->colnames, i)); + } + Assert(real_colname != NULL); + + /* In an unnamed join, just report child column names as-is */ + if (rte->alias == NULL) + { + colinfo->colnames[i] = real_colname; + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Calculate number of columns the join would have if it were re-parsed + * now, and create storage for the new_colnames and is_new_col arrays. + * + * Note: colname_is_unique will be consulting new_colnames[] during the + * loops below, so its not-yet-filled entries must be zeroes. + */ + nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - + list_length(colinfo->usingNames); + colinfo->num_new_cols = nnewcolumns; + colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); + + /* + * Generating the new_colnames array is a bit tricky since any new columns + * added since parse time must be inserted in the right places. This code + * must match the parser, which will order a join's columns as merged + * columns first (in USING-clause order), then non-merged columns from the + * left input (in attnum order), then non-merged columns from the right + * input (ditto). If one of the inputs is itself a join, its columns will + * be ordered according to the same rule, which means newly-added columns + * might not be at the end. We can figure out what's what by consulting + * the leftattnos and rightattnos arrays plus the input is_new_col arrays. + * + * In these loops, i indexes leftattnos/rightattnos (so it's join varattno + * less one), j indexes new_colnames/is_new_col, and ic/jc have similar + * meanings for the current child RTE. + */ + + /* Handle merged columns; they are first and can't be new */ + i = j = 0; + while (i < noldcolumns && + colinfo->leftattnos[i] != 0 && + colinfo->rightattnos[i] != 0) + { + /* column name is already determined and known unique */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + colinfo->is_new_col[j] = false; + + /* build bitmapsets of child attnums of merged columns */ + if (colinfo->leftattnos[i] > 0) + leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); + if (colinfo->rightattnos[i] > 0) + rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); + + i++, j++; + } + + /* Handle non-merged left-child columns */ + ic = 0; + for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) + { + char *child_colname = leftcolinfo->new_colnames[jc]; + + if (!leftcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of left child */ + while (ic < leftcolinfo->num_cols && + leftcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < leftcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, leftmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->leftattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; + j++; + } + + /* Handle non-merged right-child columns in exactly the same way */ + ic = 0; + for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) + { + char *child_colname = rightcolinfo->new_colnames[jc]; + + if (!rightcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of right child */ + while (ic < rightcolinfo->num_cols && + rightcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < rightcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, rightmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->rightattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; + j++; + } + + /* Assert we processed the right number of columns */ +#ifdef USE_ASSERT_CHECKING + while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) + i++; + Assert(i == colinfo->num_cols); + Assert(j == nnewcolumns); +#endif + + /* + * For a named join, print column aliases if we changed any from the child + * names. Unnamed joins cannot print aliases. + */ + if (rte->alias != NULL) + colinfo->printaliases = changed_any; + else + colinfo->printaliases = false; +} + +/* + * colname_is_unique: is colname distinct from already-chosen column names? + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static bool +colname_is_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + int i; + ListCell *lc; + + /* Check against already-assigned column aliases within RTE */ + for (i = 0; i < colinfo->num_cols; i++) + { + char *oldname = colinfo->colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* + * If we're building a new_colnames array, check that too (this will be + * partially but not completely redundant with the previous checks) + */ + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *oldname = colinfo->new_colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against USING-column names that must be globally unique */ + foreach(lc, dpns->using_names) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against names already assigned for parent-join USING cols */ + foreach(lc, colinfo->parentUsing) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + return true; +} + +/* + * make_colname_unique: modify colname if necessary to make it unique + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static char * +make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + /* + * If the selected name isn't unique, append digits to make it so. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (!colname_is_unique(colname, dpns, colinfo)) + { + int colnamelen = strlen(colname); + char *modname = (char *) palloc(colnamelen + 16); + int i = 0; + + do + { + i++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave if the + * data is not valid in what libc thinks is the prevailing + * encoding. + */ + memcpy(modname, colname, colnamelen); + sprintf(modname + colnamelen, "_%d", i); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from colname to keep all the digits */ + colnamelen = pg_mbcliplen(colname, colnamelen, + colnamelen - 1); + } + } while (!colname_is_unique(modname, dpns, colinfo)); + colname = modname; + } + return colname; +} + +/* + * expand_colnames_array_to: make colinfo->colnames at least n items long + * + * Any added array entries are initialized to zero. + */ +static void +expand_colnames_array_to(deparse_columns *colinfo, int n) +{ + if (n > colinfo->num_cols) + { + if (colinfo->colnames == NULL) + colinfo->colnames = (char **) palloc0(n * sizeof(char *)); + else + { + colinfo->colnames = (char **) repalloc(colinfo->colnames, + n * sizeof(char *)); + memset(colinfo->colnames + colinfo->num_cols, 0, + (n - colinfo->num_cols) * sizeof(char *)); + } + colinfo->num_cols = n; + } +} + +/* + * identify_join_columns: figure out where columns of a join come from + * + * Fills the join-specific fields of the colinfo struct, except for + * usingNames which is filled later. + */ +static void +identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo) +{ + int numjoincols; + int i; + ListCell *lc; + + /* Extract left/right child RT indexes */ + if (IsA(j->larg, RangeTblRef)) + colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; + else if (IsA(j->larg, JoinExpr)) + colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->larg)); + if (IsA(j->rarg, RangeTblRef)) + colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; + else if (IsA(j->rarg, JoinExpr)) + colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->rarg)); + + /* Assert children will be processed earlier than join in second pass */ + Assert(colinfo->leftrti < j->rtindex); + Assert(colinfo->rightrti < j->rtindex); + + /* Initialize result arrays with zeroes */ + numjoincols = list_length(jrte->joinaliasvars); + Assert(numjoincols == list_length(jrte->eref->colnames)); + colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); + colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); + + /* Scan the joinaliasvars list to identify simple column references */ + i = 0; + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + /* get rid of any implicit coercion above the Var */ + aliasvar = (Var *) strip_implicit_coercions((Node *) aliasvar); + + if (aliasvar == NULL) + { + /* It's a dropped column; nothing to do here */ + } + else if (IsA(aliasvar, Var)) + { + Assert(aliasvar->varlevelsup == 0); + Assert(aliasvar->varattno != 0); + if (aliasvar->varno == colinfo->leftrti) + colinfo->leftattnos[i] = aliasvar->varattno; + else if (aliasvar->varno == colinfo->rightrti) + colinfo->rightattnos[i] = aliasvar->varattno; + else + elog(ERROR, "unexpected varno %d in JOIN RTE", + aliasvar->varno); + } + else if (IsA(aliasvar, CoalesceExpr)) + { + /* + * It's a merged column in FULL JOIN USING. Ignore it for now and + * let the code below identify the merged columns. + */ + } + else + elog(ERROR, "unrecognized node type in join alias vars: %d", + (int) nodeTag(aliasvar)); + + i++; + } + + /* + * If there's a USING clause, deconstruct the join quals to identify the + * merged columns. This is a tad painful but if we cannot rely on the + * column names, there is no other representation of which columns were + * joined by USING. (Unless the join type is FULL, we can't tell from the + * joinaliasvars list which columns are merged.) Note: we assume that the + * merged columns are the first output column(s) of the join. + */ + if (j->usingClause) + { + List *leftvars = NIL; + List *rightvars = NIL; + ListCell *lc2; + + /* Extract left- and right-side Vars from the qual expression */ + flatten_join_using_qual(j->quals, &leftvars, &rightvars); + Assert(list_length(leftvars) == list_length(j->usingClause)); + Assert(list_length(rightvars) == list_length(j->usingClause)); + + /* Mark the output columns accordingly */ + i = 0; + forboth(lc, leftvars, lc2, rightvars) + { + Var *leftvar = (Var *) lfirst(lc); + Var *rightvar = (Var *) lfirst(lc2); + + Assert(leftvar->varlevelsup == 0); + Assert(leftvar->varattno != 0); + if (leftvar->varno != colinfo->leftrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + leftvar->varno); + colinfo->leftattnos[i] = leftvar->varattno; + + Assert(rightvar->varlevelsup == 0); + Assert(rightvar->varattno != 0); + if (rightvar->varno != colinfo->rightrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + rightvar->varno); + colinfo->rightattnos[i] = rightvar->varattno; + + i++; + } + } +} + +/* + * flatten_join_using_qual: extract Vars being joined from a JOIN/USING qual + * + * We assume that transformJoinUsingClause won't have produced anything except + * AND nodes, equality operator nodes, and possibly implicit coercions, and + * that the AND node inputs match left-to-right with the original USING list. + * + * Caller must initialize the result lists to NIL. + */ +static void +flatten_join_using_qual(Node *qual, List **leftvars, List **rightvars) +{ + if (IsA(qual, BoolExpr)) + { + /* Handle AND nodes by recursion */ + BoolExpr *b = (BoolExpr *) qual; + ListCell *lc; + + Assert(b->boolop == AND_EXPR); + foreach(lc, b->args) + { + flatten_join_using_qual((Node *) lfirst(lc), + leftvars, rightvars); + } + } + else if (IsA(qual, OpExpr)) + { + /* Otherwise we should have an equality operator */ + OpExpr *op = (OpExpr *) qual; + Var *var; + + if (list_length(op->args) != 2) + elog(ERROR, "unexpected unary operator in JOIN/USING qual"); + /* Arguments should be Vars with perhaps implicit coercions */ + var = (Var *) strip_implicit_coercions((Node *) linitial(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *leftvars = lappend(*leftvars, var); + var = (Var *) strip_implicit_coercions((Node *) lsecond(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *rightvars = lappend(*rightvars, var); + } + else + { + /* Perhaps we have an implicit coercion to boolean? */ + Node *q = strip_implicit_coercions(qual); + + if (q != qual) + flatten_join_using_qual(q, leftvars, rightvars); + else + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(qual)); + } +} + +/* + * get_rtable_name: convenience function to get a previously assigned RTE alias + * + * The RTE must belong to the topmost namespace level in "context". + */ +static char * +get_rtable_name(int rtindex, deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); + return (char *) list_nth(dpns->rtable_names, rtindex - 1); +} + +/* + * set_deparse_planstate: set up deparse_namespace to parse subexpressions + * of a given PlanState node + * + * This sets the planstate, outer_planstate, inner_planstate, outer_tlist, + * inner_tlist, and index_tlist fields. Caller is responsible for adjusting + * the ancestors list if necessary. Note that the rtable and ctes fields do + * not need to change when shifting attention to different plan nodes in a + * single plan tree. + */ +static void +set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) +{ + dpns->planstate = ps; + + /* + * We special-case Append and MergeAppend to pretend that the first child + * plan is the OUTER referent; we have to interpret OUTER Vars in their + * tlists according to one of the children, and the first one is the most + * natural choice. Likewise special-case ModifyTable to pretend that the + * first child plan is the OUTER referent; this is to support RETURNING + * lists containing references to non-target relations. + */ + if (IsA(ps, AppendState)) + dpns->outer_planstate = ((AppendState *) ps)->appendplans[0]; + else if (IsA(ps, MergeAppendState)) + dpns->outer_planstate = ((MergeAppendState *) ps)->mergeplans[0]; + else if (IsA(ps, ModifyTableState)) + dpns->outer_planstate = ((ModifyTableState *) ps)->mt_plans[0]; + else + dpns->outer_planstate = outerPlanState(ps); + + if (dpns->outer_planstate) + dpns->outer_tlist = dpns->outer_planstate->plan->targetlist; + else + dpns->outer_tlist = NIL; + + /* + * For a SubqueryScan, pretend the subplan is INNER referent. (We don't + * use OUTER because that could someday conflict with the normal meaning.) + * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). + */ + if (IsA(ps, SubqueryScanState)) + dpns->inner_planstate = ((SubqueryScanState *) ps)->subplan; + else if (IsA(ps, CteScanState)) + dpns->inner_planstate = ((CteScanState *) ps)->cteplanstate; + else if (IsA(ps, ModifyTableState)) + dpns->inner_planstate = ps; + else + dpns->inner_planstate = innerPlanState(ps); + + if (IsA(ps, ModifyTableState)) + dpns->inner_tlist = ((ModifyTableState *) ps)->mt_excludedtlist; + else if (dpns->inner_planstate) + dpns->inner_tlist = dpns->inner_planstate->plan->targetlist; + else + dpns->inner_tlist = NIL; + + /* Set up referent for INDEX_VAR Vars, if needed */ + if (IsA(ps->plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) ps->plan)->indextlist; + else if (IsA(ps->plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) ps->plan)->fdw_scan_tlist; + else if (IsA(ps->plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) ps->plan)->custom_scan_tlist; + else + dpns->index_tlist = NIL; +} + +/* + * push_child_plan: temporarily transfer deparsing attention to a child plan + * + * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the + * deparse context in case the referenced expression itself uses + * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid + * affecting levelsup issues (although in a Plan tree there really shouldn't + * be any). + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_child_plan. + */ +static void +push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns) +{ + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Link current plan node into ancestors list */ + dpns->ancestors = lcons(dpns->planstate, dpns->ancestors); + + /* Set attention on selected child */ + set_deparse_planstate(dpns, ps); +} + +/* + * pop_child_plan: undo the effects of push_child_plan + */ +static void +pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + List *ancestors; + + /* Get rid of ancestors list cell added by push_child_plan */ + ancestors = list_delete_first(dpns->ancestors); + + /* Restore fields changed by push_child_plan */ + *dpns = *save_dpns; + + /* Make sure dpns->ancestors is right (may be unnecessary) */ + dpns->ancestors = ancestors; +} + +/* + * push_ancestor_plan: temporarily transfer deparsing attention to an + * ancestor plan + * + * When expanding a Param reference, we must adjust the deparse context + * to match the plan node that contains the expression being printed; + * otherwise we'd fail if that expression itself contains a Param or + * OUTER_VAR/INNER_VAR/INDEX_VAR variable. + * + * The target ancestor is conveniently identified by the ListCell holding it + * in dpns->ancestors. + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_ancestor_plan. + */ +static void +push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns) +{ + PlanState *ps = (PlanState *) lfirst(ancestor_cell); + List *ancestors; + + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Build a new ancestor list with just this node's ancestors */ + ancestors = NIL; + while ((ancestor_cell = lnext(ancestor_cell)) != NULL) + ancestors = lappend(ancestors, lfirst(ancestor_cell)); + dpns->ancestors = ancestors; + + /* Set attention on selected ancestor */ + set_deparse_planstate(dpns, ps); +} + +/* + * pop_ancestor_plan: undo the effects of push_ancestor_plan + */ +static void +pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + /* Free the ancestor list made in push_ancestor_plan */ + list_free(dpns->ancestors); + + /* Restore fields changed by push_ancestor_plan */ + *dpns = *save_dpns; +} + + +/* ---------- + * deparse_shard_query - Parse back a query for execution on a shard + * + * Builds an SQL string to perform the provided query on a specific shard and + * places this string into the provided buffer. + * ---------- + */ +void +deparse_shard_query(Query *query, Oid distrelid, int64 shardid, + StringInfo buffer) +{ + get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, 0, + WRAP_COLUMN_DEFAULT, 0); +} + + +/* ---------- + * get_query_def - Parse back one query parsetree + * + * If resultDesc is not NULL, then it is the output tuple descriptor for + * the view represented by a SELECT query. + * ---------- + */ +static void +get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, + prettyFlags, wrapColumn, startIndent); +} + + +/* ---------- + * get_query_def_extended - Parse back one query parsetree, optionally + * with extension using a shard identifier. + * + * If distrelid is valid and shardid is positive, the provided shardid is added + * any time the provided relid is deparsed, so that the query may be executed + * on a placement for the given shard. + * ---------- + */ +static void +get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, + Oid distrelid, int64 shardid, TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + deparse_context context; + deparse_namespace dpns; + + OverrideSearchPath *overridePath = NULL; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Before we begin to examine the query, acquire locks on referenced + * relations, and fix up deleted columns in JOIN RTEs. This ensures + * consistent results. Note we assume it's OK to scribble on the passed + * querytree! + * + * We are only deparsing the query (we are not about to execute it), so we + * only need AccessShareLock on the relations it mentions. + */ + AcquireRewriteLocks(query, false, false); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buf; + context.namespaces = lcons(&dpns, list_copy(parentnamespace)); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = (parentnamespace != NIL || + list_length(query->rtable) != 1); + context.prettyFlags = prettyFlags; + context.wrapColumn = wrapColumn; + context.indentLevel = startIndent; + context.special_exprkind = EXPR_KIND_NONE; + context.distrelid = distrelid; + context.shardid = shardid; + + set_deparse_for_query(&dpns, query, parentnamespace); + + switch (query->commandType) + { + case CMD_SELECT: + get_select_query_def(query, &context, resultDesc); + break; + + case CMD_UPDATE: + get_update_query_def(query, &context); + break; + + case CMD_INSERT: + get_insert_query_def(query, &context); + break; + + case CMD_DELETE: + get_delete_query_def(query, &context); + break; + + case CMD_NOTHING: + appendStringInfoString(buf, "NOTHING"); + break; + + case CMD_UTILITY: + get_utility_query_def(query, &context); + break; + + default: + elog(ERROR, "unrecognized query command type: %d", + query->commandType); + break; + } + + /* revert back to original search_path */ + PopOverrideSearchPath(); +} + +/* ---------- + * get_values_def - Parse back a VALUES list + * ---------- + */ +static void +get_values_def(List *values_lists, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first_list = true; + ListCell *vtl; + + appendStringInfoString(buf, "VALUES "); + + foreach(vtl, values_lists) + { + List *sublist = (List *) lfirst(vtl); + bool first_col = true; + ListCell *lc; + + if (first_list) + first_list = false; + else + appendStringInfoString(buf, ", "); + + appendStringInfoChar(buf, '('); + foreach(lc, sublist) + { + Node *col = (Node *) lfirst(lc); + + if (first_col) + first_col = false; + else + appendStringInfoChar(buf, ','); + + /* + * Print the value. Whole-row Vars need special treatment. + */ + get_rule_expr_toplevel(col, context, false); + } + appendStringInfoChar(buf, ')'); + } +} + +/* ---------- + * get_with_clause - Parse back a WITH clause + * ---------- + */ +static void +get_with_clause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + if (query->cteList == NIL) + return; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + if (query->hasRecursive) + sep = "WITH RECURSIVE "; + else + sep = "WITH "; + foreach(l, query->cteList) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); + + appendStringInfoString(buf, sep); + appendStringInfoString(buf, quote_identifier(cte->ctename)); + if (cte->aliascolnames) + { + bool first = true; + ListCell *col; + + appendStringInfoChar(buf, '('); + foreach(col, cte->aliascolnames) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(col)))); + } + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, " AS ("); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + appendStringInfoChar(buf, ')'); + sep = ", "; + } + + if (PRETTY_INDENT(context)) + { + context->indentLevel -= PRETTYINDENT_STD; + appendContextKeyword(context, "", 0, 0, 0); + } + else + appendStringInfoChar(buf, ' '); +} + +/* ---------- + * get_select_query_def - Parse back a SELECT parsetree + * ---------- + */ +static void +get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + List *save_windowclause; + List *save_windowtlist; + bool force_colno; + ListCell *l; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* Set up context for possible window functions */ + save_windowclause = context->windowClause; + context->windowClause = query->windowClause; + save_windowtlist = context->windowTList; + context->windowTList = query->targetList; + + /* + * If the Query node has a setOperations tree, then it's the top level of + * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT + * fields are interesting in the top query itself. + */ + if (query->setOperations) + { + get_setop_query(query->setOperations, query, context, resultDesc); + /* ORDER BY clauses must be simple in this case */ + force_colno = true; + } + else + { + get_basic_select_query(query, context, resultDesc); + force_colno = false; + } + + /* Add the ORDER BY clause if given */ + if (query->sortClause != NIL) + { + appendContextKeyword(context, " ORDER BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_orderby(query->sortClause, query->targetList, + force_colno, context); + } + + /* Add the LIMIT clause if given */ + if (query->limitOffset != NULL) + { + appendContextKeyword(context, " OFFSET ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitOffset, context, false); + } + if (query->limitCount != NULL) + { + appendContextKeyword(context, " LIMIT ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + if (IsA(query->limitCount, Const) && + ((Const *) query->limitCount)->constisnull) + appendStringInfoString(buf, "ALL"); + else + get_rule_expr(query->limitCount, context, false); + } + + /* Add FOR [KEY] UPDATE/SHARE clauses if present */ + if (query->hasForUpdate) + { + foreach(l, query->rowMarks) + { + RowMarkClause *rc = (RowMarkClause *) lfirst(l); + + /* don't print implicit clauses */ + if (rc->pushedDown) + continue; + + switch (rc->strength) + { + case LCS_NONE: + /* we intentionally throw an error for LCS_NONE */ + elog(ERROR, "unrecognized LockClauseStrength %d", + (int) rc->strength); + break; + case LCS_FORKEYSHARE: + appendContextKeyword(context, " FOR KEY SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORSHARE: + appendContextKeyword(context, " FOR SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORNOKEYUPDATE: + appendContextKeyword(context, " FOR NO KEY UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORUPDATE: + appendContextKeyword(context, " FOR UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + } + + appendStringInfo(buf, " OF %s", + quote_identifier(get_rtable_name(rc->rti, + context))); + if (rc->waitPolicy == LockWaitError) + appendStringInfoString(buf, " NOWAIT"); + else if (rc->waitPolicy == LockWaitSkip) + appendStringInfoString(buf, " SKIP LOCKED"); + } + } + + context->windowClause = save_windowclause; + context->windowTList = save_windowtlist; +} + +/* + * Detect whether query looks like SELECT ... FROM VALUES(); + * if so, return the VALUES RTE. Otherwise return NULL. + */ +static RangeTblEntry * +get_simple_values_rte(Query *query) +{ + RangeTblEntry *result = NULL; + ListCell *lc; + + /* + * We want to return TRUE even if the Query also contains OLD or NEW rule + * RTEs. So the idea is to scan the rtable and see if there is only one + * inFromCl RTE that is a VALUES RTE. + */ + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + if (rte->rtekind == RTE_VALUES && rte->inFromCl) + { + if (result) + return NULL; /* multiple VALUES (probably not possible) */ + result = rte; + } + else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) + continue; /* ignore rule entries */ + else + return NULL; /* something else -> not simple VALUES */ + } + + /* + * We don't need to check the targetlist in any great detail, because + * parser/analyze.c will never generate a "bare" VALUES RTE --- they only + * appear inside auto-generated sub-queries with very restricted + * structure. However, DefineView might have modified the tlist by + * injecting new column aliases; so compare tlist resnames against the + * RTE's names to detect that. + */ + if (result) + { + ListCell *lcn; + + if (list_length(query->targetList) != list_length(result->eref->colnames)) + return NULL; /* this probably cannot happen */ + forboth(lc, query->targetList, lcn, result->eref->colnames) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *cname = strVal(lfirst(lcn)); + + if (tle->resjunk) + return NULL; /* this probably cannot happen */ + if (tle->resname == NULL || strcmp(tle->resname, cname) != 0) + return NULL; /* column name has been changed */ + } + } + + return result; +} + +static void +get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + RangeTblEntry *values_rte; + char *sep; + ListCell *l; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + /* + * If the query looks like SELECT * FROM (VALUES ...), then print just the + * VALUES part. This reverses what transformValuesClause() did at parse + * time. + */ + values_rte = get_simple_values_rte(query); + if (values_rte) + { + get_values_def(values_rte->values_lists, context); + return; + } + + /* + * Build up the query string - first we say SELECT + */ + appendStringInfoString(buf, "SELECT"); + + /* Add the DISTINCT clause if given */ + if (query->distinctClause != NIL) + { + if (query->hasDistinctOn) + { + appendStringInfoString(buf, " DISTINCT ON ("); + sep = ""; + foreach(l, query->distinctClause) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DISTINCT"); + } + + /* Then we tell what to select (the targetlist) */ + get_target_list(query->targetList, context, resultDesc); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add the WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add the GROUP BY clause if given */ + if (query->groupClause != NULL || query->groupingSets != NULL) + { + ParseExprKind save_exprkind; + + appendContextKeyword(context, " GROUP BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + + save_exprkind = context->special_exprkind; + context->special_exprkind = EXPR_KIND_GROUP_BY; + + if (query->groupingSets == NIL) + { + sep = ""; + foreach(l, query->groupClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + } + else + { + sep = ""; + foreach(l, query->groupingSets) + { + GroupingSet *grp = lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_groupingset(grp, query->targetList, true, context); + sep = ", "; + } + } + + context->special_exprkind = save_exprkind; + } + + /* Add the HAVING clause if given */ + if (query->havingQual != NULL) + { + appendContextKeyword(context, " HAVING ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->havingQual, context, false); + } + + /* Add the WINDOW clause if needed */ + if (query->windowClause != NIL) + get_rule_windowclause(query, context); +} + +/* ---------- + * get_target_list - Parse back a SELECT target list + * + * This is also used for RETURNING lists in INSERT/UPDATE/DELETE. + * ---------- + */ +static void +get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + StringInfoData targetbuf; + bool last_was_multiline = false; + char *sep; + int colno; + ListCell *l; + + /* we use targetbuf to hold each TLE's text temporarily */ + initStringInfo(&targetbuf); + + sep = " "; + colno = 0; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + char *colname; + char *attname; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + colno++; + + /* + * Put the new field text into targetbuf so we can decide after we've + * got it whether or not it needs to go on a new line. + */ + resetStringInfo(&targetbuf); + context->buf = &targetbuf; + + /* + * We special-case Var nodes rather than using get_rule_expr. This is + * needed because get_rule_expr will display a whole-row Var as + * "foo.*", which is the preferred notation in most contexts, but at + * the top level of a SELECT list it's not right (the parser will + * expand that notation into multiple columns, yielding behavior + * different from a whole-row Var). We need to call get_variable + * directly so that we can tell it to do the right thing, and so that + * we can get the attribute name which is the default AS label. + */ + if (tle->expr && (IsA(tle->expr, Var))) + { + attname = get_variable((Var *) tle->expr, 0, true, context); + } + else + { + get_rule_expr((Node *) tle->expr, context, true); + /* We'll show the AS name unless it's this: */ + attname = "?column?"; + } + + /* + * Figure out what the result column should be called. In the context + * of a view, use the view's tuple descriptor (so as to pick up the + * effects of any column RENAME that's been done on the view). + * Otherwise, just use what we can find in the TLE. + */ + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(resultDesc->attrs[colno - 1]->attname); + else + colname = tle->resname; + + /* Show AS unless the column's name is correct as-is */ + if (colname) /* resname could be NULL */ + { + if (attname == NULL || strcmp(attname, colname) != 0) + appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); + } + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + int leading_nl_pos; + + /* Does the new field start with a new line? */ + if (targetbuf.len > 0 && targetbuf.data[0] == '\n') + leading_nl_pos = 0; + else + leading_nl_pos = -1; + + /* If so, we shouldn't add anything */ + if (leading_nl_pos >= 0) + { + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the output buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new field is + * not the first and either the new field would cause an + * overflow or the last field used more than one line. + */ + if (colno > 1 && + ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || + last_was_multiline)) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, PRETTYINDENT_VAR); + } + + /* Remember this field's multiline status for next iteration */ + last_was_multiline = + (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); + } + + /* Add the new field */ + appendStringInfoString(buf, targetbuf.data); + } + + /* clean up */ + pfree(targetbuf.data); +} + +static void +get_setop_query(Node *setOp, Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + bool need_paren; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + if (IsA(setOp, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) setOp; + RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); + Query *subquery = rte->subquery; + + Assert(subquery != NULL); + Assert(subquery->setOperations == NULL); + /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ + need_paren = (subquery->cteList || + subquery->sortClause || + subquery->rowMarks || + subquery->limitOffset || + subquery->limitCount); + if (need_paren) + appendStringInfoChar(buf, '('); + get_query_def(subquery, buf, context->namespaces, resultDesc, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (need_paren) + appendStringInfoChar(buf, ')'); + } + else if (IsA(setOp, SetOperationStmt)) + { + SetOperationStmt *op = (SetOperationStmt *) setOp; + int subindent; + + /* + * We force parens when nesting two SetOperationStmts, except when the + * lefthand input is another setop of the same kind. Syntactically, + * we could omit parens in rather more cases, but it seems best to use + * parens to flag cases where the setop operator changes. If we use + * parens, we also increase the indentation level for the child query. + * + * There are some cases in which parens are needed around a leaf query + * too, but those are more easily handled at the next level down (see + * code above). + */ + if (IsA(op->larg, SetOperationStmt)) + { + SetOperationStmt *lop = (SetOperationStmt *) op->larg; + + if (op->op == lop->op && op->all == lop->all) + need_paren = false; + else + need_paren = true; + } + else + need_paren = false; + + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + appendContextKeyword(context, "", subindent, 0, 0); + } + else + subindent = 0; + + get_setop_query(op->larg, query, context, resultDesc); + + if (need_paren) + appendContextKeyword(context, ") ", -subindent, 0, 0); + else if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", -subindent, 0, 0); + else + appendStringInfoChar(buf, ' '); + + switch (op->op) + { + case SETOP_UNION: + appendStringInfoString(buf, "UNION "); + break; + case SETOP_INTERSECT: + appendStringInfoString(buf, "INTERSECT "); + break; + case SETOP_EXCEPT: + appendStringInfoString(buf, "EXCEPT "); + break; + default: + elog(ERROR, "unrecognized set op: %d", + (int) op->op); + } + if (op->all) + appendStringInfoString(buf, "ALL "); + + /* Always parenthesize if RHS is another setop */ + need_paren = IsA(op->rarg, SetOperationStmt); + + /* + * The indentation code here is deliberately a bit different from that + * for the lefthand input, because we want the line breaks in + * different places. + */ + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + } + else + subindent = 0; + appendContextKeyword(context, "", subindent, 0, 0); + + get_setop_query(op->rarg, query, context, resultDesc); + + if (PRETTY_INDENT(context)) + context->indentLevel -= subindent; + if (need_paren) + appendContextKeyword(context, ")", 0, 0, 0); + } + else + { + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(setOp)); + } +} + +/* + * Display a sort/group clause. + * + * Also returns the expression tree, so caller need not find it again. + */ +static Node * +get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, + deparse_context *context) +{ + StringInfo buf = context->buf; + TargetEntry *tle; + Node *expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = (Node *) tle->expr; + + /* + * Use column-number form if requested by caller. Otherwise, if + * expression is a constant, force it to be dumped with an explicit cast + * as decoration --- this is because a simple integer constant is + * ambiguous (and will be misinterpreted by findTargetlistEntry()) if we + * dump it without any decoration. If it's anything more complex than a + * simple Var, then force extra parens around it, to ensure it can't be + * misinterpreted as a cube() or rollup() construct. + */ + if (force_colno) + { + Assert(!tle->resjunk); + appendStringInfo(buf, "%d", tle->resno); + } + else if (expr && IsA(expr, Const)) + get_const_expr((Const *) expr, context, 1); + else if (!expr || IsA(expr, Var)) + get_rule_expr(expr, context, true); + else + { + /* + * We must force parens for function-like expressions even if + * PRETTY_PAREN is off, since those are the ones in danger of + * misparsing. For other expressions we need to force them only if + * PRETTY_PAREN is on, since otherwise the expression will output them + * itself. (We can't skip the parens.) + */ + bool need_paren = (PRETTY_PAREN(context) + || IsA(expr, FuncExpr) + ||IsA(expr, Aggref) + ||IsA(expr, WindowFunc)); + + if (need_paren) + appendStringInfoString(context->buf, "("); + get_rule_expr(expr, context, true); + if (need_paren) + appendStringInfoString(context->buf, ")"); + } + + return expr; +} + +/* + * Display a GroupingSet + */ +static void +get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context) +{ + ListCell *l; + StringInfo buf = context->buf; + bool omit_child_parens = true; + char *sep = ""; + + switch (gset->kind) + { + case GROUPING_SET_EMPTY: + appendStringInfoString(buf, "()"); + return; + + case GROUPING_SET_SIMPLE: + { + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoString(buf, "("); + + foreach(l, gset->content) + { + Index ref = lfirst_int(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(ref, targetlist, + false, context); + sep = ", "; + } + + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoString(buf, ")"); + } + return; + + case GROUPING_SET_ROLLUP: + appendStringInfoString(buf, "ROLLUP("); + break; + case GROUPING_SET_CUBE: + appendStringInfoString(buf, "CUBE("); + break; + case GROUPING_SET_SETS: + appendStringInfoString(buf, "GROUPING SETS ("); + omit_child_parens = false; + break; + } + + foreach(l, gset->content) + { + appendStringInfoString(buf, sep); + get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); + sep = ", "; + } + + appendStringInfoString(buf, ")"); +} + +/* + * Display an ORDER BY list. + */ +static void +get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = ""; + foreach(l, orderList) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + Node *sortexpr; + Oid sortcoltype; + TypeCacheEntry *typentry; + + appendStringInfoString(buf, sep); + sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, + force_colno, context); + sortcoltype = exprType(sortexpr); + /* See whether operator is default < or > for datatype */ + typentry = lookup_type_cache(sortcoltype, + TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + if (srt->sortop == typentry->lt_opr) + { + /* ASC is default, so emit nothing for it */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + } + else if (srt->sortop == typentry->gt_opr) + { + appendStringInfoString(buf, " DESC"); + /* DESC defaults to NULLS FIRST */ + if (!srt->nulls_first) + appendStringInfoString(buf, " NULLS LAST"); + } + else + { + appendStringInfo(buf, " USING %s", + generate_operator_name(srt->sortop, + sortcoltype, + sortcoltype)); + /* be specific to eliminate ambiguity */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + else + appendStringInfoString(buf, " NULLS LAST"); + } + sep = ", "; + } +} + +/* + * Display a WINDOW clause. + * + * Note that the windowClause list might contain only anonymous window + * specifications, in which case we should print nothing here. + */ +static void +get_rule_windowclause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = NULL; + foreach(l, query->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->name == NULL) + continue; /* ignore anonymous windows */ + + if (sep == NULL) + appendContextKeyword(context, " WINDOW ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + else + appendStringInfoString(buf, sep); + + appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); + + get_rule_windowspec(wc, query->targetList, context); + + sep = ", "; + } +} + +/* + * Display a window definition + */ +static void +get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context) +{ + StringInfo buf = context->buf; + bool needspace = false; + const char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + if (wc->refname) + { + appendStringInfoString(buf, quote_identifier(wc->refname)); + needspace = true; + } + /* partition clauses are always inherited, so only print if no refname */ + if (wc->partitionClause && !wc->refname) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "PARTITION BY "); + sep = ""; + foreach(l, wc->partitionClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, + false, context); + sep = ", "; + } + needspace = true; + } + /* print ordering clause only if not inherited */ + if (wc->orderClause && !wc->copiedOrder) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "ORDER BY "); + get_rule_orderby(wc->orderClause, targetList, false, context); + needspace = true; + } + /* framing clause is never inherited, so print unless it's default */ + if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) + { + if (needspace) + appendStringInfoChar(buf, ' '); + if (wc->frameOptions & FRAMEOPTION_RANGE) + appendStringInfoString(buf, "RANGE "); + else if (wc->frameOptions & FRAMEOPTION_ROWS) + appendStringInfoString(buf, "ROWS "); + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + appendStringInfoString(buf, "BETWEEN "); + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) + appendStringInfoString(buf, "UNBOUNDED PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_START_VALUE) + { + get_rule_expr(wc->startOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_START_VALUE_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_VALUE_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + { + appendStringInfoString(buf, "AND "); + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) + appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); + else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_END_VALUE) + { + get_rule_expr(wc->endOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_END_VALUE_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_END_VALUE_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + } + /* we will now have a trailing space; remove it */ + buf->len--; + } + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_insert_query_def - Parse back an INSERT parsetree + * ---------- + */ +static void +get_insert_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *select_rte = NULL; + RangeTblEntry *values_rte = NULL; + RangeTblEntry *rte; + char *sep; + ListCell *l; + List *strippedexprs; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * If it's an INSERT ... SELECT or multi-row VALUES, there will be a + * single RTE for the SELECT or VALUES. Plain VALUES has neither. + */ + foreach(l, query->rtable) + { + rte = (RangeTblEntry *) lfirst(l); + + if (rte->rtekind == RTE_SUBQUERY) + { + if (select_rte) + elog(ERROR, "too many subquery RTEs in INSERT"); + select_rte = rte; + } + + if (rte->rtekind == RTE_VALUES) + { + if (values_rte) + elog(ERROR, "too many values RTEs in INSERT"); + values_rte = rte; + } + } + if (select_rte && values_rte) + elog(ERROR, "both subquery and values RTEs in INSERT"); + + /* + * Start the query with INSERT INTO relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + appendStringInfo(buf, "INSERT INTO %s ", + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + /* INSERT requires AS keyword for target alias */ + if (rte->alias != NULL) + appendStringInfo(buf, "AS %s ", + quote_identifier(rte->alias->aliasname)); + + /* + * Add the insert-column-names list. Any indirection decoration needed on + * the column names can be inferred from the top targetlist. + */ + strippedexprs = NIL; + sep = ""; + if (query->targetList) + appendStringInfoChar(buf, '('); + foreach(l, query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_relid_attribute_name(rte->relid, + tle->resno))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + * Add the stripped expressions to strippedexprs. (If it's a + * single-VALUES statement, the stripped expressions are the VALUES to + * print below. Otherwise they're just Vars and not really + * interesting.) + */ + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + if (query->targetList) + appendStringInfoString(buf, ") "); + + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + + if (select_rte) + { + /* Add the SELECT */ + get_query_def(select_rte->subquery, buf, NIL, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + } + else if (values_rte) + { + /* Add the multi-VALUES expression lists */ + get_values_def(values_rte->values_lists, context); + } + else if (strippedexprs) + { + /* Add the single-VALUES expression list */ + appendContextKeyword(context, "VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_expr((Node *) strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + { + /* No expressions, so it must be DEFAULT VALUES */ + appendStringInfoString(buf, "DEFAULT VALUES"); + } + + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + appendStringInfoString(buf, " ON CONFLICT"); + + if (confl->arbiterElems) + { + /* Add the single-VALUES expression list */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) confl->arbiterElems, context, false); + appendStringInfoChar(buf, ')'); + + /* Add a WHERE clause (for partial indexes) if given */ + if (confl->arbiterWhere != NULL) + { + bool save_varprefix; + + /* + * Force non-prefixing of Vars, since parser assumes that they + * belong to target relation. WHERE clause does not use + * InferenceElem, so this is separately required. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->arbiterWhere, context, false); + + context->varprefix = save_varprefix; + } + } + else if (OidIsValid(confl->constraint)) + { + char *constraint = get_constraint_name(confl->constraint); + int64 shardId = context->shardid; + + if (shardId > 0) + { + AppendShardIdToName(&constraint, shardId); + } + + if (!constraint) + elog(ERROR, "cache lookup failed for constraint %u", + confl->constraint); + appendStringInfo(buf, " ON CONSTRAINT %s", + quote_identifier(constraint)); + } + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " DO NOTHING"); + } + else + { + appendStringInfoString(buf, " DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_def - Parse back an UPDATE parsetree + * ---------- + */ +static void +get_update_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with UPDATE relname SET + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + appendStringInfoString(buf, " SET "); + + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + + /* + * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks + * into a list. We expect them to appear, in ID order, in resjunk tlist + * entries. + */ + ma_sublinks = NIL; + if (query->hasSubLinks) /* else there can't be any */ + { + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk && IsA(tle->expr, SubLink)) + { + SubLink *sl = (SubLink *) tle->expr; + + if (sl->subLinkType == MULTIEXPR_SUBLINK) + { + ma_sublinks = lappend(ma_sublinks, sl); + Assert(sl->subLinkId == list_length(ma_sublinks)); + } + } + } + } + next_ma_cell = list_head(ma_sublinks); + cur_ma_sublink = NULL; + remaining_ma_columns = 0; + + /* Add the comma separated list of 'attname = value' */ + sep = ""; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *expr; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + /* Emit separator (OK whether we're in multiassignment or not) */ + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Check to see if we're starting a multiassignment group: if so, + * output a left paren. + */ + if (next_ma_cell != NULL && cur_ma_sublink == NULL) + { + /* + * We must dig down into the expr to see if it's a PARAM_MULTIEXPR + * Param. That could be buried under FieldStores and ArrayRefs + * (cf processIndirection()), and underneath those there could be + * an implicit type coercion. + */ + expr = (Node *) tle->expr; + while (expr) + { + if (IsA(expr, FieldStore)) + { + FieldStore *fstore = (FieldStore *) expr; + + expr = (Node *) linitial(fstore->newvals); + } + else if (IsA(expr, ArrayRef)) + { + ArrayRef *aref = (ArrayRef *) expr; + + if (aref->refassgnexpr == NULL) + break; + expr = (Node *) aref->refassgnexpr; + } + else + break; + } + expr = strip_implicit_coercions(expr); + + if (expr && IsA(expr, Param) && + ((Param *) expr)->paramkind == PARAM_MULTIEXPR) + { + cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); + next_ma_cell = lnext(next_ma_cell); + remaining_ma_columns = count_nonjunk_tlist_entries( + ((Query *) cur_ma_sublink->subselect)->targetList); + Assert(((Param *) expr)->paramid == + ((cur_ma_sublink->subLinkId << 16) | 1)); + appendStringInfoChar(buf, '('); + } + } + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_relid_attribute_name(rte->relid, + tle->resno))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + */ + expr = processIndirection((Node *) tle->expr, context); + + /* + * If we're in a multiassignment, skip printing anything more, unless + * this is the last column; in which case, what we print should be the + * sublink, not the Param. + */ + if (cur_ma_sublink != NULL) + { + if (--remaining_ma_columns > 0) + continue; /* not the last column of multiassignment */ + appendStringInfoChar(buf, ')'); + expr = (Node *) cur_ma_sublink; + cur_ma_sublink = NULL; + } + + appendStringInfoString(buf, " = "); + + get_rule_expr(expr, context, false); + } +} + + +/* ---------- + * get_delete_query_def - Parse back a DELETE parsetree + * ---------- + */ +static void +get_delete_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with DELETE FROM relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + + /* Add the USING clause if given */ + get_from_clause(query, " USING ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_utility_query_def - Parse back a UTILITY parsetree + * ---------- + */ +static void +get_utility_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) + { + NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + appendStringInfo(buf, "NOTIFY %s", + quote_identifier(stmt->conditionname)); + if (stmt->payload) + { + appendStringInfoString(buf, ", "); + simple_quote_literal(buf, stmt->payload); + } + } + else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) + { + TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; + List *relationList = stmt->relations; + ListCell *relationCell = NULL; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + + appendStringInfo(buf, "TRUNCATE TABLE"); + + foreach(relationCell, relationList) + { + RangeVar *relationVar = (RangeVar *) lfirst(relationCell); + Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); + char *relationName = generate_relation_or_shard_name(relationId, + context->distrelid, + context->shardid, NIL); + appendStringInfo(buf, " %s", relationName); + + if (lnext(relationCell) != NULL) + { + appendStringInfo(buf, ","); + } + } + + if (stmt->restart_seqs) + { + appendStringInfo(buf, " RESTART IDENTITY"); + } + + if (stmt->behavior == DROP_CASCADE) + { + appendStringInfo(buf, " CASCADE"); + } + } + else + { + /* Currently only NOTIFY utility commands can appear in rules */ + elog(ERROR, "unexpected utility statement type"); + } +} + +/* + * Display a Var appropriately. + * + * In some cases (currently only when recursing into an unnamed join) + * the Var's varlevelsup has to be interpreted with respect to a context + * above the current one; levelsup indicates the offset. + * + * If istoplevel is TRUE, the Var is at the top level of a SELECT's + * targetlist, which means we need special treatment of whole-row Vars. + * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a + * dirty hack to prevent "tab.*" from being expanded into multiple columns. + * (The parser will strip the useless coercion, so no inefficiency is added in + * dump and reload.) We used to print just "tab" in such cases, but that is + * ambiguous and will yield the wrong result if "tab" is also a plain column + * name in the query. + * + * Returns the attname of the Var, or NULL if the Var has no attname (because + * it is a whole-row Var or a subplan output reference). + */ +static char * +get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + deparse_columns *colinfo; + char *refname; + char *attname; + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. Also + * find the aliases previously assigned for this RTE. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, var->varno - 1); + colinfo = deparse_columns_fetch(var->varno, dpns); + attnum = var->varattno; + } + else + { + resolve_special_varno((Node *) var, context, NULL, + get_special_variable); + return NULL; + } + + /* + * The planner will sometimes emit Vars referencing resjunk elements of a + * subquery's target list (this is currently only possible if it chooses + * to generate a "physical tlist" for a SubqueryScan or CteScan node). + * Although we prefer to print subquery-referencing Vars using the + * subquery's alias, that's not possible for resjunk items since they have + * no alias. So in that case, drill down to the subplan and print the + * contents of the referenced tlist item. This works because in a plan + * tree, such Vars can only occur in a SubqueryScan or CteScan node, and + * we'll have set dpns->inner_planstate to reference the child plan node. + */ + if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && + attnum > list_length(rte->eref->colnames) && + dpns->inner_planstate) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + var->varattno, rte->eref->aliasname); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + /* + * Force parentheses because our caller probably assumed a Var is a + * simple expression. + */ + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tle->expr, context, true); + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, ')'); + + pop_child_plan(dpns, &save_dpns); + return NULL; + } + + /* + * If it's an unnamed join, look at the expansion of the alias variable. + * If it's a simple reference to one of the input vars, then recursively + * print the name of that var instead. When it's not a simple reference, + * we have to just print the unqualified join column name. (This can only + * happen with "dangerous" merged columns in a JOIN USING; we took pains + * previously to make the unqualified column name unique in such cases.) + * + * This wouldn't work in decompiling plan trees, because we don't store + * joinaliasvars lists after planning; but a plan tree should never + * contain a join alias variable. + */ + if (rte->rtekind == RTE_JOIN && rte->alias == NULL) + { + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + if (attnum > 0) + { + Var *aliasvar; + + aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); + /* we intentionally don't strip implicit coercions here */ + if (aliasvar && IsA(aliasvar, Var)) + { + return get_variable(aliasvar, var->varlevelsup + levelsup, + istoplevel, context); + } + } + + /* + * Unnamed join has no refname. (Note: since it's unnamed, there is + * no way the user could have referenced it to create a whole-row Var + * for it. So we don't have to cover that case below.) + */ + Assert(refname == NULL); + } + + if (attnum == InvalidAttrNumber) + attname = NULL; + else if (attnum > 0) + { + /* Get column name to use from the colinfo struct */ + if (attnum > colinfo->num_cols) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + attname = colinfo->colnames[attnum - 1]; + if (attname == NULL) /* dropped column? */ + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + } + else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* System column on a Citus shard */ + attname = get_relid_attribute_name(rte->relid, attnum); + } + else + { + /* System column - name is fixed, get it from the catalog */ + attname = get_rte_attribute_name(rte, attnum); + } + + if (refname && (context->varprefix || attname == NULL)) + { + appendStringInfoString(buf, quote_identifier(refname)); + appendStringInfoChar(buf, '.'); + } + if (attname) + appendStringInfoString(buf, quote_identifier(attname)); + else + { + appendStringInfoChar(buf, '*'); + if (istoplevel) + appendStringInfo(buf, "::%s", + format_type_with_typemod(var->vartype, + var->vartypmod)); + } + + return attname; +} + +/* + * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This + * routine is actually a callback for get_special_varno, which handles finding + * the correct TargetEntry. We get the expression contained in that + * TargetEntry and just need to deparse it, a job we can throw back on + * get_rule_expr. + */ +static void +get_special_variable(Node *node, deparse_context *context, void *private) +{ + StringInfo buf = context->buf; + + /* + * Force parentheses because our caller probably assumed a Var is a simple + * expression. + */ + if (!IsA(node, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr(node, context, true); + if (!IsA(node, Var)) + appendStringInfoChar(buf, ')'); +} + +/* + * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, + * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, + * invoke the callback provided. + */ +static void +resolve_special_varno(Node *node, deparse_context *context, void *private, + void (*callback) (Node *, deparse_context *, void *)) +{ + Var *var; + deparse_namespace *dpns; + + /* If it's not a Var, invoke the callback. */ + if (!IsA(node, Var)) + { + callback(node, context, private); + return; + } + + /* Find appropriate nesting depth */ + var = (Var *) node; + dpns = (deparse_namespace *) list_nth(context->namespaces, + var->varlevelsup); + + /* + * It's a special RTE, so recurse. + */ + if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + resolve_special_varno((Node *) tle->expr, context, private, callback); + return; + } + else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) + elog(ERROR, "bogus varno: %d", var->varno); + + /* Not special. Just invoke the callback. */ + callback(node, context, private); +} + +/* + * Get the name of a field of an expression of composite type. The + * expression is usually a Var, but we handle other cases too. + * + * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * + * This is fairly straightforward when the expression has a named composite + * type; we need only look up the type in the catalogs. However, the type + * could also be RECORD. Since no actual table or view column is allowed to + * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE + * or to a subquery output. We drill down to find the ultimate defining + * expression and attempt to infer the field name from it. We ereport if we + * can't determine the name. + * + * Similarly, a PARAM of type RECORD has to refer to some expression of + * a determinable composite type. + */ +static const char * +get_name_for_var_field(Var *var, int fieldno, + int levelsup, deparse_context *context) +{ + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + TupleDesc tupleDesc; + Node *expr; + + /* + * If it's a RowExpr that was expanded from a whole-row Var, use the + * column names attached to it. + */ + if (IsA(var, RowExpr)) + { + RowExpr *r = (RowExpr *) var; + + if (fieldno > 0 && fieldno <= list_length(r->colnames)) + return strVal(list_nth(r->colnames, fieldno - 1)); + } + + /* + * If it's a Param of type RECORD, try to find what the Param refers to. + */ + if (IsA(var, Param)) + { + Param *param = (Param *) var; + ListCell *ancestor_cell; + + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so recurse to decipher the field name */ + deparse_namespace save_dpns; + const char *result; + + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + pop_ancestor_plan(dpns, &save_dpns); + return result; + } + } + + /* + * If it's a Var of type RECORD, we have to find what the Var refers to; + * if not, we can use get_expr_result_type. If that fails, we try + * lookup_rowtype_tupdesc, which will probably fail too, but will ereport + * an acceptable message. + */ + if (!IsA(var, Var) || + var->vartype != RECORDOID) + { + if (get_expr_result_type((Node *) var, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) + tupleDesc = lookup_rowtype_tupdesc_copy(exprType((Node *) var), + exprTypmod((Node *) var)); + Assert(tupleDesc); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(tupleDesc->attrs[fieldno - 1]->attname); + } + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + attnum = var->varattno; + } + else if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + const char *result; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + return result; + } + else + { + elog(ERROR, "bogus varno: %d", var->varno); + return NULL; /* keep compiler quiet */ + } + + if (attnum == InvalidAttrNumber) + { + /* Var is whole-row reference to RTE, so select the right field */ + return get_rte_attribute_name(rte, fieldno); + } + + /* + * This part has essentially the same logic as the parser's + * expandRecordVariable() function, but we are dealing with a different + * representation of the input context, and we only need one field name + * not a TupleDesc. Also, we need special cases for finding subquery and + * CTE subplans when deparsing Plan trees. + */ + expr = (Node *) var; /* default if we can't drill down */ + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: + + /* + * This case should not occur: a column of a table or values list + * shouldn't have type RECORD. Fall through and fail (most + * likely) at the bottom. + */ + break; + case RTE_SUBQUERY: + /* Subselect-in-FROM: examine sub-select's output expr */ + { + if (rte->subquery) + { + TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the sub-select to see what its Var + * refers to. We have to build an additional level of + * namespace to keep in step with varlevelsup in the + * subselect. + */ + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, rte->subquery, + context->namespaces); + + context->namespaces = lcons(&mydpns, + context->namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = + list_delete_first(context->namespaces); + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have complete + * RTE entries (in particular, rte->subquery is NULL). But + * the only place we'd see a Var directly referencing a + * SUBQUERY RTE is in a SubqueryScan plan node, and we can + * look into the child plan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for subquery %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + case RTE_JOIN: + /* Join RTE --- recursively inspect the alias variable */ + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); + expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); + Assert(expr != NULL); + /* we intentionally don't strip implicit coercions here */ + if (IsA(expr, Var)) + return get_name_for_var_field((Var *) expr, fieldno, + var->varlevelsup + levelsup, + context); + /* else fall through to inspect the expression */ + break; + case RTE_FUNCTION: + case RTE_TABLEFUNC: + + /* + * We couldn't get here unless a function is declared with one of + * its result columns as RECORD, which is not allowed. + */ + break; + case RTE_CTE: + /* CTE reference: examine subquery's output expr */ + { + CommonTableExpr *cte = NULL; + Index ctelevelsup; + ListCell *lc; + + /* + * Try to find the referenced CTE using the namespace stack. + */ + ctelevelsup = rte->ctelevelsup + netlevelsup; + if (ctelevelsup >= list_length(context->namespaces)) + lc = NULL; + else + { + deparse_namespace *ctedpns; + + ctedpns = (deparse_namespace *) + list_nth(context->namespaces, ctelevelsup); + foreach(lc, ctedpns->ctes) + { + cte = (CommonTableExpr *) lfirst(lc); + if (strcmp(cte->ctename, rte->ctename) == 0) + break; + } + } + if (lc != NULL) + { + Query *ctequery = (Query *) cte->ctequery; + TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the CTE to see what its Var refers to. + * We have to build an additional level of namespace + * to keep in step with varlevelsup in the CTE. + * Furthermore it could be an outer CTE, so we may + * have to delete some levels of namespace. + */ + List *save_nslist = context->namespaces; + List *new_nslist; + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, ctequery, + context->namespaces); + + new_nslist = list_copy_tail(context->namespaces, + ctelevelsup); + context->namespaces = lcons(&mydpns, new_nslist); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have a CTE + * list. But the only place we'd see a Var directly + * referencing a CTE RTE is in a CteScan plan node, and we + * can look into the subplan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for CTE %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + } + + /* + * We now have an expression we can't expand any more, so see if + * get_expr_result_type() can do anything with it. If not, pass to + * lookup_rowtype_tupdesc() which will probably fail, but will give an + * appropriate error message while failing. + */ + if (get_expr_result_type(expr, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) + tupleDesc = lookup_rowtype_tupdesc_copy(exprType(expr), + exprTypmod(expr)); + Assert(tupleDesc); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(tupleDesc->attrs[fieldno - 1]->attname); +} + +/* + * Try to find the referenced expression for a PARAM_EXEC Param that might + * reference a parameter supplied by an upper NestLoop or SubPlan plan node. + * + * If successful, return the expression and set *dpns_p and *ancestor_cell_p + * appropriately for calling push_ancestor_plan(). If no referent can be + * found, return NULL. + */ +static Node * +find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p) +{ + /* Initialize output parameters to prevent compiler warnings */ + *dpns_p = NULL; + *ancestor_cell_p = NULL; + + /* + * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or + * SubPlan argument. This will necessarily be in some ancestor of the + * current expression's PlanState. + */ + if (param->paramkind == PARAM_EXEC) + { + deparse_namespace *dpns; + PlanState *child_ps; + bool in_same_plan_level; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + child_ps = dpns->planstate; + in_same_plan_level = true; + + foreach(lc, dpns->ancestors) + { + PlanState *ps = (PlanState *) lfirst(lc); + ListCell *lc2; + + /* + * NestLoops transmit params to their inner child only; also, once + * we've crawled up out of a subplan, this couldn't possibly be + * the right match. + */ + if (IsA(ps, NestLoopState) && + child_ps == innerPlanState(ps) && + in_same_plan_level) + { + NestLoop *nl = (NestLoop *) ps->plan; + + foreach(lc2, nl->nestParams) + { + NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); + + if (nlp->paramno == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return (Node *) nlp->paramval; + } + } + } + + /* + * Check to see if we're crawling up from a subplan. + */ + foreach(lc2, ps->subPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + SubPlan *subplan = sstate->subplan; + ListCell *lc3; + ListCell *lc4; + + if (child_ps != sstate->planstate) + continue; + + /* Matched subplan, so check its arguments */ + forboth(lc3, subplan->parParam, lc4, subplan->args) + { + int paramid = lfirst_int(lc3); + Node *arg = (Node *) lfirst(lc4); + + if (paramid == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return arg; + } + } + + /* Keep looking, but we are emerging from a subplan. */ + in_same_plan_level = false; + break; + } + + /* + * Likewise check to see if we're emerging from an initplan. + * Initplans never have any parParams, so no need to search that + * list, but we need to know if we should reset + * in_same_plan_level. + */ + foreach(lc2, ps->initPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + + if (child_ps != sstate->planstate) + continue; + + /* No parameters to be had here. */ + Assert(sstate->subplan->parParam == NIL); + + /* Keep looking, but we are emerging from an initplan. */ + in_same_plan_level = false; + break; + } + + /* No luck, crawl up to next ancestor */ + child_ps = ps; + } + } + + /* No referent found */ + return NULL; +} + +/* + * Display a Param appropriately. + */ +static void +get_parameter(Param *param, deparse_context *context) +{ + Node *expr; + deparse_namespace *dpns; + ListCell *ancestor_cell; + + /* + * If it's a PARAM_EXEC parameter, try to locate the expression from which + * the parameter was computed. Note that failing to find a referent isn't + * an error, since the Param might well be a subplan output rather than an + * input. + */ + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so print it */ + deparse_namespace save_dpns; + bool save_varprefix; + bool need_paren; + + /* Switch attention to the ancestor plan node */ + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + + /* + * Force prefixing of Vars, since they won't belong to the relation + * being scanned in the original plan node. + */ + save_varprefix = context->varprefix; + context->varprefix = true; + + /* + * A Param's expansion is typically a Var, Aggref, or upper-level + * Param, which wouldn't need extra parentheses. Otherwise, insert + * parens to ensure the expression looks atomic. + */ + need_paren = !(IsA(expr, Var) || + IsA(expr, Aggref) || + IsA(expr, Param)); + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(expr, context, false); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); + + context->varprefix = save_varprefix; + + pop_ancestor_plan(dpns, &save_dpns); + + return; + } + + /* + * Not PARAM_EXEC, or couldn't find referent: just print $N. + */ + appendStringInfo(context->buf, "$%d", param->paramid); +} + +/* + * get_simple_binary_op_name + * + * helper function for isSimpleNode + * will return single char binary operator name, or NULL if it's not + */ +static const char * +get_simple_binary_op_name(OpExpr *expr) +{ + List *args = expr->args; + + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + const char *op; + + op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); + if (strlen(op) == 1) + return op; + } + return NULL; +} + + +/* + * isSimpleNode - check if given node is simple (doesn't need parenthesizing) + * + * true : simple in the context of parent node's type + * false : not simple + */ +static bool +isSimpleNode(Node *node, Node *parentNode, int prettyFlags) +{ + if (!node) + return false; + + switch (nodeTag(node)) + { + case T_Var: + case T_Const: + case T_Param: + case T_CoerceToDomainValue: + case T_SetToDefault: + case T_CurrentOfExpr: + /* single words: always simple */ + return true; + + case T_ArrayRef: + case T_ArrayExpr: + case T_RowExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_NullIfExpr: + case T_Aggref: + case T_WindowFunc: + case T_FuncExpr: + /* function-like: name(..) or name[..] */ + return true; + + /* CASE keywords act as parentheses */ + case T_CaseExpr: + return true; + + case T_FieldSelect: + + /* + * appears simple since . has top precedence, unless parent is + * T_FieldSelect itself! + */ + return (IsA(parentNode, FieldSelect) ? false : true); + + case T_FieldStore: + + /* + * treat like FieldSelect (probably doesn't matter) + */ + return (IsA(parentNode, FieldStore) ? false : true); + + case T_CoerceToDomain: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, + node, prettyFlags); + case T_RelabelType: + return isSimpleNode((Node *) ((RelabelType *) node)->arg, + node, prettyFlags); + case T_CoerceViaIO: + return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, + node, prettyFlags); + case T_ArrayCoerceExpr: + return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, + node, prettyFlags); + case T_ConvertRowtypeExpr: + return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, + node, prettyFlags); + + case T_OpExpr: + { + /* depends on parent node type; needs further checking */ + if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) + { + const char *op; + const char *parentOp; + bool is_lopriop; + bool is_hipriop; + bool is_lopriparent; + bool is_hipriparent; + + op = get_simple_binary_op_name((OpExpr *) node); + if (!op) + return false; + + /* We know only the basic operators + - and * / % */ + is_lopriop = (strchr("+-", *op) != NULL); + is_hipriop = (strchr("*/%", *op) != NULL); + if (!(is_lopriop || is_hipriop)) + return false; + + parentOp = get_simple_binary_op_name((OpExpr *) parentNode); + if (!parentOp) + return false; + + is_lopriparent = (strchr("+-", *parentOp) != NULL); + is_hipriparent = (strchr("*/%", *parentOp) != NULL); + if (!(is_lopriparent || is_hipriparent)) + return false; + + if (is_hipriop && is_lopriparent) + return true; /* op binds tighter than parent */ + + if (is_lopriop && is_hipriparent) + return false; + + /* + * Operators are same priority --- can skip parens only if + * we have (a - b) - c, not a - (b - c). + */ + if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) + return true; + + return false; + } + /* else do the same stuff as for T_SubLink et al. */ + /* FALL THROUGH */ + } + + case T_SubLink: + case T_NullTest: + case T_BooleanTest: + case T_DistinctExpr: + switch (nodeTag(parentNode)) + { + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_BoolExpr: /* lower precedence */ + case T_ArrayRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + case T_BoolExpr: + switch (nodeTag(parentNode)) + { + case T_BoolExpr: + if (prettyFlags & PRETTYFLAG_PAREN) + { + BoolExprType type; + BoolExprType parentType; + + type = ((BoolExpr *) node)->boolop; + parentType = ((BoolExpr *) parentNode)->boolop; + switch (type) + { + case NOT_EXPR: + case AND_EXPR: + if (parentType == AND_EXPR || parentType == OR_EXPR) + return true; + break; + case OR_EXPR: + if (parentType == OR_EXPR) + return true; + break; + } + } + return false; + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_ArrayRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + default: + break; + } + /* those we don't know: in dubio complexo */ + return false; +} + + +/* + * appendContextKeyword - append a keyword to buffer + * + * If prettyPrint is enabled, perform a line break, and adjust indentation. + * Otherwise, just append the keyword. + */ +static void +appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus) +{ + StringInfo buf = context->buf; + + if (PRETTY_INDENT(context)) + { + int indentAmount; + + context->indentLevel += indentBefore; + + /* remove any trailing spaces currently in the buffer ... */ + removeStringInfoSpaces(buf); + /* ... then add a newline and some spaces */ + appendStringInfoChar(buf, '\n'); + + if (context->indentLevel < PRETTYINDENT_LIMIT) + indentAmount = Max(context->indentLevel, 0) + indentPlus; + else + { + /* + * If we're indented more than PRETTYINDENT_LIMIT characters, try + * to conserve horizontal space by reducing the per-level + * indentation. For best results the scale factor here should + * divide all the indent amounts that get added to indentLevel + * (PRETTYINDENT_STD, etc). It's important that the indentation + * not grow unboundedly, else deeply-nested trees use O(N^2) + * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. + */ + indentAmount = PRETTYINDENT_LIMIT + + (context->indentLevel - PRETTYINDENT_LIMIT) / + (PRETTYINDENT_STD / 2); + indentAmount %= PRETTYINDENT_LIMIT; + /* scale/wrap logic affects indentLevel, but not indentPlus */ + indentAmount += indentPlus; + } + appendStringInfoSpaces(buf, indentAmount); + + appendStringInfoString(buf, str); + + context->indentLevel += indentAfter; + if (context->indentLevel < 0) + context->indentLevel = 0; + } + else + appendStringInfoString(buf, str); +} + +/* + * removeStringInfoSpaces - delete trailing spaces from a buffer. + * + * Possibly this should move to stringinfo.c at some point. + */ +static void +removeStringInfoSpaces(StringInfo str) +{ + while (str->len > 0 && str->data[str->len - 1] == ' ') + str->data[--(str->len)] = '\0'; +} + + +/* + * get_rule_expr_paren - deparse expr using get_rule_expr, + * embracing the string with parentheses if necessary for prettyPrint. + * + * Never embrace if prettyFlags=0, because it's done in the calling node. + * + * Any node that does *not* embrace its argument node by sql syntax (with + * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should + * use get_rule_expr_paren instead of get_rule_expr so parentheses can be + * added. + */ +static void +get_rule_expr_paren(Node *node, deparse_context *context, + bool showimplicit, Node *parentNode) +{ + bool need_paren; + + need_paren = PRETTY_PAREN(context) && + !isSimpleNode(node, parentNode, context->prettyFlags); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(node, context, showimplicit); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); +} + + +/* ---------- + * get_rule_expr - Parse back an expression + * + * Note: showimplicit determines whether we display any implicit cast that + * is present at the top of the expression tree. It is a passed argument, + * not a field of the context struct, because we change the value as we + * recurse down into the expression. In general we suppress implicit casts + * when the result type is known with certainty (eg, the arguments of an + * OR must be boolean). We display implicit casts for arguments of functions + * and operators, since this is needed to be certain that the same function + * or operator will be chosen when the expression is re-parsed. + * ---------- + */ +static void +get_rule_expr(Node *node, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + + if (node == NULL) + return; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Each level of get_rule_expr must emit an indivisible term + * (parenthesized if necessary) to ensure result is reparsed into the same + * expression tree. The only exception is that when the input is a List, + * we emit the component items comma-separated with no surrounding + * decoration; this is convenient for most callers. + */ + switch (nodeTag(node)) + { + case T_Var: + (void) get_variable((Var *) node, 0, false, context); + break; + + case T_Const: + get_const_expr((Const *) node, context, 0); + break; + + case T_Param: + get_parameter((Param *) node, context); + break; + + case T_Aggref: + get_agg_expr((Aggref *) node, context, (Aggref *) node); + break; + + case T_GroupingFunc: + { + GroupingFunc *gexpr = (GroupingFunc *) node; + + appendStringInfoString(buf, "GROUPING("); + get_rule_expr((Node *) gexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_WindowFunc: + get_windowfunc_expr((WindowFunc *) node, context); + break; + + case T_ArrayRef: + { + ArrayRef *aref = (ArrayRef *) node; + bool need_parens; + + /* + * If the argument is a CaseTestExpr, we must be inside a + * FieldStore, ie, we are assigning to an element of an array + * within a composite column. Since we already punted on + * displaying the FieldStore's target information, just punt + * here too, and display only the assignment source + * expression. + */ + if (IsA(aref->refexpr, CaseTestExpr)) + { + Assert(aref->refassgnexpr); + get_rule_expr((Node *) aref->refassgnexpr, + context, showimplicit); + break; + } + + /* + * Parenthesize the argument unless it's a simple Var or a + * FieldSelect. (In particular, if it's another ArrayRef, we + * *must* parenthesize to avoid confusion.) + */ + need_parens = !IsA(aref->refexpr, Var) && + !IsA(aref->refexpr, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) aref->refexpr, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * If there's a refassgnexpr, we want to print the node in the + * format "array[subscripts] := refassgnexpr". This is not + * legal SQL, so decompilation of INSERT or UPDATE statements + * should always use processIndirection as part of the + * statement-level syntax. We should only see this when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. + */ + if (aref->refassgnexpr) + { + Node *refassgnexpr; + + /* + * Use processIndirection to print this node's subscripts + * as well as any additional field selections or + * subscripting in immediate descendants. It returns the + * RHS expr that is actually being "assigned". + */ + refassgnexpr = processIndirection(node, context); + appendStringInfoString(buf, " := "); + get_rule_expr(refassgnexpr, context, showimplicit); + } + else + { + /* Just an ordinary array fetch, so print subscripts */ + printSubscripts(aref, context); + } + } + break; + + case T_FuncExpr: + get_func_expr((FuncExpr *) node, context, showimplicit); + break; + + case T_NamedArgExpr: + { + NamedArgExpr *na = (NamedArgExpr *) node; + + appendStringInfo(buf, "%s => ", quote_identifier(na->name)); + get_rule_expr((Node *) na->arg, context, showimplicit); + } + break; + + case T_OpExpr: + get_oper_expr((OpExpr *) node, context); + break; + + case T_DistinctExpr: + { + DistinctExpr *expr = (DistinctExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfoString(buf, " IS DISTINCT FROM "); + get_rule_expr_paren(arg2, context, true, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullIfExpr: + { + NullIfExpr *nullifexpr = (NullIfExpr *) node; + + appendStringInfoString(buf, "NULLIF("); + get_rule_expr((Node *) nullifexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfo(buf, " %s %s (", + generate_operator_name(expr->opno, + exprType(arg1), + get_base_element_type(exprType(arg2))), + expr->useOr ? "ANY" : "ALL"); + get_rule_expr_paren(arg2, context, true, node); + + /* + * There's inherent ambiguity in "x op ANY/ALL (y)" when y is + * a bare sub-SELECT. Since we're here, the sub-SELECT must + * be meant as a scalar sub-SELECT yielding an array value to + * be used in ScalarArrayOpExpr; but the grammar will + * preferentially interpret such a construct as an ANY/ALL + * SubLink. To prevent misparsing the output that way, insert + * a dummy coercion (which will be stripped by parse analysis, + * so no inefficiency is added in dump and reload). This is + * indeed most likely what the user wrote to get the construct + * accepted in the first place. + */ + if (IsA(arg2, SubLink) && + ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) + appendStringInfo(buf, "::%s", + format_type_with_typemod(exprType(arg2), + exprTypmod(arg2))); + appendStringInfoChar(buf, ')'); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *) node; + Node *first_arg = linitial(expr->args); + ListCell *arg = lnext(list_head(expr->args)); + + switch (expr->boolop) + { + case AND_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " AND "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case OR_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " OR "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case NOT_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + appendStringInfoString(buf, "NOT "); + get_rule_expr_paren(first_arg, context, + false, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + default: + elog(ERROR, "unrecognized boolop: %d", + (int) expr->boolop); + } + } + break; + + case T_SubLink: + get_sublink_expr((SubLink *) node, context); + break; + + case T_SubPlan: + { + SubPlan *subplan = (SubPlan *) node; + + /* + * We cannot see an already-planned subplan in rule deparsing, + * only while EXPLAINing a query plan. We don't try to + * reconstruct the original SQL, just reference the subplan + * that appears elsewhere in EXPLAIN's result. + */ + if (subplan->useHashTable) + appendStringInfo(buf, "(hashed %s)", subplan->plan_name); + else + appendStringInfo(buf, "(%s)", subplan->plan_name); + } + break; + + case T_AlternativeSubPlan: + { + AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; + ListCell *lc; + + /* As above, this can only happen during EXPLAIN */ + appendStringInfoString(buf, "(alternatives: "); + foreach(lc, asplan->subplans) + { + SubPlan *splan = lfirst_node(SubPlan, lc); + + if (splan->useHashTable) + appendStringInfo(buf, "hashed %s", splan->plan_name); + else + appendStringInfoString(buf, splan->plan_name); + if (lnext(lc)) + appendStringInfoString(buf, " or "); + } + appendStringInfoChar(buf, ')'); + } + break; + + case T_FieldSelect: + { + FieldSelect *fselect = (FieldSelect *) node; + Node *arg = (Node *) fselect->arg; + int fno = fselect->fieldnum; + const char *fieldname; + bool need_parens; + + /* + * Parenthesize the argument unless it's an ArrayRef or + * another FieldSelect. Note in particular that it would be + * WRONG to not parenthesize a Var argument; simplicity is not + * the issue here, having the right number of names is. + */ + need_parens = !IsA(arg, ArrayRef) &&!IsA(arg, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr(arg, context, true); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * Get and print the field name. + */ + fieldname = get_name_for_var_field((Var *) arg, fno, + 0, context); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + } + break; + + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *) node; + bool need_parens; + + /* + * There is no good way to represent a FieldStore as real SQL, + * so decompilation of INSERT or UPDATE statements should + * always use processIndirection as part of the + * statement-level syntax. We should only get here when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. The plan case is even harder than + * ordinary rules would be, because the planner tries to + * collapse multiple assignments to the same field or subfield + * into one FieldStore; so we can see a list of target fields + * not just one, and the arguments could be FieldStores + * themselves. We don't bother to try to print the target + * field names; we just print the source arguments, with a + * ROW() around them if there's more than one. This isn't + * terribly complete, but it's probably good enough for + * EXPLAIN's purposes; especially since anything more would be + * either hopelessly confusing or an even poorer + * representation of what the plan is actually doing. + */ + need_parens = (list_length(fstore->newvals) != 1); + if (need_parens) + appendStringInfoString(buf, "ROW("); + get_rule_expr((Node *) fstore->newvals, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + } + break; + + case T_RelabelType: + { + RelabelType *relabel = (RelabelType *) node; + Node *arg = (Node *) relabel->arg; + + if (relabel->relabelformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + relabel->resulttype, + relabel->resulttypmod, + node); + } + } + break; + + case T_CoerceViaIO: + { + CoerceViaIO *iocoerce = (CoerceViaIO *) node; + Node *arg = (Node *) iocoerce->arg; + + if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + iocoerce->resulttype, + -1, + node); + } + } + break; + + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; + Node *arg = (Node *) acoerce->arg; + + if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + acoerce->resulttype, + acoerce->resulttypmod, + node); + } + } + break; + + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; + Node *arg = (Node *) convert->arg; + + if (convert->convertformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + convert->resulttype, -1, + node); + } + } + break; + + case T_CollateExpr: + { + CollateExpr *collate = (CollateExpr *) node; + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *) node; + ListCell *temp; + + appendContextKeyword(context, "CASE", + 0, PRETTYINDENT_VAR, 0); + if (caseexpr->arg) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) caseexpr->arg, context, true); + } + foreach(temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *) lfirst(temp); + Node *w = (Node *) when->expr; + + if (caseexpr->arg) + { + /* + * The parser should have produced WHEN clauses of the + * form "CaseTestExpr = RHS", possibly with an + * implicit coercion inserted above the CaseTestExpr. + * For accurate decompilation of rules it's essential + * that we show just the RHS. However in an + * expression that's been through the optimizer, the + * WHEN clause could be almost anything (since the + * equality operator could have been expanded into an + * inline function). If we don't recognize the form + * of the WHEN clause, just punt and display it as-is. + */ + if (IsA(w, OpExpr)) + { + List *args = ((OpExpr *) w)->args; + + if (list_length(args) == 2 && + IsA(strip_implicit_coercions(linitial(args)), + CaseTestExpr)) + w = (Node *) lsecond(args); + } + } + + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "WHEN ", + 0, 0, 0); + get_rule_expr(w, context, false); + appendStringInfoString(buf, " THEN "); + get_rule_expr((Node *) when->result, context, true); + } + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "ELSE ", + 0, 0, 0); + get_rule_expr((Node *) caseexpr->defresult, context, true); + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "END", + -PRETTYINDENT_VAR, 0, 0); + } + break; + + case T_CaseTestExpr: + { + /* + * Normally we should never get here, since for expressions + * that can contain this node type we attempt to avoid + * recursing to it. But in an optimized expression we might + * be unable to avoid that (see comments for CaseExpr). If we + * do see one, print it as CASE_TEST_EXPR. + */ + appendStringInfoString(buf, "CASE_TEST_EXPR"); + } + break; + + case T_ArrayExpr: + { + ArrayExpr *arrayexpr = (ArrayExpr *) node; + + appendStringInfoString(buf, "ARRAY["); + get_rule_expr((Node *) arrayexpr->elements, context, true); + appendStringInfoChar(buf, ']'); + + /* + * If the array isn't empty, we assume its elements are + * coerced to the desired type. If it's empty, though, we + * need an explicit coercion to the array type. + */ + if (arrayexpr->elements == NIL) + appendStringInfo(buf, "::%s", + format_type_with_typemod(arrayexpr->array_typeid, -1)); + } + break; + + case T_RowExpr: + { + RowExpr *rowexpr = (RowExpr *) node; + TupleDesc tupdesc = NULL; + ListCell *arg; + int i; + char *sep; + + /* + * If it's a named type and not RECORD, we may have to skip + * dropped columns and/or claim there are NULLs for added + * columns. + */ + if (rowexpr->row_typeid != RECORDOID) + { + tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); + Assert(list_length(rowexpr->args) <= tupdesc->natts); + } + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "ROW("); + sep = ""; + i = 0; + foreach(arg, rowexpr->args) + { + Node *e = (Node *) lfirst(arg); + + if (tupdesc == NULL || + !tupdesc->attrs[i]->attisdropped) + { + appendStringInfoString(buf, sep); + /* Whole-row Vars need special treatment here */ + get_rule_expr_toplevel(e, context, true); + sep = ", "; + } + i++; + } + if (tupdesc != NULL) + { + while (i < tupdesc->natts) + { + if (!tupdesc->attrs[i]->attisdropped) + { + appendStringInfoString(buf, sep); + appendStringInfoString(buf, "NULL"); + sep = ", "; + } + i++; + } + + ReleaseTupleDesc(tupdesc); + } + appendStringInfoChar(buf, ')'); + if (rowexpr->row_format == COERCE_EXPLICIT_CAST) + appendStringInfo(buf, "::%s", + format_type_with_typemod(rowexpr->row_typeid, -1)); + } + break; + + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *) node; + ListCell *arg; + char *sep; + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "(ROW("); + sep = ""; + foreach(arg, rcexpr->largs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + + /* + * We assume that the name of the first-column operator will + * do for all the rest too. This is definitely open to + * failure, eg if some but not all operators were renamed + * since the construct was parsed, but there seems no way to + * be perfect. + */ + appendStringInfo(buf, ") %s ROW(", + generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs)))); + sep = ""; + foreach(arg, rcexpr->rargs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + appendStringInfoString(buf, "))"); + } + break; + + case T_CoalesceExpr: + { + CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; + + appendStringInfoString(buf, "COALESCE("); + get_rule_expr((Node *) coalesceexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_MinMaxExpr: + { + MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; + + switch (minmaxexpr->op) + { + case IS_GREATEST: + appendStringInfoString(buf, "GREATEST("); + break; + case IS_LEAST: + appendStringInfoString(buf, "LEAST("); + break; + } + get_rule_expr((Node *) minmaxexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *) node; + bool needcomma = false; + ListCell *arg; + ListCell *narg; + Const *con; + + switch (xexpr->op) + { + case IS_XMLCONCAT: + appendStringInfoString(buf, "XMLCONCAT("); + break; + case IS_XMLELEMENT: + appendStringInfoString(buf, "XMLELEMENT("); + break; + case IS_XMLFOREST: + appendStringInfoString(buf, "XMLFOREST("); + break; + case IS_XMLPARSE: + appendStringInfoString(buf, "XMLPARSE("); + break; + case IS_XMLPI: + appendStringInfoString(buf, "XMLPI("); + break; + case IS_XMLROOT: + appendStringInfoString(buf, "XMLROOT("); + break; + case IS_XMLSERIALIZE: + appendStringInfoString(buf, "XMLSERIALIZE("); + break; + case IS_DOCUMENT: + break; + } + if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) + { + if (xexpr->xmloption == XMLOPTION_DOCUMENT) + appendStringInfoString(buf, "DOCUMENT "); + else + appendStringInfoString(buf, "CONTENT "); + } + if (xexpr->name) + { + appendStringInfo(buf, "NAME %s", + quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); + needcomma = true; + } + if (xexpr->named_args) + { + if (xexpr->op != IS_XMLFOREST) + { + if (needcomma) + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, "XMLATTRIBUTES("); + needcomma = false; + } + forboth(arg, xexpr->named_args, narg, xexpr->arg_names) + { + Node *e = (Node *) lfirst(arg); + char *argname = strVal(lfirst(narg)); + + if (needcomma) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) e, context, true); + appendStringInfo(buf, " AS %s", + quote_identifier(map_xml_name_to_sql_identifier(argname))); + needcomma = true; + } + if (xexpr->op != IS_XMLFOREST) + appendStringInfoChar(buf, ')'); + } + if (xexpr->args) + { + if (needcomma) + appendStringInfoString(buf, ", "); + switch (xexpr->op) + { + case IS_XMLCONCAT: + case IS_XMLELEMENT: + case IS_XMLFOREST: + case IS_XMLPI: + case IS_XMLSERIALIZE: + /* no extra decoration needed */ + get_rule_expr((Node *) xexpr->args, context, true); + break; + case IS_XMLPARSE: + Assert(list_length(xexpr->args) == 2); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + con = lsecond_node(Const, xexpr->args); + Assert(!con->constisnull); + if (DatumGetBool(con->constvalue)) + appendStringInfoString(buf, + " PRESERVE WHITESPACE"); + else + appendStringInfoString(buf, + " STRIP WHITESPACE"); + break; + case IS_XMLROOT: + Assert(list_length(xexpr->args) == 3); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + appendStringInfoString(buf, ", VERSION "); + con = (Const *) lsecond(xexpr->args); + if (IsA(con, Const) && + con->constisnull) + appendStringInfoString(buf, "NO VALUE"); + else + get_rule_expr((Node *) con, context, false); + + con = lthird_node(Const, xexpr->args); + if (con->constisnull) + /* suppress STANDALONE NO VALUE */ ; + else + { + switch (DatumGetInt32(con->constvalue)) + { + case XML_STANDALONE_YES: + appendStringInfoString(buf, + ", STANDALONE YES"); + break; + case XML_STANDALONE_NO: + appendStringInfoString(buf, + ", STANDALONE NO"); + break; + case XML_STANDALONE_NO_VALUE: + appendStringInfoString(buf, + ", STANDALONE NO VALUE"); + break; + default: + break; + } + } + break; + case IS_DOCUMENT: + get_rule_expr_paren((Node *) xexpr->args, context, false, node); + break; + } + + } + if (xexpr->op == IS_XMLSERIALIZE) + appendStringInfo(buf, " AS %s", + format_type_with_typemod(xexpr->type, + xexpr->typmod)); + if (xexpr->op == IS_DOCUMENT) + appendStringInfoString(buf, " IS DOCUMENT"); + else + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullTest: + { + NullTest *ntest = (NullTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) ntest->arg, context, true, node); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, + * which is shorter and traditional. If it's a rowtype input + * but we're applying a scalar test, must print IS [NOT] + * DISTINCT FROM NULL to be semantically correct. + */ + if (ntest->argisrow || + !type_is_rowtype(exprType((Node *) ntest->arg))) + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS NOT NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + else + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS DISTINCT FROM NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BooleanTest: + { + BooleanTest *btest = (BooleanTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) btest->arg, context, false, node); + switch (btest->booltesttype) + { + case IS_TRUE: + appendStringInfoString(buf, " IS TRUE"); + break; + case IS_NOT_TRUE: + appendStringInfoString(buf, " IS NOT TRUE"); + break; + case IS_FALSE: + appendStringInfoString(buf, " IS FALSE"); + break; + case IS_NOT_FALSE: + appendStringInfoString(buf, " IS NOT FALSE"); + break; + case IS_UNKNOWN: + appendStringInfoString(buf, " IS UNKNOWN"); + break; + case IS_NOT_UNKNOWN: + appendStringInfoString(buf, " IS NOT UNKNOWN"); + break; + default: + elog(ERROR, "unrecognized booltesttype: %d", + (int) btest->booltesttype); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CoerceToDomain: + { + CoerceToDomain *ctest = (CoerceToDomain *) node; + Node *arg = (Node *) ctest->arg; + + if (ctest->coercionformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr(arg, context, false); + } + else + { + get_coercion_expr(arg, context, + ctest->resulttype, + ctest->resulttypmod, + node); + } + } + break; + + case T_CoerceToDomainValue: + appendStringInfoString(buf, "VALUE"); + break; + + case T_SetToDefault: + appendStringInfoString(buf, "DEFAULT"); + break; + + case T_CurrentOfExpr: + { + CurrentOfExpr *cexpr = (CurrentOfExpr *) node; + + if (cexpr->cursor_name) + appendStringInfo(buf, "CURRENT OF %s", + quote_identifier(cexpr->cursor_name)); + else + appendStringInfo(buf, "CURRENT OF $%d", + cexpr->cursor_param); + } + break; + + case T_InferenceElem: + { + InferenceElem *iexpr = (InferenceElem *) node; + bool save_varprefix; + bool need_parens; + + /* + * InferenceElem can only refer to target relation, so a + * prefix is not useful, and indeed would cause parse errors. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + /* + * Parenthesize the element unless it's a simple Var or a bare + * function call. Follows pg_get_indexdef_worker(). + */ + need_parens = !IsA(iexpr->expr, Var); + if (IsA(iexpr->expr, FuncExpr) && + ((FuncExpr *) iexpr->expr)->funcformat == + COERCE_EXPLICIT_CALL) + need_parens = false; + + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) iexpr->expr, + context, false); + if (need_parens) + appendStringInfoChar(buf, ')'); + + context->varprefix = save_varprefix; + + if (iexpr->infercollid) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(iexpr->infercollid)); + + /* Add the operator class name, if not default */ + if (iexpr->inferopclass) + { + Oid inferopclass = iexpr->inferopclass; + Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); + + get_opclass_name(inferopclass, inferopcinputtype, buf); + } + } + break; + + case T_PartitionBoundSpec: + { + PartitionBoundSpec *spec = (PartitionBoundSpec *) node; + ListCell *cell; + char *sep; + + switch (spec->strategy) + { + case PARTITION_STRATEGY_LIST: + Assert(spec->listdatums != NIL); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfoString(buf, " IN ("); + sep = ""; + foreach(cell, spec->listdatums) + { + Const *val = lfirst(cell); + + appendStringInfoString(buf, sep); + get_const_expr(val, context, -1); + sep = ", "; + } + + appendStringInfoString(buf, ")"); + break; + + case PARTITION_STRATEGY_RANGE: + Assert(spec->lowerdatums != NIL && + spec->upperdatums != NIL && + list_length(spec->lowerdatums) == + list_length(spec->upperdatums)); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfoString(buf, " FROM"); + appendStringInfoString(buf, " ("); + sep = ""; + foreach(cell, spec->lowerdatums) + { + PartitionRangeDatum *datum = lfirst(cell); + Const *val; + + appendStringInfoString(buf, sep); + if (datum->infinite) + appendStringInfoString(buf, "UNBOUNDED"); + else + { + val = (Const *) datum->value; + get_const_expr(val, context, -1); + } + sep = ", "; + } + appendStringInfoString(buf, ")"); + + appendStringInfoString(buf, " TO"); + appendStringInfoString(buf, " ("); + sep = ""; + foreach(cell, spec->upperdatums) + { + PartitionRangeDatum *datum = lfirst(cell); + Const *val; + + appendStringInfoString(buf, sep); + if (datum->infinite) + appendStringInfoString(buf, "UNBOUNDED"); + else + { + val = (Const *) datum->value; + get_const_expr(val, context, -1); + } + sep = ", "; + } + appendStringInfoString(buf, ")"); + break; + + default: + elog(ERROR, "unrecognized partition strategy: %d", + (int) spec->strategy); + break; + } + } + break; + + case T_List: + { + char *sep; + ListCell *l; + + sep = ""; + foreach(l, (List *) node) + { + appendStringInfoString(buf, sep); + get_rule_expr((Node *) lfirst(l), context, showimplicit); + sep = ", "; + } + } + break; + + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + + default: + elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); + break; + } +} + +/* + * get_rule_expr_toplevel - Parse back a toplevel expression + * + * Same as get_rule_expr(), except that if the expr is just a Var, we pass + * istoplevel = true not false to get_variable(). This causes whole-row Vars + * to get printed with decoration that will prevent expansion of "*". + * We need to use this in contexts such as ROW() and VALUES(), where the + * parser would expand "foo.*" appearing at top level. (In principle we'd + * use this in get_target_list() too, but that has additional worries about + * whether to print AS, so it needs to invoke get_variable() directly anyway.) + */ +static void +get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit) +{ + if (node && IsA(node, Var)) + (void) get_variable((Var *) node, 0, true, context); + else + get_rule_expr(node, context, showimplicit); +} + + +/* + * get_oper_expr - Parse back an OpExpr node + */ +static void +get_oper_expr(OpExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid opno = expr->opno; + List *args = expr->args; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + get_rule_expr_paren(arg1, context, true, (Node *) expr); + appendStringInfo(buf, " %s ", + generate_operator_name(opno, + exprType(arg1), + exprType(arg2))); + get_rule_expr_paren(arg2, context, true, (Node *) expr); + } + else + { + /* unary operator --- but which side? */ + Node *arg = (Node *) linitial(args); + HeapTuple tp; + Form_pg_operator optup; + + tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for operator %u", opno); + optup = (Form_pg_operator) GETSTRUCT(tp); + switch (optup->oprkind) + { + case 'l': + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); + break; + case 'r': + get_rule_expr_paren(arg, context, true, (Node *) expr); + appendStringInfo(buf, " %s", + generate_operator_name(opno, + exprType(arg), + InvalidOid)); + break; + default: + elog(ERROR, "bogus oprkind: %d", optup->oprkind); + } + ReleaseSysCache(tp); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); +} + +/* + * get_func_expr - Parse back a FuncExpr node + */ +static void +get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + bool use_variadic; + ListCell *l; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument --- unless caller wants to see implicit coercions. + */ + if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) + { + get_rule_expr_paren((Node *) linitial(expr->args), context, + false, (Node *) expr); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (expr->funcformat == COERCE_EXPLICIT_CAST || + expr->funcformat == COERCE_IMPLICIT_CAST) + { + Node *arg = linitial(expr->args); + Oid rettype = expr->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); + + get_coercion_expr(arg, context, + rettype, coercedTypmod, + (Node *) expr); + + return; + } + + /* + * Normal function: display as proname(args). First we need to extract + * the argument datatypes. + */ + if (list_length(expr->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, expr->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(funcoid, nargs, + argnames, argtypes, + expr->funcvariadic, + &use_variadic, + context->special_exprkind)); + nargs = 0; + foreach(l, expr->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(l) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(l), context, true); + } + appendStringInfoChar(buf, ')'); +} + +/* + * get_agg_expr - Parse back an Aggref node + */ +static void +get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + bool use_variadic; + + /* + * For a combining aggregate, we look up and deparse the corresponding + * partial aggregate instead. This is necessary because our input + * argument list has been replaced; the new argument list always has just + * one element, which will point to a partial Aggref that supplies us with + * transition states to combine. + */ + if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) + { + TargetEntry *tle = linitial_node(TargetEntry, aggref->args); + + Assert(list_length(aggref->args) == 1); + resolve_special_varno((Node *) tle->expr, context, original_aggref, + get_agg_combine_expr); + return; + } + + /* + * Mark as PARTIAL, if appropriate. We look to the original aggref so as + * to avoid printing this when recursing from the code just above. + */ + if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) + appendStringInfoString(buf, "PARTIAL "); + + /* Extract the argument types as seen by the parser */ + nargs = get_aggregate_argtypes(aggref, argtypes); + + /* Print the aggregate name, schema-qualified if needed */ + appendStringInfo(buf, "%s(%s", + generate_function_name(aggref->aggfnoid, nargs, + NIL, argtypes, + aggref->aggvariadic, + &use_variadic, + context->special_exprkind), + (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); + + if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) + { + /* + * Ordered-set aggregates do not use "*" syntax. Also, we needn't + * worry about inserting VARIADIC. So we can just dump the direct + * args as-is. + */ + Assert(!aggref->aggvariadic); + get_rule_expr((Node *) aggref->aggdirectargs, context, true); + Assert(aggref->aggorder != NIL); + appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + else + { + /* aggstar can be set only in zero-argument aggregates */ + if (aggref->aggstar) + appendStringInfoChar(buf, '*'); + else + { + ListCell *l; + int i; + + i = 0; + foreach(l, aggref->args) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *arg = (Node *) tle->expr; + + Assert(!IsA(arg, NamedArgExpr)); + if (tle->resjunk) + continue; + if (i++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && i == nargs) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr(arg, context, true); + } + } + + if (aggref->aggorder != NIL) + { + appendStringInfoString(buf, " ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + } + + if (aggref->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) aggref->aggfilter, context, false); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * This is a helper function for get_agg_expr(). It's used when we deparse + * a combining Aggref; resolve_special_varno locates the corresponding partial + * Aggref and then calls this. + */ +static void +get_agg_combine_expr(Node *node, deparse_context *context, void *private) +{ + Aggref *aggref; + Aggref *original_aggref = private; + + if (!IsA(node, Aggref)) + elog(ERROR, "combining Aggref does not point to an Aggref"); + + aggref = (Aggref *) node; + get_agg_expr(aggref, context, original_aggref); +} + +/* + * get_windowfunc_expr - Parse back a WindowFunc node + */ +static void +get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + ListCell *l; + + if (list_length(wfunc->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, wfunc->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(wfunc->winfnoid, nargs, + argnames, argtypes, + false, NULL, + context->special_exprkind)); + /* winstar can be set only in zero-argument aggregates */ + if (wfunc->winstar) + appendStringInfoChar(buf, '*'); + else + get_rule_expr((Node *) wfunc->args, context, true); + + if (wfunc->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) wfunc->aggfilter, context, false); + } + + appendStringInfoString(buf, ") OVER "); + + foreach(l, context->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->winref == wfunc->winref) + { + if (wc->name) + appendStringInfoString(buf, quote_identifier(wc->name)); + else + get_rule_windowspec(wc, context->windowTList, context); + break; + } + } + if (l == NULL) + { + if (context->windowClause) + elog(ERROR, "could not find window clause for winref %u", + wfunc->winref); + + /* + * In EXPLAIN, we don't have window context information available, so + * we have to settle for this: + */ + appendStringInfoString(buf, "(?)"); + } +} + +/* ---------- + * get_coercion_expr + * + * Make a string representation of a value coerced to a specific type + * ---------- + */ +static void +get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode) +{ + StringInfo buf = context->buf; + + /* + * Since parse_coerce.c doesn't immediately collapse application of + * length-coercion functions to constants, what we'll typically see in + * such cases is a Const with typmod -1 and a length-coercion function + * right above it. Avoid generating redundant output. However, beware of + * suppressing casts when the user actually wrote something like + * 'foo'::text::char(3). + * + * Note: it might seem that we are missing the possibility of needing to + * print a COLLATE clause for such a Const. However, a Const could only + * have nondefault collation in a post-constant-folding tree, in which the + * length coercion would have been folded too. See also the special + * handling of CollateExpr in coerce_to_target_type(): any collation + * marking will be above the coercion node, not below it. + */ + if (arg && IsA(arg, Const) && + ((Const *) arg)->consttype == resulttype && + ((Const *) arg)->consttypmod == -1) + { + /* Show the constant without normal ::typename decoration */ + get_const_expr((Const *) arg, context, -1); + } + else + { + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, false, parentNode); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + appendStringInfo(buf, "::%s", + format_type_with_typemod(resulttype, resulttypmod)); +} + +/* ---------- + * get_const_expr + * + * Make a string representation of a Const + * + * showtype can be -1 to never show "::typename" decoration, or +1 to always + * show it, or 0 to show it only if the constant wouldn't be assumed to be + * the right type by default. + * + * If the Const's collation isn't default for its type, show that too. + * We mustn't do this when showtype is -1 (since that means the caller will + * print "::typename", and we can't put a COLLATE clause in between). It's + * caller's responsibility that collation isn't missed in such cases. + * ---------- + */ +static void +get_const_expr(Const *constval, deparse_context *context, int showtype) +{ + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char *extval; + bool needlabel = false; + + if (constval->constisnull) + { + /* + * Always label the type of a NULL constant to prevent misdecisions + * about type when reparsing. + */ + appendStringInfoString(buf, "NULL"); + if (showtype >= 0) + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + get_const_collation(constval, context); + } + return; + } + + getTypeOutputInfo(constval->consttype, + &typoutput, &typIsVarlena); + + extval = OidOutputFunctionCall(typoutput, constval->constvalue); + + switch (constval->consttype) + { + case INT4OID: + + /* + * INT4 can be printed without any decoration, unless it is + * negative; in that case print it as '-nnn'::integer to ensure + * that the output will re-parse as a constant, not as a constant + * plus operator. In most cases we could get away with printing + * (-nnn) instead, because of the way that gram.y handles negative + * literals; but that doesn't work for INT_MIN, and it doesn't + * seem that much prettier anyway. + */ + if (extval[0] != '-') + appendStringInfoString(buf, extval); + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case NUMERICOID: + + /* + * NUMERIC can be printed without quotes if it looks like a float + * constant (not an integer, and not Infinity or NaN) and doesn't + * have a leading sign (for the same reason as for INT4). + */ + if (isdigit((unsigned char) extval[0]) && + strcspn(extval, "eE.") != strlen(extval)) + { + appendStringInfoString(buf, extval); + } + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + + case BOOLOID: + if (strcmp(extval, "t") == 0) + appendStringInfoString(buf, "true"); + else + appendStringInfoString(buf, "false"); + break; + + default: + simple_quote_literal(buf, extval); + break; + } + + pfree(extval); + + if (showtype < 0) + return; + + /* + * For showtype == 0, append ::typename unless the constant will be + * implicitly typed as the right type when it is read in. + * + * XXX this code has to be kept in sync with the behavior of the parser, + * especially make_const. + */ + switch (constval->consttype) + { + case BOOLOID: + case UNKNOWNOID: + /* These types can be left unlabeled */ + needlabel = false; + break; + case INT4OID: + /* We determined above whether a label is needed */ + break; + case NUMERICOID: + + /* + * Float-looking constants will be typed as numeric, which we + * checked above; but if there's a nondefault typmod we need to + * show it. + */ + needlabel |= (constval->consttypmod >= 0); + break; + default: + needlabel = true; + break; + } + if (needlabel || showtype > 0) + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + + get_const_collation(constval, context); +} + +/* + * helper for get_const_expr: append COLLATE if needed + */ +static void +get_const_collation(Const *constval, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (OidIsValid(constval->constcollid)) + { + Oid typcollation = get_typcollation(constval->consttype); + + if (constval->constcollid != typcollation) + { + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(constval->constcollid)); + } + } +} + +/* + * simple_quote_literal - Format a string as a SQL literal, append to buf + */ +static void +simple_quote_literal(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * We form the string literal according to the prevailing setting of + * standard_conforming_strings; we never use E''. User is responsible for + * making sure result is used correctly. + */ + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} + + +/* ---------- + * get_sublink_expr - Parse back a sublink + * ---------- + */ +static void +get_sublink_expr(SubLink *sublink, deparse_context *context) +{ + StringInfo buf = context->buf; + Query *query = (Query *) (sublink->subselect); + char *opname = NULL; + bool need_paren; + + if (sublink->subLinkType == ARRAY_SUBLINK) + appendStringInfoString(buf, "ARRAY("); + else + appendStringInfoChar(buf, '('); + + /* + * Note that we print the name of only the first operator, when there are + * multiple combining operators. This is an approximation that could go + * wrong in various scenarios (operators in different schemas, renamed + * operators, etc) but there is not a whole lot we can do about it, since + * the syntax allows only one operator to be shown. + */ + if (sublink->testexpr) + { + if (IsA(sublink->testexpr, OpExpr)) + { + /* single combining operator */ + OpExpr *opexpr = (OpExpr *) sublink->testexpr; + + get_rule_expr(linitial(opexpr->args), context, true); + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + } + else if (IsA(sublink->testexpr, BoolExpr)) + { + /* multiple combining operators, = or <> cases */ + char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(l, ((BoolExpr *) sublink->testexpr)->args) + { + OpExpr *opexpr = lfirst_node(OpExpr, l); + + appendStringInfoString(buf, sep); + get_rule_expr(linitial(opexpr->args), context, true); + if (!opname) + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else if (IsA(sublink->testexpr, RowCompareExpr)) + { + /* multiple combining operators, < <= > >= cases */ + RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) rcexpr->largs, context, true); + opname = generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs))); + appendStringInfoChar(buf, ')'); + } + else + elog(ERROR, "unrecognized testexpr type: %d", + (int) nodeTag(sublink->testexpr)); + } + + need_paren = true; + + switch (sublink->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS "); + break; + + case ANY_SUBLINK: + if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ + appendStringInfoString(buf, " IN "); + else + appendStringInfo(buf, " %s ANY ", opname); + break; + + case ALL_SUBLINK: + appendStringInfo(buf, " %s ALL ", opname); + break; + + case ROWCOMPARE_SUBLINK: + appendStringInfo(buf, " %s ", opname); + break; + + case EXPR_SUBLINK: + case MULTIEXPR_SUBLINK: + case ARRAY_SUBLINK: + need_paren = false; + break; + + case CTE_SUBLINK: /* shouldn't occur in a SubLink */ + default: + elog(ERROR, "unrecognized sublink type: %d", + (int) sublink->subLinkType); + break; + } + + if (need_paren) + appendStringInfoChar(buf, '('); + + get_query_def(query, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + + if (need_paren) + appendStringInfoString(buf, "))"); + else + appendStringInfoChar(buf, ')'); +} + + +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + /* XMLTABLE is the only existing implementation. */ + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + l2 = list_head(tf->coltypes); + l3 = list_head(tf->coltypmods); + l4 = list_head(tf->colexprs); + l5 = list_head(tf->coldefexprs); + + appendStringInfoString(buf, " COLUMNS "); + foreach(l1, tf->colnames) + { + char *colname = strVal(lfirst(l1)); + Oid typid; + int32 typmod; + Node *colexpr; + Node *coldefexpr; + bool ordinality = tf->ordinalitycol == colnum; + bool notnull = bms_is_member(colnum, tf->notnulls); + + typid = lfirst_oid(l2); + l2 = lnext(l2); + typmod = lfirst_int(l3); + l3 = lnext(l3); + colexpr = (Node *) lfirst(l4); + l4 = lnext(l4); + coldefexpr = (Node *) lfirst(l5); + l5 = lnext(l5); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_from_clause - Parse back a FROM clause + * + * "prefix" is the keyword that denotes the start of the list of FROM + * elements. It is FROM when used to parse back SELECT and UPDATE, but + * is USING when parsing back DELETE. + * ---------- + */ +static void +get_from_clause(Query *query, const char *prefix, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first = true; + ListCell *l; + + /* + * We use the query's jointree as a guide to what to print. However, we + * must ignore auto-added RTEs that are marked not inFromCl. (These can + * only appear at the top level of the jointree, so it's sufficient to + * check here.) This check also ensures we ignore the rule pseudo-RTEs + * for NEW and OLD. + */ + foreach(l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + + if (!rte->inFromCl) + continue; + } + + if (first) + { + appendContextKeyword(context, prefix, + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + first = false; + + get_from_clause_item(jtnode, query, context); + } + else + { + StringInfoData itembuf; + + appendStringInfoString(buf, ", "); + + /* + * Put the new FROM item's text into itembuf so we can decide + * after we've got it whether or not it needs to go on a new line. + */ + initStringInfo(&itembuf); + context->buf = &itembuf; + + get_from_clause_item(jtnode, query, context); + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + /* Does the new item start with a new line? */ + if (itembuf.len > 0 && itembuf.data[0] == '\n') + { + /* If so, we shouldn't add anything */ + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new item + * would cause an overflow. + */ + if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_VAR); + } + } + + /* Add the new item */ + appendStringInfoString(buf, itembuf.data); + + /* clean up */ + pfree(itembuf.data); + } + } +} + +static void +get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + char *refname = get_rtable_name(varno, context); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + RangeTblFunction *rtfunc1 = NULL; + bool printalias; + + if (rte->lateral) + appendStringInfoString(buf, "LATERAL "); + + /* Print the FROM item proper */ + switch (rte->rtekind) + { + case RTE_RELATION: + /* Normal relation RTE */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, + context->namespaces)); + break; + case RTE_SUBQUERY: + /* Subquery RTE */ + appendStringInfoChar(buf, '('); + get_query_def(rte->subquery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + appendStringInfoChar(buf, ')'); + break; + case RTE_FUNCTION: + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* Use schema and table name from the remote alias */ + appendStringInfoString(buf, + generate_fragment_name(fragmentSchemaName, + fragmentTableName)); + break; + } + + /* Function RTE */ + rtfunc1 = (RangeTblFunction *) linitial(rte->functions); + + /* + * Omit ROWS FROM() syntax for just one function, unless it + * has both a coldeflist and WITH ORDINALITY. If it has both, + * we must use ROWS FROM() syntax to avoid ambiguity about + * whether the coldeflist includes the ordinality column. + */ + if (list_length(rte->functions) == 1 && + (rtfunc1->funccolnames == NIL || !rte->funcordinality)) + { + get_rule_expr(rtfunc1->funcexpr, context, true); + /* we'll print the coldeflist below, if it has one */ + } + else + { + bool all_unnest; + ListCell *lc; + + /* + * If all the function calls in the list are to unnest, + * and none need a coldeflist, then collapse the list back + * down to UNNEST(args). (If we had more than one + * built-in unnest function, this would get more + * difficult.) + * + * XXX This is pretty ugly, since it makes not-terribly- + * future-proof assumptions about what the parser would do + * with the output; but the alternative is to emit our + * nonstandard ROWS FROM() notation for what might have + * been a perfectly spec-compliant multi-argument + * UNNEST(). + */ + all_unnest = true; + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (!IsA(rtfunc->funcexpr, FuncExpr) || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + rtfunc->funccolnames != NIL) + { + all_unnest = false; + break; + } + } + + if (all_unnest) + { + List *allargs = NIL; + + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + List *args = ((FuncExpr *) rtfunc->funcexpr)->args; + + allargs = list_concat(allargs, list_copy(args)); + } + + appendStringInfoString(buf, "UNNEST("); + get_rule_expr((Node *) allargs, context, true); + appendStringInfoChar(buf, ')'); + } + else + { + int funcno = 0; + + appendStringInfoString(buf, "ROWS FROM("); + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (funcno > 0) + appendStringInfoString(buf, ", "); + get_rule_expr(rtfunc->funcexpr, context, true); + if (rtfunc->funccolnames != NIL) + { + /* Reconstruct the column definition list */ + appendStringInfoString(buf, " AS "); + get_from_clause_coldeflist(rtfunc, + NULL, + context); + } + funcno++; + } + appendStringInfoChar(buf, ')'); + } + /* prevent printing duplicate coldeflist below */ + rtfunc1 = NULL; + } + if (rte->funcordinality) + appendStringInfoString(buf, " WITH ORDINALITY"); + break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; + case RTE_VALUES: + /* Values list RTE */ + appendStringInfoChar(buf, '('); + get_values_def(rte->values_lists, context); + appendStringInfoChar(buf, ')'); + break; + case RTE_CTE: + appendStringInfoString(buf, quote_identifier(rte->ctename)); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + break; + } + + /* Print the relation alias, if needed */ + printalias = false; + if (rte->alias != NULL) + { + /* Always print alias if user provided one */ + printalias = true; + } + else if (colinfo->printaliases) + { + /* Always print alias if we need to print column aliases */ + printalias = true; + } + else if (rte->rtekind == RTE_RELATION) + { + /* + * No need to print alias if it's same as relation name (this + * would normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, get_relation_name(rte->relid)) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_FUNCTION) + { + /* + * For a function RTE, always print alias. This covers possible + * renaming of the function and/or instability of the + * FigureColname rules for things that aren't simple functions. + * Note we'd need to force it anyway for the columndef list case. + */ + printalias = true; + } + else if (rte->rtekind == RTE_VALUES) + { + /* Alias is syntactically required for VALUES */ + printalias = true; + } + else if (rte->rtekind == RTE_CTE) + { + /* + * No need to print alias if it's same as CTE name (this would + * normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, rte->ctename) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_SUBQUERY) + { + /* subquery requires alias too */ + printalias = true; + } + if (printalias) + appendStringInfo(buf, " %s", quote_identifier(refname)); + + /* Print the column definitions or aliases, if needed */ + if (rtfunc1 && rtfunc1->funccolnames != NIL) + { + /* Reconstruct the columndef list, which is also the aliases */ + get_from_clause_coldeflist(rtfunc1, colinfo, context); + } + else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD) + { + /* Else print column aliases as needed */ + get_column_alias_list(colinfo, context); + } + + /* Tablesample clause must go after any alias */ + if (rte->rtekind == RTE_RELATION && rte->tablesample) + get_tablesample_def(rte->tablesample, context); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + bool need_paren_on_right; + + need_paren_on_right = PRETTY_PAREN(context) && + !IsA(j->rarg, RangeTblRef) && + !(IsA(j->rarg, JoinExpr) &&((JoinExpr *) j->rarg)->alias != NULL); + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, '('); + + get_from_clause_item(j->larg, query, context); + + switch (j->jointype) + { + case JOIN_INNER: + if (j->quals) + appendContextKeyword(context, " JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + else + appendContextKeyword(context, " CROSS JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_LEFT: + appendContextKeyword(context, " LEFT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_FULL: + appendContextKeyword(context, " FULL JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_RIGHT: + appendContextKeyword(context, " RIGHT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + default: + elog(ERROR, "unrecognized join type: %d", + (int) j->jointype); + } + + if (need_paren_on_right) + appendStringInfoChar(buf, '('); + get_from_clause_item(j->rarg, query, context); + if (need_paren_on_right) + appendStringInfoChar(buf, ')'); + + if (j->usingClause) + { + ListCell *lc; + bool first = true; + + appendStringInfoString(buf, " USING ("); + /* Use the assigned names, not what's in usingClause */ + foreach(lc, colinfo->usingNames) + { + char *colname = (char *) lfirst(lc); + + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + appendStringInfoChar(buf, ')'); + } + else if (j->quals) + { + appendStringInfoString(buf, " ON "); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr(j->quals, context, false); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, ')'); + + /* Yes, it's correct to put alias after the right paren ... */ + if (j->alias != NULL) + { + appendStringInfo(buf, " %s", + quote_identifier(j->alias->aliasname)); + get_column_alias_list(colinfo, context); + } + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * get_column_alias_list - print column alias list for an RTE + * + * Caller must already have printed the relation's alias name. + */ +static void +get_column_alias_list(deparse_columns *colinfo, deparse_context *context) +{ + StringInfo buf = context->buf; + int i; + bool first = true; + + /* Don't print aliases if not needed */ + if (!colinfo->printaliases) + return; + + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *colname = colinfo->new_colnames[i]; + + if (first) + { + appendStringInfoChar(buf, '('); + first = false; + } + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + if (!first) + appendStringInfoChar(buf, ')'); +} + +/* + * get_from_clause_coldeflist - reproduce FROM clause coldeflist + * + * When printing a top-level coldeflist (which is syntactically also the + * relation's column alias list), use column names from colinfo. But when + * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the + * original coldeflist's names, which are available in rtfunc->funccolnames. + * Pass NULL for colinfo to select the latter behavior. + * + * The coldeflist is appended immediately (no space) to buf. Caller is + * responsible for ensuring that an alias or AS is present before it. + */ +static void +get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + int i; + + appendStringInfoChar(buf, '('); + + /* there's no forfour(), so must chase one list the hard way */ + i = 0; + l4 = list_head(rtfunc->funccolnames); + forthree(l1, rtfunc->funccoltypes, + l2, rtfunc->funccoltypmods, + l3, rtfunc->funccolcollations) + { + Oid atttypid = lfirst_oid(l1); + int32 atttypmod = lfirst_int(l2); + Oid attcollation = lfirst_oid(l3); + char *attname; + + if (colinfo) + attname = colinfo->colnames[i]; + else + attname = strVal(lfirst(l4)); + + Assert(attname); /* shouldn't be any dropped columns here */ + + if (i > 0) + appendStringInfoString(buf, ", "); + appendStringInfo(buf, "%s %s", + quote_identifier(attname), + format_type_with_typemod(atttypid, atttypmod)); + if (OidIsValid(attcollation) && + attcollation != get_typcollation(atttypid)) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(attcollation)); + + l4 = lnext(l4); + i++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_tablesample_def - print a TableSampleClause + */ +static void +get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[1]; + int nargs; + ListCell *l; + + /* + * We should qualify the handler's function name if it wouldn't be + * resolved by lookup in the current search path. + */ + argtypes[0] = INTERNALOID; + appendStringInfo(buf, " TABLESAMPLE %s (", + generate_function_name(tablesample->tsmhandler, 1, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + nargs = 0; + foreach(l, tablesample->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfirst(l), context, false); + } + appendStringInfoChar(buf, ')'); + + if (tablesample->repeatable != NULL) + { + appendStringInfoString(buf, " REPEATABLE ("); + get_rule_expr((Node *) tablesample->repeatable, context, false); + appendStringInfoChar(buf, ')'); + } +} + +/* + * get_opclass_name - fetch name of an index operator class + * + * The opclass name is appended (after a space) to buf. + * + * Output is suppressed if the opclass is the default for the given + * actual_datatype. (If you don't want this behavior, just pass + * InvalidOid for actual_datatype.) + */ +static void +get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf) +{ + HeapTuple ht_opc; + Form_pg_opclass opcrec; + char *opcname; + char *nspname; + + ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); + if (!HeapTupleIsValid(ht_opc)) + elog(ERROR, "cache lookup failed for opclass %u", opclass); + opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); + + if (!OidIsValid(actual_datatype) || + GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) + { + /* Okay, we need the opclass name. Do we need to qualify it? */ + opcname = NameStr(opcrec->opcname); + if (OpclassIsVisible(opclass)) + appendStringInfo(buf, " %s", quote_identifier(opcname)); + else + { + nspname = get_namespace_name(opcrec->opcnamespace); + appendStringInfo(buf, " %s.%s", + quote_identifier(nspname), + quote_identifier(opcname)); + } + } + ReleaseSysCache(ht_opc); +} + +/* + * processIndirection - take care of array and subfield assignment + * + * We strip any top-level FieldStore or assignment ArrayRef nodes that + * appear in the input, printing them as decoration for the base column + * name (which we assume the caller just printed). Return the subexpression + * that's to be assigned. + */ +static Node * +processIndirection(Node *node, deparse_context *context) +{ + StringInfo buf = context->buf; + + for (;;) + { + if (node == NULL) + break; + if (IsA(node, FieldStore)) + { + FieldStore *fstore = (FieldStore *) node; + Oid typrelid; + char *fieldname; + + /* lookup tuple type */ + typrelid = get_typ_typrelid(fstore->resulttype); + if (!OidIsValid(typrelid)) + elog(ERROR, "argument type %s of FieldStore is not a tuple type", + format_type_be(fstore->resulttype)); + + /* + * Print the field name. There should only be one target field in + * stored rules. There could be more than that in executable + * target lists, but this function cannot be used for that case. + */ + Assert(list_length(fstore->fieldnums) == 1); + fieldname = get_relid_attribute_name(typrelid, + linitial_int(fstore->fieldnums)); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + + /* + * We ignore arg since it should be an uninteresting reference to + * the target column or subcolumn. + */ + node = (Node *) linitial(fstore->newvals); + } + else if (IsA(node, ArrayRef)) + { + ArrayRef *aref = (ArrayRef *) node; + + if (aref->refassgnexpr == NULL) + break; + printSubscripts(aref, context); + + /* + * We ignore refexpr since it should be an uninteresting reference + * to the target column or subcolumn. + */ + node = (Node *) aref->refassgnexpr; + } + else + break; + } + + return node; +} + +static void +printSubscripts(ArrayRef *aref, deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *lowlist_item; + ListCell *uplist_item; + + lowlist_item = list_head(aref->reflowerindexpr); /* could be NULL */ + foreach(uplist_item, aref->refupperindexpr) + { + appendStringInfoChar(buf, '['); + if (lowlist_item) + { + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(lowlist_item), context, false); + appendStringInfoChar(buf, ':'); + lowlist_item = lnext(lowlist_item); + } + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(uplist_item), context, false); + appendStringInfoChar(buf, ']'); + } +} + +/* + * get_relation_name + * Get the unqualified name of a relation specified by OID + * + * This differs from the underlying get_rel_name() function in that it will + * throw error instead of silently returning NULL if the OID is bad. + */ +static char * +get_relation_name(Oid relid) +{ + char *relname = get_rel_name(relid); + + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", relid); + return relname; +} + +/* + * generate_relation_or_shard_name + * Compute the name to display for a relation or shard + * + * If the provided relid is equal to the provided distrelid, this function + * returns a shard-extended relation name; otherwise, it falls through to a + * simple generate_relation_name call. + */ +static char * +generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, + List *namespaces) +{ + char *relname = NULL; + + if (relid == distrelid) + { + relname = get_relation_name(relid); + + if (shardid > 0) + { + Oid schemaOid = get_rel_namespace(relid); + char *schemaName = get_namespace_name(schemaOid); + + AppendShardIdToName(&relname, shardid); + + relname = quote_qualified_identifier(schemaName, relname); + } + } + else + { + relname = generate_relation_name(relid, namespaces); + } + + return relname; +} + +/* + * generate_relation_name + * Compute the name to display for a relation specified by OID + * + * The result includes all necessary quoting and schema-prefixing. + * + * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. + * We will forcibly qualify the relation name if it equals any CTE name + * visible in the namespace list. + */ +char * +generate_relation_name(Oid relid, List *namespaces) +{ + HeapTuple tp; + Form_pg_class reltup; + bool need_qual; + ListCell *nslist; + char *relname; + char *nspname; + char *result; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for relation %u", relid); + reltup = (Form_pg_class) GETSTRUCT(tp); + relname = NameStr(reltup->relname); + + /* Check for conflicting CTE name */ + need_qual = false; + foreach(nslist, namespaces) + { + deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); + ListCell *ctlist; + + foreach(ctlist, dpns->ctes) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); + + if (strcmp(cte->ctename, relname) == 0) + { + need_qual = true; + break; + } + } + if (need_qual) + break; + } + + /* Otherwise, qualify the name if not visible in search path */ + if (!need_qual) + need_qual = !RelationIsVisible(relid); + + if (need_qual) + nspname = get_namespace_name(reltup->relnamespace); + else + nspname = NULL; + + result = quote_qualified_identifier(nspname, relname); + + ReleaseSysCache(tp); + + return result; +} + +/* + * generate_fragment_name + * Compute the name to display for a shard or merged table + * + * The result includes all necessary quoting and schema-prefixing. The schema + * name can be NULL for regular shards. For merged tables, they are always + * declared within a job-specific schema, and therefore can't have null schema + * names. + */ +static char * +generate_fragment_name(char *schemaName, char *tableName) +{ + StringInfo fragmentNameString = makeStringInfo(); + + if (schemaName != NULL) + { + appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), + quote_identifier(tableName)); + } + else + { + appendStringInfoString(fragmentNameString, quote_identifier(tableName)); + } + + return fragmentNameString->data; +} + +/* + * generate_function_name + * Compute the name to display for a function specified by OID, + * given that it is being called with the specified actual arg names and + * types. (Those matter because of ambiguous-function resolution rules.) + * + * If we're dealing with a potentially variadic function (in practice, this + * means a FuncExpr or Aggref, not some other way of calling a function), then + * has_variadic must specify whether variadic arguments have been merged, + * and *use_variadic_p will be set to indicate whether to print VARIADIC in + * the output. For non-FuncExpr cases, has_variadic should be FALSE and + * use_variadic_p can be NULL. + * + * The result includes all necessary quoting and schema-prefixing. + */ +static char * +generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind) +{ + char *result; + HeapTuple proctup; + Form_pg_proc procform; + char *proname; + bool use_variadic; + char *nspname; + FuncDetailCode p_result; + Oid p_funcid; + Oid p_rettype; + bool p_retset; + int p_nvargs; + Oid p_vatype; + Oid *p_true_typeids; + bool force_qualify = false; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + procform = (Form_pg_proc) GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* + * Due to parser hacks to avoid needing to reserve CUBE, we need to force + * qualification in some special cases. + */ + if (special_exprkind == EXPR_KIND_GROUP_BY) + { + if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) + force_qualify = true; + } + + /* + * Determine whether VARIADIC should be printed. We must do this first + * since it affects the lookup rules in func_get_detail(). + * + * Currently, we always print VARIADIC if the function has a merged + * variadic-array argument. Note that this is always the case for + * functions taking a VARIADIC argument type other than VARIADIC ANY. + * + * In principle, if VARIADIC wasn't originally specified and the array + * actual argument is deconstructable, we could print the array elements + * separately and not print VARIADIC, thus more nearly reproducing the + * original input. For the moment that seems like too much complication + * for the benefit, and anyway we do not know whether VARIADIC was + * originally specified if it's a non-ANY type. + */ + if (use_variadic_p) + { + /* Parser should not have set funcvariadic unless fn is variadic */ + Assert(!has_variadic || OidIsValid(procform->provariadic)); + use_variadic = has_variadic; + *use_variadic_p = use_variadic; + } + else + { + Assert(!has_variadic); + use_variadic = false; + } + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct function given the unqualified func name with the + * specified argtypes and VARIADIC flag. But if we already decided to + * force qualification, then we can skip the lookup and pretend we didn't + * find it. + */ + if (!force_qualify) + p_result = func_get_detail(list_make1(makeString(proname)), + NIL, argnames, nargs, argtypes, + !use_variadic, true, + &p_funcid, &p_rettype, + &p_retset, &p_nvargs, &p_vatype, + &p_true_typeids, NULL); + else + { + p_result = FUNCDETAIL_NOTFOUND; + p_funcid = InvalidOid; + } + + if ((p_result == FUNCDETAIL_NORMAL || + p_result == FUNCDETAIL_AGGREGATE || + p_result == FUNCDETAIL_WINDOWFUNC) && + p_funcid == funcid) + nspname = NULL; + else + nspname = get_namespace_name(procform->pronamespace); + + result = quote_qualified_identifier(nspname, proname); + + ReleaseSysCache(proctup); + + return result; +} + +/* + * generate_operator_name + * Compute the name to display for an operator specified by OID, + * given that it is being called with the specified actual arg types. + * (Arg types matter because of ambiguous-operator resolution rules. + * Pass InvalidOid for unused arg of a unary operator.) + * + * The result includes all necessary quoting and schema-prefixing, + * plus the OPERATOR() decoration needed to use a qualified operator name + * in an expression. + */ +static char * +generate_operator_name(Oid operid, Oid arg1, Oid arg2) +{ + StringInfoData buf; + HeapTuple opertup; + Form_pg_operator operform; + char *oprname; + char *nspname; + Operator p_result; + + initStringInfo(&buf); + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); + if (!HeapTupleIsValid(opertup)) + elog(ERROR, "cache lookup failed for operator %u", operid); + operform = (Form_pg_operator) GETSTRUCT(opertup); + oprname = NameStr(operform->oprname); + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct operator given the unqualified op name with the + * specified argtypes. + */ + switch (operform->oprkind) + { + case 'b': + p_result = oper(NULL, list_make1(makeString(oprname)), arg1, arg2, + true, -1); + break; + case 'l': + p_result = left_oper(NULL, list_make1(makeString(oprname)), arg2, + true, -1); + break; + case 'r': + p_result = right_oper(NULL, list_make1(makeString(oprname)), arg1, + true, -1); + break; + default: + elog(ERROR, "unrecognized oprkind: %d", operform->oprkind); + p_result = NULL; /* keep compiler quiet */ + break; + } + + if (p_result != NULL && oprid(p_result) == operid) + nspname = NULL; + else + { + nspname = get_namespace_name(operform->oprnamespace); + appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); + } + + appendStringInfoString(&buf, oprname); + + if (nspname) + appendStringInfoChar(&buf, ')'); + + if (p_result != NULL) + ReleaseSysCache(p_result); + + ReleaseSysCache(opertup); + + return buf.data; +} + +#endif /* (PG_VERSION_NUM >= 100000) */ diff --git a/src/backend/distributed/worker/task_tracker.c b/src/backend/distributed/worker/task_tracker.c index 1afabb371..3b94dfe0f 100644 --- a/src/backend/distributed/worker/task_tracker.c +++ b/src/backend/distributed/worker/task_tracker.c @@ -578,6 +578,13 @@ TaskTrackerShmemInit(void) if (!alreadyInitialized) { +#if (PG_VERSION_NUM >= 100000) + WorkerTasksSharedState->taskHashTrancheId = LWLockNewTrancheId(); + WorkerTasksSharedState->taskHashTrancheName = "Worker Task Hash Tranche"; + LWLockRegisterTranche(WorkerTasksSharedState->taskHashTrancheId, + WorkerTasksSharedState->taskHashTrancheName); +#else + /* initialize lwlock protecting the task tracker hash table */ LWLockTranche *tranche = &WorkerTasksSharedState->taskHashLockTranche; @@ -586,6 +593,8 @@ TaskTrackerShmemInit(void) tranche->array_stride = sizeof(LWLock); tranche->name = "Worker Task Hash Tranche"; LWLockRegisterTranche(WorkerTasksSharedState->taskHashTrancheId, tranche); +#endif + LWLockInitialize(&WorkerTasksSharedState->taskHashLock, WorkerTasksSharedState->taskHashTrancheId); } diff --git a/src/backend/distributed/worker/task_tracker_protocol.c b/src/backend/distributed/worker/task_tracker_protocol.c index 25634f953..9a8d452d6 100644 --- a/src/backend/distributed/worker/task_tracker_protocol.c +++ b/src/backend/distributed/worker/task_tracker_protocol.c @@ -291,11 +291,17 @@ CreateJobSchema(StringInfo schemaName) createSchemaStmt = makeNode(CreateSchemaStmt); createSchemaStmt->schemaname = schemaName->data; - createSchemaStmt->authrole = (Node *) ¤tUserRole; createSchemaStmt->schemaElts = NIL; /* actually create schema with the current user as owner */ +#if (PG_VERSION_NUM >= 100000) + createSchemaStmt->authrole = ¤tUserRole; + CreateSchemaCommand(createSchemaStmt, queryString, -1, -1); +#else + createSchemaStmt->authrole = (Node *) ¤tUserRole; CreateSchemaCommand(createSchemaStmt, queryString); +#endif + CommandCounterIncrement(); /* and reset environment */ diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 30c197242..739d29d84 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -33,6 +33,7 @@ #include "distributed/multi_client_executor.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_server_executor.h" +#include "distributed/multi_utility.h" #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" @@ -44,6 +45,10 @@ #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#if (PG_VERSION_NUM >= 100000) +#include "utils/regproc.h" +#include "utils/varlena.h" +#endif /* Config variable managed via guc.c */ @@ -428,8 +433,8 @@ worker_apply_shard_ddl_command(PG_FUNCTION_ARGS) /* extend names in ddl command and apply extended command */ RelayEventExtendNames(ddlCommandNode, schemaName, shardId); - ProcessUtility(ddlCommandNode, ddlCommand, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); + CitusProcessUtility(ddlCommandNode, ddlCommand, PROCESS_UTILITY_TOPLEVEL, NULL, + None_Receiver, NULL); PG_RETURN_VOID(); } @@ -460,8 +465,8 @@ worker_apply_inter_shard_ddl_command(PG_FUNCTION_ARGS) RelayEventExtendNamesForInterShardCommands(ddlCommandNode, leftShardId, leftShardSchemaName, rightShardId, rightShardSchemaName); - ProcessUtility(ddlCommandNode, ddlCommand, PROCESS_UTILITY_TOPLEVEL, NULL, - None_Receiver, NULL); + CitusProcessUtility(ddlCommandNode, ddlCommand, PROCESS_UTILITY_TOPLEVEL, NULL, + None_Receiver, NULL); PG_RETURN_VOID(); } @@ -496,8 +501,8 @@ worker_apply_sequence_command(PG_FUNCTION_ARGS) } /* run the CREATE SEQUENCE command */ - ProcessUtility(commandNode, commandString, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); + CitusProcessUtility(commandNode, commandString, PROCESS_UTILITY_TOPLEVEL, NULL, + None_Receiver, NULL); CommandCounterIncrement(); createSequenceStatement = (CreateSeqStmt *) commandNode; @@ -851,8 +856,8 @@ FetchRegularTable(const char *nodeName, uint32 nodePort, const char *tableName) StringInfo ddlCommand = (StringInfo) lfirst(ddlCommandCell); Node *ddlCommandNode = ParseTreeNode(ddlCommand->data); - ProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); + CitusProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, + NULL, None_Receiver, NULL); CommandCounterIncrement(); } @@ -870,8 +875,8 @@ FetchRegularTable(const char *nodeName, uint32 nodePort, const char *tableName) queryString = makeStringInfo(); appendStringInfo(queryString, COPY_IN_COMMAND, tableName, localFilePath->data); - ProcessUtility((Node *) localCopyCommand, queryString->data, - PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); + CitusProcessUtility((Node *) localCopyCommand, queryString->data, + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); /* finally delete the temporary file we created */ DeleteFile(localFilePath->data); @@ -945,8 +950,8 @@ FetchForeignTable(const char *nodeName, uint32 nodePort, const char *tableName) StringInfo ddlCommand = (StringInfo) lfirst(ddlCommandCell); Node *ddlCommandNode = ParseTreeNode(ddlCommand->data); - ProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); + CitusProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, + NULL, None_Receiver, NULL); CommandCounterIncrement(); } @@ -1130,6 +1135,22 @@ ExecuteRemoteQuery(const char *nodeName, uint32 nodePort, char *runAsUser, */ Node * ParseTreeNode(const char *ddlCommand) +{ + Node *parseTreeNode = ParseTreeRawStmt(ddlCommand); + +#if (PG_VERSION_NUM >= 100000) + parseTreeNode = ((RawStmt *) parseTreeNode)->stmt; +#endif + + return parseTreeNode; +} + + +/* + * Parses the given DDL command, and returns the tree node for parsed command. + */ +Node * +ParseTreeRawStmt(const char *ddlCommand) { Node *parseTreeNode = NULL; List *parseTreeList = NULL; @@ -1237,8 +1258,8 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) appendStringInfo(queryString, COPY_IN_COMMAND, shardQualifiedName, localFilePath->data); - ProcessUtility((Node *) localCopyCommand, queryString->data, - PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); + CitusProcessUtility((Node *) localCopyCommand, queryString->data, + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); /* finally delete the temporary file we created */ DeleteFile(localFilePath->data); @@ -1299,6 +1320,14 @@ AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName) Form_pg_sequence sequenceData = pg_get_sequencedef(sequenceId); int64 startValue = 0; int64 maxValue = 0; +#if (PG_VERSION_NUM >= 100000) + int64 sequenceMaxValue = sequenceData->seqmax; + int64 sequenceMinValue = sequenceData->seqmin; +#else + int64 sequenceMaxValue = sequenceData->max_value; + int64 sequenceMinValue = sequenceData->min_value; +#endif + /* calculate min/max values that the sequence can generate in this worker */ startValue = (((int64) GetLocalGroupId()) << 48) + 1; @@ -1309,7 +1338,7 @@ AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName) * their correct values. This happens when the sequence has been created * during shard, before the current worker having the metadata. */ - if (sequenceData->min_value != startValue || sequenceData->max_value != maxValue) + if (sequenceMinValue != startValue || sequenceMaxValue != maxValue) { StringInfo startNumericString = makeStringInfo(); StringInfo maxNumericString = makeStringInfo(); @@ -1337,8 +1366,8 @@ AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName) SetDefElemArg(alterSequenceStatement, "restart", startFloatArg); /* since the command is an AlterSeqStmt, a dummy command string works fine */ - ProcessUtility((Node *) alterSequenceStatement, dummyString, - PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); + CitusProcessUtility((Node *) alterSequenceStatement, dummyString, + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); } } @@ -1368,6 +1397,11 @@ SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg) } } +#if (PG_VERSION_NUM >= 100000) + defElem = makeDefElem((char *) name, arg, -1); +#else defElem = makeDefElem((char *) name, arg); +#endif + statement->options = lappend(statement->options, defElem); } diff --git a/src/backend/distributed/worker/worker_merge_protocol.c b/src/backend/distributed/worker/worker_merge_protocol.c index f1714e709..394e12236 100644 --- a/src/backend/distributed/worker/worker_merge_protocol.c +++ b/src/backend/distributed/worker/worker_merge_protocol.c @@ -329,7 +329,6 @@ RemoveJobSchema(StringInfo schemaName) if (OidIsValid(schemaId)) { ObjectAddress schemaObject = { 0, 0, 0 }; - bool showNotices = false; bool permissionsOK = pg_namespace_ownercheck(schemaId, GetUserId()); if (!permissionsOK) @@ -347,7 +346,16 @@ RemoveJobSchema(StringInfo schemaName) * can suppress notice messages that are typically displayed during * cascading deletes. */ - deleteWhatDependsOn(&schemaObject, showNotices); +#if (PG_VERSION_NUM >= 100000) + performDeletion(&schemaObject, DROP_CASCADE, + PERFORM_DELETION_INTERNAL | + PERFORM_DELETION_QUIETLY | + PERFORM_DELETION_SKIP_ORIGINAL | + PERFORM_DELETION_SKIP_EXTENSIONS); +#else + deleteWhatDependsOn(&schemaObject, false); +#endif + CommandCounterIncrement(); /* drop the empty schema */ @@ -386,7 +394,12 @@ CreateTaskTable(StringInfo schemaName, StringInfo relationName, createStatement = CreateStatement(relation, columnDefinitionList); +#if (PG_VERSION_NUM >= 100000) + relationObject = DefineRelation(createStatement, RELKIND_RELATION, InvalidOid, NULL, + NULL); +#else relationObject = DefineRelation(createStatement, RELKIND_RELATION, InvalidOid, NULL); +#endif relationId = relationObject.objectId; Assert(relationId != InvalidOid); @@ -510,11 +523,27 @@ CopyTaskFilesFromDirectory(StringInfo schemaName, StringInfo relationName, copyStatement = CopyStatement(relation, fullFilename->data); if (BinaryWorkerCopyFormat) { +#if (PG_VERSION_NUM >= 100000) + DefElem *copyOption = makeDefElem("format", (Node *) makeString("binary"), + -1); +#else DefElem *copyOption = makeDefElem("format", (Node *) makeString("binary")); +#endif copyStatement->options = list_make1(copyOption); } +#if (PG_VERSION_NUM >= 100000) + { + ParseState *pstate = make_parsestate(NULL); + pstate->p_sourcetext = queryString; + + DoCopy(pstate, copyStatement, -1, -1, &copiedRowCount); + + free_parsestate(pstate); + } +#else DoCopy(copyStatement, queryString, &copiedRowCount); +#endif copiedRowTotal += copiedRowCount; CommandCounterIncrement(); } diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index b15915f78..fe859d874 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -16,6 +16,7 @@ #include "postgres.h" #include "funcapi.h" +#include "pgstat.h" #include #include @@ -742,7 +743,12 @@ FileOutputStreamFlush(FileOutputStream file) int written = 0; errno = 0; +#if (PG_VERSION_NUM >= 100000) + written = FileWrite(file.fileDescriptor, fileBuffer->data, fileBuffer->len, + PG_WAIT_IO); +#else written = FileWrite(file.fileDescriptor, fileBuffer->data, fileBuffer->len); +#endif if (written != fileBuffer->len) { ereport(ERROR, (errcode_for_file_access(), diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index e79224326..213dec2fa 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -14,6 +14,9 @@ #include "postgres.h" /* IWYU pragma: keep */ #include "c.h" +#if (PG_VERSION_NUM >= 100000) +#include "catalog/pg_sequence.h" +#endif #include "commands/sequence.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index 9f08abc4a..959fcf8c3 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -14,8 +14,10 @@ #ifndef MASTER_METADATA_UTILITY_H #define MASTER_METADATA_UTILITY_H +#include "access/heapam.h" #include "access/htup.h" #include "access/tupdesc.h" +#include "catalog/indexing.h" #include "distributed/citus_nodes.h" #include "distributed/relay_utility.h" #include "utils/acl.h" @@ -29,6 +31,27 @@ #define PG_RELATION_SIZE_FUNCTION "pg_relation_size(%s)" #define PG_TOTAL_RELATION_SIZE_FUNCTION "pg_total_relation_size(%s)" +#if (PG_VERSION_NUM < 100000) +static inline void +CatalogTupleUpdate(Relation heapRel, ItemPointer otid, HeapTuple tup) +{ + simple_heap_update(heapRel, otid, tup); + CatalogUpdateIndexes(heapRel, tup); +} + + +static inline Oid +CatalogTupleInsert(Relation heapRel, HeapTuple tup) +{ + Oid oid = simple_heap_insert(heapRel, tup); + CatalogUpdateIndexes(heapRel, tup); + + return oid; +} + + +#endif + /* In-memory representation of a typed tuple in pg_dist_shard. */ typedef struct ShardInterval { diff --git a/src/include/distributed/multi_join_order.h b/src/include/distributed/multi_join_order.h index a2466e57a..c063700b5 100644 --- a/src/include/distributed/multi_join_order.h +++ b/src/include/distributed/multi_join_order.h @@ -90,7 +90,7 @@ extern OpExpr * DualPartitionJoinClause(List *applicableJoinClauses); extern Var * LeftColumn(OpExpr *joinClause); extern Var * RightColumn(OpExpr *joinClause); extern Var * PartitionColumn(Oid relationId, uint32 rangeTableId); -extern Var * PartitionKey(Oid relationId); +extern Var * DistPartitionKey(Oid relationId); extern char PartitionMethod(Oid relationId); extern char TableReplicationModel(Oid relationId); diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 44edac1f4..4593da4da 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -55,8 +55,14 @@ typedef enum CitusRTEKind CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ CITUS_RTE_JOIN = RTE_JOIN, /* join */ CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ +#if (PG_VERSION_NUM >= 100000) + CITUS_RTE_TABLEFUNC = RTE_TABLEFUNC, /* TableFunc(.., column list) */ +#endif CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ +#if (PG_VERSION_NUM >= 100000) + CITUS_RTE_NAMEDTUPLESTORE = RTE_NAMEDTUPLESTORE, /* tuplestore, e.g. for triggers */ +#endif CITUS_RTE_SHARD, CITUS_RTE_REMOTE_QUERY } CitusRTEKind; diff --git a/src/include/distributed/multi_utility.h b/src/include/distributed/multi_utility.h index b81262c4d..7e1ec9fc2 100644 --- a/src/include/distributed/multi_utility.h +++ b/src/include/distributed/multi_utility.h @@ -29,9 +29,20 @@ typedef struct DDLJob List *taskList; /* worker DDL tasks to execute */ } DDLJob; -extern void multi_ProcessUtility(Node *parsetree, const char *queryString, +#if (PG_VERSION_NUM < 100000) +struct QueryEnvironment; /* forward-declare to appease compiler */ +#endif + +extern void multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, - DestReceiver *dest, char *completionTag); + struct QueryEnvironment *queryEnv, DestReceiver *dest, + char *completionTag); +extern void multi_ProcessUtility9x(Node *parsetree, const char *queryString, + ProcessUtilityContext context, ParamListInfo params, + DestReceiver *dest, char *completionTag); +extern void CitusProcessUtility(Node *node, const char *queryString, + ProcessUtilityContext context, ParamListInfo params, + DestReceiver *dest, char *completionTag); extern List * PlanGrantStmt(GrantStmt *grantStmt); extern void ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, Var *distributionColumn, uint32 colocationId); diff --git a/src/include/distributed/task_tracker.h b/src/include/distributed/task_tracker.h index 1b76d9547..20aa0a9cf 100644 --- a/src/include/distributed/task_tracker.h +++ b/src/include/distributed/task_tracker.h @@ -99,7 +99,11 @@ typedef struct WorkerTasksSharedStateData /* Lock protecting workerNodesHash */ int taskHashTrancheId; +#if (PG_VERSION_NUM >= 100000) + char *taskHashTrancheName; +#else LWLockTranche taskHashLockTranche; +#endif LWLock taskHashLock; } WorkerTasksSharedStateData; diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index bf0760159..6fb8efa94 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -135,6 +135,7 @@ extern Datum CompareCall2(FmgrInfo *funcInfo, Datum leftArgument, Datum rightArg /* Function declaration for parsing tree node */ extern Node * ParseTreeNode(const char *ddlCommand); +extern Node * ParseTreeRawStmt(const char *ddlCommand); /* Function declarations for applying distributed execution primitives */ extern Datum worker_fetch_partition_file(PG_FUNCTION_ARGS); diff --git a/src/test/regress/expected/multi_alter_table_add_constraints.out b/src/test/regress/expected/multi_alter_table_add_constraints.out index 93ce385ec..c9688c4d4 100644 --- a/src/test/regress/expected/multi_alter_table_add_constraints.out +++ b/src/test/regress/expected/multi_alter_table_add_constraints.out @@ -456,17 +456,17 @@ INSERT INTO products VALUES(1,'product_1', 10, 8); ERROR: single-shard DML commands must not appear in transaction blocks which contain multi-shard data modifications ROLLBACK; -- There should be no constraint on master and worker(s) -\d products - Table "public.products" - Column | Type | Modifiers -------------------+---------+----------- - product_no | integer | - name | text | - price | numeric | - discounted_price | numeric | +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='products'::regclass; + Constraint | Definition +------------+------------ +(0 rows) \c - - - :worker_1_port -\d products_1450199 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.products_1450202'::regclass; + Constraint | Definition +------------+------------ +(0 rows) + \c - - - :master_port -- Tests to check the effect of rollback BEGIN; @@ -478,16 +478,16 @@ ALTER TABLE products ADD CONSTRAINT check_price CHECK(price > discounted_price); ALTER TABLE products ADD CONSTRAINT p_key_product PRIMARY KEY(product_no); ROLLBACK; -- There should be no constraint on master and worker(s) -\d products - Table "public.products" - Column | Type | Modifiers -------------------+---------+----------- - product_no | integer | - name | text | - price | numeric | - discounted_price | numeric | +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='products'::regclass; + Constraint | Definition +------------+------------ +(0 rows) \c - - - :worker_1_port -\d products_1450199 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.products_1450202'::regclass; + Constraint | Definition +------------+------------ +(0 rows) + \c - - - :master_port DROP TABLE products; diff --git a/src/test/regress/expected/multi_colocation_utils.out b/src/test/regress/expected/multi_colocation_utils.out index 4507e56fa..efe140527 100644 --- a/src/test/regress/expected/multi_colocation_utils.out +++ b/src/test/regress/expected/multi_colocation_utils.out @@ -617,18 +617,18 @@ ERROR: cannot colocate tables table1_groupe and table_bigint DETAIL: Distribution column types don't match for table1_groupe and table_bigint. -- check worker table schemas \c - - - :worker_1_port -\d table3_groupE_1300050 -Table "public.table3_groupe_1300050" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.table3_groupE_1300050'::regclass; Column | Type | Modifiers --------------+---------+----------- dummy_column | text | id | integer | +(2 rows) -\d schema_collocation.table4_groupE_1300052 -Table "schema_collocation.table4_groupe_1300052" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='schema_collocation.table4_groupE_1300052'::regclass; Column | Type | Modifiers --------+---------+----------- id | integer | +(1 row) \c - - - :master_port CREATE TABLE table1_groupF ( id int ); diff --git a/src/test/regress/expected/multi_create_table_constraints.out b/src/test/regress/expected/multi_create_table_constraints.out index b273e3529..6781d26d6 100644 --- a/src/test/regress/expected/multi_create_table_constraints.out +++ b/src/test/regress/expected/multi_create_table_constraints.out @@ -401,80 +401,19 @@ SELECT master_create_worker_shards('check_example', '2', '2'); (1 row) \c - - - :worker_1_port -\d check_example* - Table "public.check_example_365040" - Column | Type | Modifiers ------------------+---------+----------- - partition_col | integer | - other_col | integer | - other_other_col | integer | -Indexes: - "check_example_partition_col_key_365040" UNIQUE CONSTRAINT, btree (partition_col) -Check constraints: - "check_example_other_col_check" CHECK (other_col >= 100) - "check_example_other_other_col_check" CHECK (abs(other_other_col) >= 100) - - Table "public.check_example_365041" - Column | Type | Modifiers ------------------+---------+----------- - partition_col | integer | - other_col | integer | - other_other_col | integer | -Indexes: - "check_example_partition_col_key_365041" UNIQUE CONSTRAINT, btree (partition_col) -Check constraints: - "check_example_other_col_check" CHECK (other_col >= 100) - "check_example_other_other_col_check" CHECK (abs(other_other_col) >= 100) - +\d check_example_partition_col_key_365040 Index "public.check_example_partition_col_key_365040" Column | Type | Definition ---------------+---------+--------------- partition_col | integer | partition_col unique, btree, for table "public.check_example_365040" -Index "public.check_example_partition_col_key_365041" - Column | Type | Definition ----------------+---------+--------------- - partition_col | integer | partition_col -unique, btree, for table "public.check_example_365041" - -\c - - - :worker_2_port -\d check_example* - Table "public.check_example_365040" - Column | Type | Modifiers ------------------+---------+----------- - partition_col | integer | - other_col | integer | - other_other_col | integer | -Indexes: - "check_example_partition_col_key_365040" UNIQUE CONSTRAINT, btree (partition_col) -Check constraints: - "check_example_other_col_check" CHECK (other_col >= 100) - "check_example_other_other_col_check" CHECK (abs(other_other_col) >= 100) - - Table "public.check_example_365041" - Column | Type | Modifiers ------------------+---------+----------- - partition_col | integer | - other_col | integer | - other_other_col | integer | -Indexes: - "check_example_partition_col_key_365041" UNIQUE CONSTRAINT, btree (partition_col) -Check constraints: - "check_example_other_col_check" CHECK (other_col >= 100) - "check_example_other_other_col_check" CHECK (abs(other_other_col) >= 100) - -Index "public.check_example_partition_col_key_365040" - Column | Type | Definition ----------------+---------+--------------- - partition_col | integer | partition_col -unique, btree, for table "public.check_example_365040" - -Index "public.check_example_partition_col_key_365041" - Column | Type | Definition ----------------+---------+--------------- - partition_col | integer | partition_col -unique, btree, for table "public.check_example_365041" +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.check_example_365040'::regclass; + Constraint | Definition +-------------------------------------+------------------------------------- + check_example_other_col_check | CHECK (other_col >= 100) + check_example_other_other_col_check | CHECK (abs(other_other_col) >= 100) +(2 rows) \c - - - :master_port -- drop unnecessary tables @@ -501,15 +440,11 @@ SELECT create_distributed_table('raw_table_2', 'user_id'); (1 row) -- see that the constraint exists -\d raw_table_2 - Table "public.raw_table_2" - Column | Type | Modifiers ----------+---------+----------- - user_id | integer | -Indexes: - "raw_table_2_user_id_key" UNIQUE CONSTRAINT, btree (user_id) -Foreign-key constraints: - "raw_table_2_user_id_fkey" FOREIGN KEY (user_id) REFERENCES raw_table_1(user_id) +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='raw_table_2'::regclass; + Constraint | Definition +--------------------------+------------------------------------------------------- + raw_table_2_user_id_fkey | FOREIGN KEY (user_id) REFERENCES raw_table_1(user_id) +(1 row) -- should be prevented by the foreign key DROP TABLE raw_table_1; @@ -520,13 +455,10 @@ HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP TABLE raw_table_1 CASCADE; NOTICE: drop cascades to constraint raw_table_2_user_id_fkey on table raw_table_2 -- see that the constraint also dropped -\d raw_table_2 - Table "public.raw_table_2" - Column | Type | Modifiers ----------+---------+----------- - user_id | integer | -Indexes: - "raw_table_2_user_id_key" UNIQUE CONSTRAINT, btree (user_id) +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='raw_table_2'::regclass; + Constraint | Definition +------------+------------ +(0 rows) -- drop the table as well DROP TABLE raw_table_2; diff --git a/src/test/regress/expected/multi_create_table_new_features.out b/src/test/regress/expected/multi_create_table_new_features.out new file mode 100644 index 000000000..1029ae8f9 --- /dev/null +++ b/src/test/regress/expected/multi_create_table_new_features.out @@ -0,0 +1,28 @@ +-- +-- MULTI_CREATE_TABLE_NEW_FEATURES +-- +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 10 +(1 row) + +-- Verify that the GENERATED ... AS IDENTITY feature in PostgreSQL 10 +-- is forbidden in distributed tables. +CREATE TABLE table_identity_col ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + payload text ); +SELECT master_create_distributed_table('table_identity_col', 'id', 'append'); +ERROR: cannot distribute relation: table_identity_col +DETAIL: Distributed relations must not use GENERATED ... AS IDENTITY. +SELECT create_distributed_table('table_identity_col', 'id'); +ERROR: cannot distribute relation: table_identity_col +DETAIL: Distributed relations must not use GENERATED ... AS IDENTITY. +SELECT create_distributed_table('table_identity_col', 'text'); +ERROR: cannot distribute relation: table_identity_col +DETAIL: Distributed relations must not use GENERATED ... AS IDENTITY. +SELECT create_reference_table('table_identity_col'); +ERROR: cannot distribute relation: table_identity_col +DETAIL: Distributed relations must not use GENERATED ... AS IDENTITY. diff --git a/src/test/regress/expected/multi_create_table_new_features_0.out b/src/test/regress/expected/multi_create_table_new_features_0.out new file mode 100644 index 000000000..5bec9da7a --- /dev/null +++ b/src/test/regress/expected/multi_create_table_new_features_0.out @@ -0,0 +1,35 @@ +-- +-- MULTI_CREATE_TABLE_NEW_FEATURES +-- +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 9 +(1 row) + +-- Verify that the GENERATED ... AS IDENTITY feature in PostgreSQL 10 +-- is forbidden in distributed tables. +CREATE TABLE table_identity_col ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + payload text ); +ERROR: syntax error at or near "GENERATED" +LINE 2: id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + ^ +SELECT master_create_distributed_table('table_identity_col', 'id', 'append'); +ERROR: relation "table_identity_col" does not exist +LINE 1: SELECT master_create_distributed_table('table_identity_col',... + ^ +SELECT create_distributed_table('table_identity_col', 'id'); +ERROR: relation "table_identity_col" does not exist +LINE 1: SELECT create_distributed_table('table_identity_col', 'id'); + ^ +SELECT create_distributed_table('table_identity_col', 'text'); +ERROR: relation "table_identity_col" does not exist +LINE 1: SELECT create_distributed_table('table_identity_col', 'text'... + ^ +SELECT create_reference_table('table_identity_col'); +ERROR: relation "table_identity_col" does not exist +LINE 1: SELECT create_reference_table('table_identity_col'); + ^ diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 126319156..6f09a66e4 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -6,7 +6,7 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 570000; SELECT substring(version(), '\d+(?:\.\d+)?') AS major_version; major_version --------------- - 9.6 + 10 (1 row) \a\t @@ -903,6 +903,8 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) SET parallel_setup_cost=0; SET parallel_tuple_cost=0; SET min_parallel_relation_size=0; +ERROR: unrecognized configuration parameter "min_parallel_relation_size" +SET min_parallel_table_scan_size=0; SET max_parallel_workers_per_gather=4; -- ensure local plans display correctly CREATE TABLE lineitem_clone (LIKE lineitem); diff --git a/src/test/regress/expected/multi_explain_0.out b/src/test/regress/expected/multi_explain_0.out index b0f5db6b5..ef25cf790 100644 --- a/src/test/regress/expected/multi_explain_0.out +++ b/src/test/regress/expected/multi_explain_0.out @@ -6,7 +6,7 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 570000; SELECT substring(version(), '\d+(?:\.\d+)?') AS major_version; major_version --------------- - 9.5 + 9.6 (1 row) \a\t @@ -42,7 +42,7 @@ EXPLAIN (COSTS FALSE, FORMAT TEXT) SELECT l_quantity, count(*) count_quantity FROM lineitem GROUP BY l_quantity ORDER BY count_quantity, l_quantity; Sort - Sort Key: COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity + Sort Key: COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity -> HashAggregate Group Key: remote_scan.l_quantity -> Custom Scan (Citus Real-Time) @@ -61,18 +61,22 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) { "Plan": { "Node Type": "Sort", - "Sort Key": ["COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)", "remote_scan.l_quantity"], + "Parallel Aware": false, + "Sort Key": ["COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)", "remote_scan.l_quantity"], "Plans": [ { "Node Type": "Aggregate", "Strategy": "Hashed", + "Partial Mode": "Simple", "Parent Relationship": "Outer", + "Parallel Aware": false, "Group Key": ["remote_scan.l_quantity"], "Plans": [ { "Node Type": "Custom Scan", "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Real-Time", + "Parallel Aware": false, "Distributed Query": { "Job": { "Task Count": 8, @@ -86,11 +90,14 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plan": { "Node Type": "Aggregate", "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parallel Aware": false, "Group Key": ["l_quantity"], "Plans": [ { "Node Type": "Seq Scan", "Parent Relationship": "Outer", + "Parallel Aware": false, "Relation Name": "lineitem_290001", "Alias": "lineitem" } @@ -124,15 +131,18 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Sort + false - COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint) + COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint) remote_scan.l_quantity Aggregate Hashed + Simple Outer + false remote_scan.l_quantity @@ -141,6 +151,7 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan Outer Citus Real-Time + false 8 @@ -154,6 +165,8 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Aggregate Hashed + Simple + false l_quantity @@ -161,6 +174,7 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Seq Scan Outer + false lineitem_290001 lineitem @@ -191,19 +205,23 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) GROUP BY l_quantity ORDER BY count_quantity, l_quantity; - Plan: Node Type: "Sort" + Parallel Aware: false Sort Key: - - "COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)" + - "COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)" - "remote_scan.l_quantity" Plans: - Node Type: "Aggregate" Strategy: "Hashed" + Partial Mode: "Simple" Parent Relationship: "Outer" + Parallel Aware: false Group Key: - "remote_scan.l_quantity" Plans: - Node Type: "Custom Scan" Parent Relationship: "Outer" Custom Plan Provider: "Citus Real-Time" + Parallel Aware: false Distributed Query: Job: Task Count: 8 @@ -214,11 +232,14 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - Plan: Node Type: "Aggregate" Strategy: "Hashed" + Partial Mode: "Simple" + Parallel Aware: false Group Key: - "l_quantity" Plans: - Node Type: "Seq Scan" Parent Relationship: "Outer" + Parallel Aware: false Relation Name: "lineitem_290001" Alias: "lineitem" @@ -227,7 +248,7 @@ EXPLAIN (COSTS FALSE, FORMAT TEXT) SELECT l_quantity, count(*) count_quantity FROM lineitem GROUP BY l_quantity ORDER BY count_quantity, l_quantity; Sort - Sort Key: COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity + Sort Key: COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity -> HashAggregate Group Key: remote_scan.l_quantity -> Custom Scan (Citus Real-Time) @@ -242,7 +263,7 @@ Sort EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem; Aggregate - Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / sum(remote_scan."?column?_2"))) + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) -> Custom Scan (Citus Real-Time) Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" Task Count: 8 @@ -344,7 +365,7 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem HAVING sum(l_quantity) > 100; Aggregate - Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / sum(remote_scan."?column?_2"))) + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) Filter: (sum(remote_scan.worker_column_4) > '100'::numeric) -> Custom Scan (Citus Real-Time) Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2", remote_scan.worker_column_4 @@ -410,11 +431,15 @@ Aggregate Node: host=localhost port=57637 dbname=regression -> Aggregate -> GroupAggregate - Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + Group Key: (((NULL::user_composite_type)).tenant_id), (((NULL::user_composite_type)).user_id) -> Sort - Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) - -> Result - One-Time Filter: false + Sort Key: (((NULL::user_composite_type)).tenant_id), (((NULL::user_composite_type)).user_id) + -> Nested Loop + Join Filter: ((NULL::user_composite_type) = events.composite_id) + -> Result + One-Time Filter: false + -> Seq Scan on events_1400027 events + Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[])) -- Union and left join subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -485,29 +510,40 @@ HashAggregate Tasks Shown: One of 4 -> Task Node: host=localhost port=57637 dbname=regression - -> HashAggregate - Group Key: COALESCE(subquery_2.hasdone, 'Has not done paying'::text) - -> GroupAggregate - Group Key: ((composite_id).tenant_id), ((composite_id).user_id), subquery_2.hasdone - -> Sort - Sort Key: ((composite_id).tenant_id), ((composite_id).user_id), subquery_2.hasdone - -> Hash Left Join - Hash Cond: (composite_id = subquery_2.composite_id) - -> Unique - -> Sort - Sort Key: ((composite_id).tenant_id), ((composite_id).user_id), composite_id, ('action=>1'::text), event_time - -> Append - -> Result - One-Time Filter: false - -> Result - One-Time Filter: false - -> Hash - -> Subquery Scan on subquery_2 + -> GroupAggregate + Group Key: subquery_top.hasdone + -> Sort + Sort Key: subquery_top.hasdone + -> Subquery Scan on subquery_top + -> GroupAggregate + Group Key: (((NULL::user_composite_type)).tenant_id), (((NULL::user_composite_type)).user_id), subquery_2.hasdone + -> Sort + Sort Key: (((NULL::user_composite_type)).tenant_id), (((NULL::user_composite_type)).user_id), subquery_2.hasdone + -> Hash Left Join + Hash Cond: ((NULL::user_composite_type) = subquery_2.composite_id) -> Unique -> Sort - Sort Key: ((events.composite_id).tenant_id), ((events.composite_id).user_id) - -> Seq Scan on events_1400027 events - Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) + Sort Key: (((NULL::user_composite_type)).tenant_id), (((NULL::user_composite_type)).user_id), (NULL::user_composite_type), ('action=>1'::text), events.event_time + -> Append + -> Nested Loop + Join Filter: ((NULL::user_composite_type) = events.composite_id) + -> Result + One-Time Filter: false + -> Seq Scan on events_1400027 events + Filter: ((event_type)::text = 'click'::text) + -> Nested Loop + Join Filter: ((NULL::user_composite_type) = events_1.composite_id) + -> Result + One-Time Filter: false + -> Seq Scan on events_1400027 events_1 + Filter: ((event_type)::text = 'submit'::text) + -> Hash + -> Subquery Scan on subquery_2 + -> Unique + -> Sort + Sort Key: ((events_2.composite_id).tenant_id), ((events_2.composite_id).user_id) + -> Seq Scan on events_1400027 events_2 + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) -- Union, left join and having subquery pushdown EXPLAIN (COSTS OFF) SELECT @@ -643,22 +679,23 @@ Limit Node: host=localhost port=57637 dbname=regression -> Limit -> Sort - Sort Key: (max(lastseen)) DESC + Sort Key: (max(users.lastseen)) DESC -> GroupAggregate - Group Key: ((composite_id).tenant_id), ((composite_id).user_id) + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) -> Sort - Sort Key: ((composite_id).tenant_id), ((composite_id).user_id) + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) -> Nested Loop Left Join -> Limit -> Sort - Sort Key: lastseen DESC - -> Result - One-Time Filter: false + Sort Key: users.lastseen DESC + -> Subquery Scan on users + -> Result + One-Time Filter: false -> Limit -> Sort Sort Key: events.event_time DESC -> Seq Scan on events_1400027 events - Filter: (composite_id = composite_id) + Filter: (composite_id = users.composite_id) -- Test all tasks output SET citus.explain_all_tasks TO on; EXPLAIN (COSTS FALSE) @@ -736,11 +773,14 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plan": { "Node Type": "Aggregate", "Strategy": "Plain", + "Partial Mode": "Simple", + "Parallel Aware": false, "Plans": [ { "Node Type": "Custom Scan", "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Task-Tracker", + "Parallel Aware": false, "Distributed Query": { "Job": { "Task Count": 1, @@ -782,11 +822,14 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Aggregate Plain + Simple + false Custom Scan Outer Citus Task-Tracker + false 1 @@ -839,10 +882,13 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - Plan: Node Type: "Aggregate" Strategy: "Plain" + Partial Mode: "Simple" + Parallel Aware: false Plans: - Node Type: "Custom Scan" Parent Relationship: "Outer" Custom Plan Provider: "Citus Task-Tracker" + Parallel Aware: false Distributed Query: Job: Task Count: 1 @@ -855,18 +901,19 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Merge Task Count: 1 -- test parallel aggregates SET parallel_setup_cost=0; -ERROR: unrecognized configuration parameter "parallel_setup_cost" SET parallel_tuple_cost=0; -ERROR: unrecognized configuration parameter "parallel_tuple_cost" SET min_parallel_relation_size=0; -ERROR: unrecognized configuration parameter "min_parallel_relation_size" +SET min_parallel_table_scan_size=0; +ERROR: unrecognized configuration parameter "min_parallel_table_scan_size" SET max_parallel_workers_per_gather=4; -ERROR: unrecognized configuration parameter "max_parallel_workers_per_gather" -- ensure local plans display correctly CREATE TABLE lineitem_clone (LIKE lineitem); EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem_clone; -Aggregate - -> Seq Scan on lineitem_clone +Finalize Aggregate + -> Gather + Workers Planned: 3 + -> Partial Aggregate + -> Parallel Seq Scan on lineitem_clone -- ensure distributed plans don't break EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem; Aggregate diff --git a/src/test/regress/expected/multi_explain_1.out b/src/test/regress/expected/multi_explain_1.out new file mode 100644 index 000000000..1dad01db7 --- /dev/null +++ b/src/test/regress/expected/multi_explain_1.out @@ -0,0 +1,1003 @@ +-- +-- MULTI_EXPLAIN +-- +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 570000; +-- print major version to make version-specific tests clear +SELECT substring(version(), '\d+(?:\.\d+)?') AS major_version; + major_version +--------------- + 9.5 +(1 row) + +\a\t +SET citus.task_executor_type TO 'real-time'; +SET citus.explain_distributed_queries TO on; +-- Function that parses explain output as JSON +CREATE FUNCTION explain_json(query text) +RETURNS jsonb +AS $BODY$ +DECLARE + result jsonb; +BEGIN + EXECUTE format('EXPLAIN (FORMAT JSON) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +-- Function that parses explain output as XML +CREATE FUNCTION explain_xml(query text) +RETURNS xml +AS $BODY$ +DECLARE + result xml; +BEGIN + EXECUTE format('EXPLAIN (FORMAT XML) %s', query) INTO result; + RETURN result; +END; +$BODY$ LANGUAGE plpgsql; +-- VACUMM related tables to ensure test outputs are stable +VACUUM ANALYZE lineitem; +VACUUM ANALYZE orders; +-- Test Text format +EXPLAIN (COSTS FALSE, FORMAT TEXT) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Sort Key: COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity + -> HashAggregate + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Real-Time) + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> HashAggregate + Group Key: l_quantity + -> Seq Scan on lineitem_290001 lineitem +-- Test JSON format +EXPLAIN (COSTS FALSE, FORMAT JSON) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +[ + { + "Plan": { + "Node Type": "Sort", + "Sort Key": ["COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)", "remote_scan.l_quantity"], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Parent Relationship": "Outer", + "Group Key": ["remote_scan.l_quantity"], + "Plans": [ + { + "Node Type": "Custom Scan", + "Parent Relationship": "Outer", + "Custom Plan Provider": "Citus Real-Time", + "Distributed Query": { + "Job": { + "Task Count": 8, + "Tasks Shown": "One of 8", + "Tasks": [ + { + "Node": "host=localhost port=57637 dbname=regression", + "Remote Plan": [ + [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Group Key": ["l_quantity"], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Relation Name": "lineitem_290001", + "Alias": "lineitem" + } + ] + } + } + ] + + ] + } + ] + } + } + } + ] + } + ] + } + } +] +-- Validate JSON format +SELECT true AS valid FROM explain_json($$ + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity$$); +t +-- Test XML format +EXPLAIN (COSTS FALSE, FORMAT XML) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; + + + + Sort + + COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint) + remote_scan.l_quantity + + + + Aggregate + Hashed + Outer + + remote_scan.l_quantity + + + + Custom Scan + Outer + Citus Real-Time + + + 8 + One of 8 + + + host=localhost port=57637 dbname=regression + + + + + Aggregate + Hashed + + l_quantity + + + + Seq Scan + Outer + lineitem_290001 + lineitem + + + + + + + + + + + + + + + + + +-- Validate XML format +SELECT true AS valid FROM explain_xml($$ + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity$$); +t +-- Test YAML format +EXPLAIN (COSTS FALSE, FORMAT YAML) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +- Plan: + Node Type: "Sort" + Sort Key: + - "COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint)" + - "remote_scan.l_quantity" + Plans: + - Node Type: "Aggregate" + Strategy: "Hashed" + Parent Relationship: "Outer" + Group Key: + - "remote_scan.l_quantity" + Plans: + - Node Type: "Custom Scan" + Parent Relationship: "Outer" + Custom Plan Provider: "Citus Real-Time" + Distributed Query: + Job: + Task Count: 8 + Tasks Shown: "One of 8" + Tasks: + - Node: "host=localhost port=57637 dbname=regression" + Remote Plan: + - Plan: + Node Type: "Aggregate" + Strategy: "Hashed" + Group Key: + - "l_quantity" + Plans: + - Node Type: "Seq Scan" + Parent Relationship: "Outer" + Relation Name: "lineitem_290001" + Alias: "lineitem" + +-- Test Text format +EXPLAIN (COSTS FALSE, FORMAT TEXT) + SELECT l_quantity, count(*) count_quantity FROM lineitem + GROUP BY l_quantity ORDER BY count_quantity, l_quantity; +Sort + Sort Key: COALESCE((sum((COALESCE((sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity + -> HashAggregate + Group Key: remote_scan.l_quantity + -> Custom Scan (Citus Real-Time) + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> HashAggregate + Group Key: l_quantity + -> Seq Scan on lineitem_290001 lineitem +-- Test verbose +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem; +Aggregate + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / sum(remote_scan."?column?_2"))) + -> Custom Scan (Citus Real-Time) + Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + Output: sum(l_quantity), sum(l_quantity), count(l_quantity) + -> Seq Scan on public.lineitem_290001 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test join +EXPLAIN (COSTS FALSE) + SELECT * FROM lineitem + JOIN orders ON l_orderkey = o_orderkey AND l_quantity < 5.0 + ORDER BY l_quantity LIMIT 10; +Limit + -> Sort + Sort Key: remote_scan.l_quantity + -> Custom Scan (Citus Real-Time) + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Limit + -> Sort + Sort Key: lineitem.l_quantity + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Scan using orders_pkey_290008 on orders_290008 orders + -> Sort + Sort Key: lineitem.l_orderkey + -> Seq Scan on lineitem_290001 lineitem + Filter: (l_quantity < 5.0) +-- Test insert +EXPLAIN (COSTS FALSE) + INSERT INTO lineitem VALUES(1,0); +Custom Scan (Citus Router) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57638 dbname=regression + -> Insert on lineitem_290000 + -> Result +-- Test update +EXPLAIN (COSTS FALSE) + UPDATE lineitem + SET l_suppkey = 12 + WHERE l_orderkey = 1 AND l_partkey = 0; +Custom Scan (Citus Router) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57638 dbname=regression + -> Update on lineitem_290000 + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 + Index Cond: (l_orderkey = 1) + Filter: (l_partkey = 0) +-- Test delete +EXPLAIN (COSTS FALSE) + DELETE FROM lineitem + WHERE l_orderkey = 1 AND l_partkey = 0; +Custom Scan (Citus Router) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57638 dbname=regression + -> Delete on lineitem_290000 + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 + Index Cond: (l_orderkey = 1) + Filter: (l_partkey = 0) +-- Test single-shard SELECT +EXPLAIN (COSTS FALSE) + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; +Custom Scan (Citus Router) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + Index Cond: (l_orderkey = 5) +SELECT true AS valid FROM explain_xml($$ + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); +t +SELECT true AS valid FROM explain_json($$ + SELECT l_quantity FROM lineitem WHERE l_orderkey = 5$$); +t +-- Test CREATE TABLE ... AS +EXPLAIN (COSTS FALSE) + CREATE TABLE explain_result AS + SELECT * FROM lineitem; +Custom Scan (Citus Real-Time) + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Seq Scan on lineitem_290001 lineitem +-- Test having +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem + HAVING sum(l_quantity) > 100; +Aggregate + Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / sum(remote_scan."?column?_2"))) + Filter: (sum(remote_scan.worker_column_4) > '100'::numeric) + -> Custom Scan (Citus Real-Time) + Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2", remote_scan.worker_column_4 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + Output: sum(l_quantity), sum(l_quantity), count(l_quantity), sum(l_quantity) + -> Seq Scan on public.lineitem_290001 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Test having without aggregate +EXPLAIN (COSTS FALSE, VERBOSE TRUE) + SELECT l_quantity FROM lineitem + GROUP BY l_quantity + HAVING l_quantity > (100 * random()); +HashAggregate + Output: remote_scan.l_quantity + Group Key: remote_scan.l_quantity + Filter: ((remote_scan.worker_column_2)::double precision > ('100'::double precision * random())) + -> Custom Scan (Citus Real-Time) + Output: remote_scan.l_quantity, remote_scan.worker_column_2 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> HashAggregate + Output: l_quantity, l_quantity + Group Key: lineitem.l_quantity + -> Seq Scan on public.lineitem_290001 lineitem + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +-- Subquery pushdown tests with explain +EXPLAIN (COSTS OFF) +SELECT + avg(array_length(events, 1)) AS event_average +FROM + (SELECT + tenant_id, + user_id, + array_agg(event_type ORDER BY event_time) AS events + FROM + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + event_type, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type IN ('click', 'submit', 'pay')) AS subquery + GROUP BY + tenant_id, + user_id) AS subquery; +Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> GroupAggregate + Group Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Sort + Sort Key: ((users.composite_id).tenant_id), ((users.composite_id).user_id) + -> Result + One-Time Filter: false +-- Union and left join subquery pushdown +EXPLAIN (COSTS OFF) +SELECT + avg(array_length(events, 1)) AS event_average, + hasdone +FROM + (SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(hasdone, 'Has not done paying') AS hasdone + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id) as composite_id, + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id) as composite_id, + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + DISTINCT ON ((composite_id).tenant_id, (composite_id).user_id) composite_id, + (composite_id).tenant_id, + (composite_id).user_id, + 'Has done paying'::TEXT AS hasdone + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay') AS subquery_2 + ON + subquery_1.composite_id = subquery_2.composite_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + hasdone) AS subquery_top +GROUP BY + hasdone; +HashAggregate + Group Key: remote_scan.hasdone + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> HashAggregate + Group Key: COALESCE(subquery_2.hasdone, 'Has not done paying'::text) + -> GroupAggregate + Group Key: ((composite_id).tenant_id), ((composite_id).user_id), subquery_2.hasdone + -> Sort + Sort Key: ((composite_id).tenant_id), ((composite_id).user_id), subquery_2.hasdone + -> Hash Left Join + Hash Cond: (composite_id = subquery_2.composite_id) + -> Unique + -> Sort + Sort Key: ((composite_id).tenant_id), ((composite_id).user_id), composite_id, ('action=>1'::text), event_time + -> Append + -> Result + One-Time Filter: false + -> Result + One-Time Filter: false + -> Hash + -> Subquery Scan on subquery_2 + -> Unique + -> Sort + Sort Key: ((events.composite_id).tenant_id), ((events.composite_id).user_id) + -> Seq Scan on events_1400027 events + Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text)) +-- Union, left join and having subquery pushdown +EXPLAIN (COSTS OFF) + SELECT + avg(array_length(events, 1)) AS event_average, + count_pay + FROM ( + SELECT + subquery_1.tenant_id, + subquery_1.user_id, + array_agg(event ORDER BY event_time) AS events, + COALESCE(count_pay, 0) AS count_pay + FROM + ( + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id), + 'action=>1'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'click') + UNION + (SELECT + (users.composite_id).tenant_id, + (users.composite_id).user_id, + (users.composite_id), + 'action=>2'AS event, + events.event_time + FROM + users, + events + WHERE + (users.composite_id) = (events.composite_id) AND + users.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + users.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'submit') + ) AS subquery_1 + LEFT JOIN + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + composite_id, + COUNT(*) AS count_pay + FROM + events + WHERE + events.composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + events.composite_id <= '(1, 9223372036854775807)'::user_composite_type AND + event_type = 'pay' + GROUP BY + composite_id + HAVING + COUNT(*) > 2) AS subquery_2 + ON + subquery_1.composite_id = subquery_2.composite_id + GROUP BY + subquery_1.tenant_id, + subquery_1.user_id, + count_pay) AS subquery_top +WHERE + array_ndims(events) > 0 +GROUP BY + count_pay +ORDER BY + count_pay; +ERROR: bogus varattno for OUTER_VAR var: 3 +-- Lateral join subquery pushdown +-- set subquery_pushdown due to limit in the query +SET citus.subquery_pushdown to ON; +EXPLAIN (COSTS OFF) +SELECT + tenant_id, + user_id, + user_lastseen, + event_array +FROM + (SELECT + tenant_id, + user_id, + max(lastseen) as user_lastseen, + array_agg(event_type ORDER BY event_time) AS event_array + FROM + (SELECT + (composite_id).tenant_id, + (composite_id).user_id, + composite_id, + lastseen + FROM + users + WHERE + composite_id >= '(1, -9223372036854775808)'::user_composite_type AND + composite_id <= '(1, 9223372036854775807)'::user_composite_type + ORDER BY + lastseen DESC + LIMIT + 10 + ) AS subquery_top + LEFT JOIN LATERAL + (SELECT + event_type, + event_time + FROM + events + WHERE + (composite_id) = subquery_top.composite_id + ORDER BY + event_time DESC + LIMIT + 99) AS subquery_lateral + ON + true + GROUP BY + tenant_id, + user_id + ) AS shard_union +ORDER BY + user_lastseen DESC +LIMIT + 10; +Limit + -> Sort + Sort Key: remote_scan.user_lastseen DESC + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Limit + -> Sort + Sort Key: (max(lastseen)) DESC + -> GroupAggregate + Group Key: ((composite_id).tenant_id), ((composite_id).user_id) + -> Sort + Sort Key: ((composite_id).tenant_id), ((composite_id).user_id) + -> Nested Loop Left Join + -> Limit + -> Sort + Sort Key: lastseen DESC + -> Result + One-Time Filter: false + -> Limit + -> Sort + Sort Key: events.event_time DESC + -> Seq Scan on events_1400027 events + Filter: (composite_id = composite_id) +-- Test all tasks output +SET citus.explain_all_tasks TO on; +EXPLAIN (COSTS FALSE) + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290005 lineitem + Filter: (l_orderkey > 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290004 lineitem + Filter: (l_orderkey > 9030) + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290007 lineitem + Filter: (l_orderkey > 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290006 lineitem + Filter: (l_orderkey > 9030) +SELECT true AS valid FROM explain_xml($$ + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030$$); +t +SELECT true AS valid FROM explain_json($$ + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030$$); +t +-- Test track tracker +SET citus.task_executor_type TO 'task-tracker'; +SET citus.explain_all_tasks TO off; +EXPLAIN (COSTS FALSE) + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +Aggregate + -> Custom Scan (Citus Task-Tracker) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290005 lineitem + Filter: (l_orderkey > 9030) +-- Test re-partition join +SET citus.large_table_shard_count TO 1; +EXPLAIN (COSTS FALSE) + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +Aggregate + -> Custom Scan (Citus Task-Tracker) + Task Count: 1 + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 1 + Merge Task Count: 1 + -> MapMergeJob + Map Task Count: 8 + Merge Task Count: 1 +EXPLAIN (COSTS FALSE, FORMAT JSON) + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +[ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Plans": [ + { + "Node Type": "Custom Scan", + "Parent Relationship": "Outer", + "Custom Plan Provider": "Citus Task-Tracker", + "Distributed Query": { + "Job": { + "Task Count": 1, + "Tasks Shown": "None, not supported for re-partition queries", + "Depended Jobs": [ + { + "Map Task Count": 1, + "Merge Task Count": 1, + "Depended Jobs": [ + { + "Map Task Count": 8, + "Merge Task Count": 1 + } + ] + } + ] + } + } + } + ] + } + } +] +SELECT true AS valid FROM explain_json($$ + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey$$); +t +EXPLAIN (COSTS FALSE, FORMAT XML) + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; + + + + Aggregate + Plain + + + Custom Scan + Outer + Citus Task-Tracker + + + 1 + None, not supported for re-partition queries + + + 1 + 1 + + + 8 + 1 + + + + + + + + + + + +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM lineitem, orders, customer, supplier + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey$$); +t +-- make sure that EXPLAIN works without +-- problems for queries that inlvolves only +-- reference tables +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM nation + WHERE n_name = 'CHINA'$$); +t +SELECT true AS valid FROM explain_xml($$ + SELECT count(*) + FROM nation, supplier + WHERE nation.n_nationkey = supplier.s_nationkey$$); +t +EXPLAIN (COSTS FALSE, FORMAT YAML) + SELECT count(*) + FROM lineitem, orders, customer, supplier_single_shard + WHERE l_orderkey = o_orderkey + AND o_custkey = c_custkey + AND l_suppkey = s_suppkey; +- Plan: + Node Type: "Aggregate" + Strategy: "Plain" + Plans: + - Node Type: "Custom Scan" + Parent Relationship: "Outer" + Custom Plan Provider: "Citus Task-Tracker" + Distributed Query: + Job: + Task Count: 1 + Tasks Shown: "None, not supported for re-partition queries" + Depended Jobs: + - Map Task Count: 1 + Merge Task Count: 1 + Depended Jobs: + - Map Task Count: 8 + Merge Task Count: 1 +-- test parallel aggregates +SET parallel_setup_cost=0; +ERROR: unrecognized configuration parameter "parallel_setup_cost" +SET parallel_tuple_cost=0; +ERROR: unrecognized configuration parameter "parallel_tuple_cost" +SET min_parallel_relation_size=0; +ERROR: unrecognized configuration parameter "min_parallel_relation_size" +SET min_parallel_table_scan_size=0; +ERROR: unrecognized configuration parameter "min_parallel_table_scan_size" +SET max_parallel_workers_per_gather=4; +ERROR: unrecognized configuration parameter "max_parallel_workers_per_gather" +-- ensure local plans display correctly +CREATE TABLE lineitem_clone (LIKE lineitem); +EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem_clone; +Aggregate + -> Seq Scan on lineitem_clone +-- ensure distributed plans don't break +EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem; +Aggregate + -> Custom Scan (Citus Task-Tracker) + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290001 lineitem +-- ensure EXPLAIN EXECUTE doesn't crash +PREPARE task_tracker_query AS + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +EXPLAIN (COSTS FALSE) EXECUTE task_tracker_query; +Aggregate + -> Custom Scan (Citus Task-Tracker) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290005 lineitem + Filter: (l_orderkey > 9030) +SET citus.task_executor_type TO 'real-time'; +PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; +EXPLAIN EXECUTE router_executor_query; +Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5) + Index Cond: (l_orderkey = 5) +PREPARE real_time_executor_query AS + SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; +EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query; +Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Seq Scan on lineitem_290005 lineitem + Filter: (l_orderkey > 9030) +-- EXPLAIN EXECUTE of parametrized prepared statements is broken, but +-- at least make sure to fail without crashing +PREPARE router_executor_query_param(int) AS SELECT l_quantity FROM lineitem WHERE l_orderkey = $1; +EXPLAIN EXECUTE router_executor_query_param(5); +Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem (cost=0.28..11.83 rows=3 width=5) + Index Cond: (l_orderkey = 5) +-- test explain in a transaction with alter table to test we use right connections +BEGIN; +CREATE TABLE explain_table(id int); +SELECT create_distributed_table('explain_table', 'id'); + +ALTER TABLE explain_table ADD COLUMN value int; +NOTICE: using one-phase commit for distributed DDL commands +HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' +EXPLAIN (COSTS FALSE) SELECT value FROM explain_table WHERE id = 1; +Custom Scan (Citus Router) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Seq Scan on explain_table_570001 explain_table + Filter: (id = 1) +ROLLBACK; +-- test explain with local INSERT ... SELECT +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part +SELECT o_orderkey FROM orders_hash_part LIMIT 3; +Custom Scan (Citus INSERT ... SELECT via coordinator) + -> Limit + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Limit + -> Seq Scan on orders_hash_part_360295 orders_hash_part +SELECT true AS valid FROM explain_json($$ + INSERT INTO lineitem_hash_part (l_orderkey) + SELECT o_orderkey FROM orders_hash_part LIMIT 3; +$$); +t +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part (l_orderkey, l_quantity) +SELECT o_orderkey, 5 FROM orders_hash_part LIMIT 3; +Custom Scan (Citus INSERT ... SELECT via coordinator) + -> Limit + -> Custom Scan (Citus Real-Time) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=57637 dbname=regression + -> Limit + -> Seq Scan on orders_hash_part_360295 orders_hash_part +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part (l_orderkey) +SELECT s FROM generate_series(1,5) s; +Custom Scan (Citus INSERT ... SELECT via coordinator) + -> Function Scan on generate_series s +EXPLAIN (COSTS OFF) +WITH cte1 AS (SELECT s FROM generate_series(1,10) s) +INSERT INTO lineitem_hash_part +WITH cte1 AS (SELECT * FROM cte1 LIMIT 5) +SELECT s FROM cte1; +Custom Scan (Citus INSERT ... SELECT via coordinator) + -> Subquery Scan on citus_insert_select_subquery + CTE cte1 + -> Function Scan on generate_series s + -> CTE Scan on cte1 + CTE cte1 + -> Limit + -> CTE Scan on cte1 cte1_1 +EXPLAIN (COSTS OFF) +INSERT INTO lineitem_hash_part +( SELECT s FROM generate_series(1,5) s) UNION +( SELECT s FROM generate_series(5,10) s); +Custom Scan (Citus INSERT ... SELECT via coordinator) + -> Subquery Scan on citus_insert_select_subquery + -> HashAggregate + Group Key: s.s + -> Append + -> Function Scan on generate_series s + -> Function Scan on generate_series s_1 diff --git a/src/test/regress/expected/multi_join_order_additional.out b/src/test/regress/expected/multi_join_order_additional.out index 05971983c..6e50b3a95 100644 --- a/src/test/regress/expected/multi_join_order_additional.out +++ b/src/test/regress/expected/multi_join_order_additional.out @@ -2,6 +2,13 @@ -- MULTI_JOIN_ORDER_ADDITIONAL -- ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 650000; +-- print whether we're running on 9.5 to make version-specific tests clear +SELECT substring(version(), '\d+(?:\.\d+)?') = '9.5' AS is_95; + is_95 +------- + f +(1 row) + -- Set configuration to print table join order and pruned shards SET citus.explain_distributed_queries TO off; SET citus.log_multi_join_order TO TRUE; diff --git a/src/test/regress/expected/multi_join_order_additional_1.out b/src/test/regress/expected/multi_join_order_additional_1.out new file mode 100644 index 000000000..8393630fa --- /dev/null +++ b/src/test/regress/expected/multi_join_order_additional_1.out @@ -0,0 +1,260 @@ +-- +-- MULTI_JOIN_ORDER_ADDITIONAL +-- +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 650000; +-- print whether we're running on 9.5 to make version-specific tests clear +SELECT substring(version(), '\d+(?:\.\d+)?') = '9.5' AS is_95; + is_95 +------- + t +(1 row) + +-- Set configuration to print table join order and pruned shards +SET citus.explain_distributed_queries TO off; +SET citus.log_multi_join_order TO TRUE; +SET citus.task_executor_type = 'task-tracker'; -- can't explain all queries otherwise +SET client_min_messages TO DEBUG2; +-- Create new table definitions for use in testing in distributed planning and +-- execution functionality. Also create indexes to boost performance. +CREATE TABLE lineitem_hash ( + l_orderkey bigint not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimal(15, 2) not null, + l_extendedprice decimal(15, 2) not null, + l_discount decimal(15, 2) not null, + l_tax decimal(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null, + PRIMARY KEY(l_orderkey, l_linenumber) ); +DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "lineitem_hash_pkey" for table "lineitem_hash" +DEBUG: building index "lineitem_hash_pkey" on table "lineitem_hash" +DEBUG: creating and filling new WAL file +DEBUG: done creating and filling new WAL file +SELECT master_create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_worker_shards('lineitem_hash', 2, 1); + master_create_worker_shards +----------------------------- + +(1 row) + +CREATE INDEX lineitem_hash_time_index ON lineitem_hash (l_shipdate); +DEBUG: building index "lineitem_hash_time_index" on table "lineitem_hash" +NOTICE: using one-phase commit for distributed DDL commands +HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' +CREATE TABLE orders_hash ( + o_orderkey bigint not null, + o_custkey integer not null, + o_orderstatus char(1) not null, + o_totalprice decimal(15,2) not null, + o_orderdate date not null, + o_orderpriority char(15) not null, + o_clerk char(15) not null, + o_shippriority integer not null, + o_comment varchar(79) not null, + PRIMARY KEY(o_orderkey) ); +DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "orders_hash_pkey" for table "orders_hash" +DEBUG: building index "orders_hash_pkey" on table "orders_hash" +SELECT master_create_distributed_table('orders_hash', 'o_orderkey', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_worker_shards('orders_hash', 2, 1); + master_create_worker_shards +----------------------------- + +(1 row) + +CREATE TABLE customer_hash ( + c_custkey integer not null, + c_name varchar(25) not null, + c_address varchar(40) not null, + c_nationkey integer not null, + c_phone char(15) not null, + c_acctbal decimal(15,2) not null, + c_mktsegment char(10) not null, + c_comment varchar(117) not null); +SELECT master_create_distributed_table('customer_hash', 'c_custkey', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +SELECT master_create_worker_shards('customer_hash', 2, 1); + master_create_worker_shards +----------------------------- + +(1 row) + +-- The following query checks that we can correctly handle self-joins +EXPLAIN SELECT l1.l_quantity FROM lineitem l1, lineitem l2 + WHERE l1.l_orderkey = l2.l_orderkey AND l1.l_quantity > 5; +LOG: join order: [ "lineitem" ][ local partition join "lineitem" ] +DEBUG: join prunable for intervals [1,1509] and [2951,4455] +DEBUG: join prunable for intervals [1,1509] and [4480,5986] +DEBUG: join prunable for intervals [1,1509] and [8997,10560] +DEBUG: join prunable for intervals [1,1509] and [10560,12036] +DEBUG: join prunable for intervals [1,1509] and [12036,13473] +DEBUG: join prunable for intervals [1,1509] and [13473,14947] +DEBUG: join prunable for intervals [1509,4964] and [8997,10560] +DEBUG: join prunable for intervals [1509,4964] and [10560,12036] +DEBUG: join prunable for intervals [1509,4964] and [12036,13473] +DEBUG: join prunable for intervals [1509,4964] and [13473,14947] +DEBUG: join prunable for intervals [2951,4455] and [1,1509] +DEBUG: join prunable for intervals [2951,4455] and [4480,5986] +DEBUG: join prunable for intervals [2951,4455] and [8997,10560] +DEBUG: join prunable for intervals [2951,4455] and [10560,12036] +DEBUG: join prunable for intervals [2951,4455] and [12036,13473] +DEBUG: join prunable for intervals [2951,4455] and [13473,14947] +DEBUG: join prunable for intervals [4480,5986] and [1,1509] +DEBUG: join prunable for intervals [4480,5986] and [2951,4455] +DEBUG: join prunable for intervals [4480,5986] and [8997,10560] +DEBUG: join prunable for intervals [4480,5986] and [10560,12036] +DEBUG: join prunable for intervals [4480,5986] and [12036,13473] +DEBUG: join prunable for intervals [4480,5986] and [13473,14947] +DEBUG: join prunable for intervals [8997,10560] and [1,1509] +DEBUG: join prunable for intervals [8997,10560] and [1509,4964] +DEBUG: join prunable for intervals [8997,10560] and [2951,4455] +DEBUG: join prunable for intervals [8997,10560] and [4480,5986] +DEBUG: join prunable for intervals [8997,10560] and [12036,13473] +DEBUG: join prunable for intervals [8997,10560] and [13473,14947] +DEBUG: join prunable for intervals [10560,12036] and [1,1509] +DEBUG: join prunable for intervals [10560,12036] and [1509,4964] +DEBUG: join prunable for intervals [10560,12036] and [2951,4455] +DEBUG: join prunable for intervals [10560,12036] and [4480,5986] +DEBUG: join prunable for intervals [10560,12036] and [13473,14947] +DEBUG: join prunable for intervals [12036,13473] and [1,1509] +DEBUG: join prunable for intervals [12036,13473] and [1509,4964] +DEBUG: join prunable for intervals [12036,13473] and [2951,4455] +DEBUG: join prunable for intervals [12036,13473] and [4480,5986] +DEBUG: join prunable for intervals [12036,13473] and [8997,10560] +DEBUG: join prunable for intervals [13473,14947] and [1,1509] +DEBUG: join prunable for intervals [13473,14947] and [1509,4964] +DEBUG: join prunable for intervals [13473,14947] and [2951,4455] +DEBUG: join prunable for intervals [13473,14947] and [4480,5986] +DEBUG: join prunable for intervals [13473,14947] and [8997,10560] +DEBUG: join prunable for intervals [13473,14947] and [10560,12036] + QUERY PLAN +-------------------------------------------------------------------- + Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(2 rows) + +-- Update configuration to treat lineitem and orders tables as large +SET citus.large_table_shard_count TO 2; +SET client_min_messages TO LOG; +-- The following queries check that we correctly handle joins and OR clauses. In +-- particular, these queries check that we factorize out OR clauses if possible, +-- and that we default to a cartesian product otherwise. +EXPLAIN SELECT count(*) FROM lineitem, orders + WHERE (l_orderkey = o_orderkey AND l_quantity > 5) + OR (l_orderkey = o_orderkey AND l_quantity < 10); +LOG: join order: [ "lineitem" ][ local partition join "orders" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT l_quantity FROM lineitem, orders + WHERE (l_orderkey = o_orderkey OR l_quantity > 5); +LOG: join order: [ "lineitem" ][ cartesian product "orders" ] +ERROR: cannot perform distributed planning on this query +DETAIL: Cartesian products are currently unsupported +-- The below queries modify the partition method in pg_dist_partition. We thus +-- begin a transaction here so the changes don't impact any other parallel +-- running tests. +BEGIN; +-- Validate that we take into account the partition method when building the +-- join-order plan. +EXPLAIN SELECT count(*) FROM orders, lineitem_hash + WHERE o_orderkey = l_orderkey; +LOG: join order: [ "orders" ][ single partition join "lineitem_hash" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Verify we handle local joins between two hash-partitioned tables. +EXPLAIN SELECT count(*) FROM orders_hash, lineitem_hash + WHERE o_orderkey = l_orderkey; +LOG: join order: [ "orders_hash" ][ local partition join "lineitem_hash" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Validate that we can handle broadcast joins with hash-partitioned tables. +EXPLAIN SELECT count(*) FROM customer_hash, nation + WHERE c_nationkey = n_nationkey; +LOG: join order: [ "customer_hash" ][ broadcast join "nation" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Update the large table shard count for all the following tests. +SET citus.large_table_shard_count TO 1; +-- Validate that we don't use a single-partition join method for a hash +-- re-partitioned table, thus preventing a partition of just the customer table. +EXPLAIN SELECT count(*) FROM orders, lineitem, customer + WHERE o_custkey = l_partkey AND o_custkey = c_nationkey; +LOG: join order: [ "orders" ][ dual partition join "lineitem" ][ dual partition join "customer" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Validate that we don't chose a single-partition join method with a +-- hash-partitioned base table +EXPLAIN SELECT count(*) FROM orders, customer_hash + WHERE c_custkey = o_custkey; +LOG: join order: [ "orders" ][ dual partition join "customer_hash" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Validate that we can re-partition a hash partitioned table to join with a +-- range partitioned one. +EXPLAIN SELECT count(*) FROM orders_hash, customer + WHERE c_custkey = o_custkey; +LOG: join order: [ "orders_hash" ][ single partition join "customer" ] + QUERY PLAN +-------------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Task-Tracker) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +COMMIT; +-- Reset client logging level to its previous value +SET client_min_messages TO NOTICE; +DROP TABLE lineitem_hash; +DROP TABLE orders_hash; +DROP TABLE customer_hash; diff --git a/src/test/regress/expected/multi_large_table_join_planning.out b/src/test/regress/expected/multi_large_table_join_planning.out index 2237b8014..41db9b4e0 100644 --- a/src/test/regress/expected/multi_large_table_join_planning.out +++ b/src/test/regress/expected/multi_large_table_join_planning.out @@ -7,17 +7,18 @@ -- executor here, as we cannot run repartition jobs with real time executor. ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 690000; SET citus.enable_unique_job_ids TO off; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 10 +(1 row) + BEGIN; SET client_min_messages TO DEBUG4; -DEBUG: CommitTransactionCommand SET citus.large_table_shard_count TO 2; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand SET citus.task_executor_type TO 'task-tracker'; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand -- Debug4 log messages display jobIds within them. We explicitly set the jobId -- sequence here so that the regression output becomes independent of the number -- of jobs executed prior to running this test. @@ -40,7 +41,6 @@ GROUP BY l_partkey, o_orderkey ORDER BY l_partkey, o_orderkey; -DEBUG: StartTransactionCommand DEBUG: join prunable for intervals [1,1509] and [8997,14946] DEBUG: join prunable for intervals [1509,4964] and [8997,14946] DEBUG: join prunable for intervals [2951,4455] and [8997,14946] @@ -112,7 +112,6 @@ DEBUG: completed cleanup query for job 2 DEBUG: completed cleanup query for job 2 DEBUG: completed cleanup query for job 1 DEBUG: completed cleanup query for job 1 -DEBUG: CommitTransactionCommand l_partkey | o_orderkey | count -----------+------------+------- 18 | 12005 | 1 @@ -157,7 +156,6 @@ GROUP BY l_partkey, o_orderkey ORDER BY l_partkey, o_orderkey; -DEBUG: StartTransactionCommand DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for task 4 @@ -234,13 +232,10 @@ DEBUG: completed cleanup query for job 4 DEBUG: completed cleanup query for job 4 DEBUG: completed cleanup query for job 5 DEBUG: completed cleanup query for job 5 -DEBUG: CommitTransactionCommand l_partkey | o_orderkey | count -----------+------------+------- (0 rows) -- Reset client logging level to its previous value SET client_min_messages TO NOTICE; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility COMMIT; diff --git a/src/test/regress/expected/multi_large_table_join_planning_0.out b/src/test/regress/expected/multi_large_table_join_planning_0.out new file mode 100644 index 000000000..569e3acc2 --- /dev/null +++ b/src/test/regress/expected/multi_large_table_join_planning_0.out @@ -0,0 +1,254 @@ +-- +-- MULTI_LARGE_TABLE_PLANNING +-- +-- Tests that cover large table join planning. Note that we explicitly start a +-- transaction block here so that we don't emit debug messages with changing +-- transaction ids in them. Also, we set the executor type to task tracker +-- executor here, as we cannot run repartition jobs with real time executor. +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 690000; +SET citus.enable_unique_job_ids TO off; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 9 +(1 row) + +BEGIN; +SET client_min_messages TO DEBUG4; +DEBUG: CommitTransactionCommand +SET citus.large_table_shard_count TO 2; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +SET citus.task_executor_type TO 'task-tracker'; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +-- Debug4 log messages display jobIds within them. We explicitly set the jobId +-- sequence here so that the regression output becomes independent of the number +-- of jobs executed prior to running this test. +-- Multi-level repartition join to verify our projection columns are correctly +-- referenced and propagated across multiple repartition jobs. The test also +-- validates that only the minimal necessary projection columns are transferred +-- between jobs. +SELECT + l_partkey, o_orderkey, count(*) +FROM + lineitem, part, orders, customer +WHERE + l_orderkey = o_orderkey AND + l_partkey = p_partkey AND + c_custkey = o_custkey AND + (l_quantity > 5.0 OR l_extendedprice > 1200.0) AND + p_size > 8 AND o_totalprice > 10.0 AND + c_acctbal < 5000.0 AND l_partkey < 1000 +GROUP BY + l_partkey, o_orderkey +ORDER BY + l_partkey, o_orderkey; +DEBUG: StartTransactionCommand +DEBUG: join prunable for intervals [1,1509] and [8997,14946] +DEBUG: join prunable for intervals [1509,4964] and [8997,14946] +DEBUG: join prunable for intervals [2951,4455] and [8997,14946] +DEBUG: join prunable for intervals [4480,5986] and [8997,14946] +DEBUG: join prunable for intervals [8997,10560] and [1,5986] +DEBUG: join prunable for intervals [10560,12036] and [1,5986] +DEBUG: join prunable for intervals [12036,13473] and [1,5986] +DEBUG: join prunable for intervals [13473,14947] and [1,5986] +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290000 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290001 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 9 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290002 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 12 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290003 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 15 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290004 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 18 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290005 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 21 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290006 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: generated sql query for task 24 +DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290007 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 18 to node localhost:57637 +DEBUG: assigned task 15 to node localhost:57638 +DEBUG: assigned task 24 to node localhost:57637 +DEBUG: assigned task 21 to node localhost:57638 +DEBUG: join prunable for intervals [1,1000] and [6001,7000] +DEBUG: join prunable for intervals [6001,7000] and [1,1000] +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000025".intermediate_column_1_0, "pg_merge_job_0001.task_000025".intermediate_column_1_1, "pg_merge_job_0001.task_000025".intermediate_column_1_2, "pg_merge_job_0001.task_000025".intermediate_column_1_3, "pg_merge_job_0001.task_000025".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000025 "pg_merge_job_0001.task_000025" JOIN part_290011 part ON (("pg_merge_job_0001.task_000025".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000034".intermediate_column_1_0, "pg_merge_job_0001.task_000034".intermediate_column_1_1, "pg_merge_job_0001.task_000034".intermediate_column_1_2, "pg_merge_job_0001.task_000034".intermediate_column_1_3, "pg_merge_job_0001.task_000034".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000034 "pg_merge_job_0001.task_000034" JOIN part_280002 part ON (("pg_merge_job_0001.task_000034".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 25 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 34 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: join prunable for intervals [1,1000] and [1001,2000] +DEBUG: join prunable for intervals [1,1000] and [6001,7000] +DEBUG: join prunable for intervals [1001,2000] and [1,1000] +DEBUG: join prunable for intervals [1001,2000] and [6001,7000] +DEBUG: join prunable for intervals [6001,7000] and [1,1000] +DEBUG: join prunable for intervals [6001,7000] and [1001,2000] +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000007".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000007".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000007 "pg_merge_job_0002.task_000007" JOIN customer_290010 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000007".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000007".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000007".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000007".intermediate_column_2_0, "pg_merge_job_0002.task_000007".intermediate_column_2_1" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000010".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000010".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000010 "pg_merge_job_0002.task_000010" JOIN customer_280001 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000010".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000010".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000010".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000010".intermediate_column_2_0, "pg_merge_job_0002.task_000010".intermediate_column_2_1" +DEBUG: generated sql query for task 9 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000013".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000013".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000013 "pg_merge_job_0002.task_000013" JOIN customer_280000 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000013".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000013".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000013".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000013".intermediate_column_2_0, "pg_merge_job_0002.task_000013".intermediate_column_2_1" +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 10 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 13 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: completed cleanup query for job 3 +DEBUG: completed cleanup query for job 3 +DEBUG: completed cleanup query for job 2 +DEBUG: completed cleanup query for job 2 +DEBUG: completed cleanup query for job 1 +DEBUG: completed cleanup query for job 1 +DEBUG: CommitTransactionCommand + l_partkey | o_orderkey | count +-----------+------------+------- + 18 | 12005 | 1 + 79 | 5121 | 1 + 91 | 2883 | 1 + 222 | 9413 | 1 + 278 | 1287 | 1 + 309 | 2374 | 1 + 318 | 321 | 1 + 321 | 5984 | 1 + 337 | 10403 | 1 + 350 | 13698 | 1 + 358 | 4323 | 1 + 364 | 9347 | 1 + 416 | 640 | 1 + 426 | 10855 | 1 + 450 | 35 | 1 + 484 | 3843 | 1 + 504 | 14566 | 1 + 510 | 13569 | 1 + 532 | 3175 | 1 + 641 | 134 | 1 + 669 | 10944 | 1 + 716 | 2885 | 1 + 738 | 4355 | 1 + 802 | 2534 | 1 + 824 | 9287 | 1 + 864 | 3175 | 1 + 957 | 4293 | 1 + 960 | 10980 | 1 + 963 | 4580 | 1 +(29 rows) + +SELECT + l_partkey, o_orderkey, count(*) +FROM + lineitem, orders +WHERE + l_suppkey = o_shippriority AND + l_quantity < 5.0 AND o_totalprice <> 4.0 +GROUP BY + l_partkey, o_orderkey +ORDER BY + l_partkey, o_orderkey; +DEBUG: StartTransactionCommand +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290002 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 8 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290003 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 10 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290004 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 12 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290005 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 14 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290006 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 16 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290007 lineitem WHERE (l_quantity < 5.0)" +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 10 to node localhost:57638 +DEBUG: assigned task 16 to node localhost:57637 +DEBUG: assigned task 14 to node localhost:57638 +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290008 orders WHERE (o_totalprice <> 4.0)" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290009 orders WHERE (o_totalprice <> 4.0)" +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: join prunable for task partitionId 0 and 1 +DEBUG: join prunable for task partitionId 0 and 2 +DEBUG: join prunable for task partitionId 0 and 3 +DEBUG: join prunable for task partitionId 1 and 0 +DEBUG: join prunable for task partitionId 1 and 2 +DEBUG: join prunable for task partitionId 1 and 3 +DEBUG: join prunable for task partitionId 2 and 0 +DEBUG: join prunable for task partitionId 2 and 1 +DEBUG: join prunable for task partitionId 2 and 3 +DEBUG: join prunable for task partitionId 3 and 0 +DEBUG: join prunable for task partitionId 3 and 1 +DEBUG: join prunable for task partitionId 3 and 2 +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000017".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000005".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000017 "pg_merge_job_0004.task_000017" JOIN pg_merge_job_0005.task_000005 "pg_merge_job_0005.task_000005" ON (("pg_merge_job_0004.task_000017".intermediate_column_4_1 = "pg_merge_job_0005.task_000005".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000017".intermediate_column_4_0, "pg_merge_job_0005.task_000005".intermediate_column_5_0" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000026".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000008".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000026 "pg_merge_job_0004.task_000026" JOIN pg_merge_job_0005.task_000008 "pg_merge_job_0005.task_000008" ON (("pg_merge_job_0004.task_000026".intermediate_column_4_1 = "pg_merge_job_0005.task_000008".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000026".intermediate_column_4_0, "pg_merge_job_0005.task_000008".intermediate_column_5_0" +DEBUG: generated sql query for task 9 +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000035".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000011".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000035 "pg_merge_job_0004.task_000035" JOIN pg_merge_job_0005.task_000011 "pg_merge_job_0005.task_000011" ON (("pg_merge_job_0004.task_000035".intermediate_column_4_1 = "pg_merge_job_0005.task_000011".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000035".intermediate_column_4_0, "pg_merge_job_0005.task_000011".intermediate_column_5_0" +DEBUG: generated sql query for task 12 +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000044".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000014".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000044 "pg_merge_job_0004.task_000044" JOIN pg_merge_job_0005.task_000014 "pg_merge_job_0005.task_000014" ON (("pg_merge_job_0004.task_000044".intermediate_column_4_1 = "pg_merge_job_0005.task_000014".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000044".intermediate_column_4_0, "pg_merge_job_0005.task_000014".intermediate_column_5_0" +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 17 +DEBUG: pruning merge fetch taskId 2 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 8 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 35 +DEBUG: pruning merge fetch taskId 8 +DETAIL: Creating dependency on merge taskId 11 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 44 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 14 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: completed cleanup query for job 6 +DEBUG: completed cleanup query for job 6 +DEBUG: completed cleanup query for job 4 +DEBUG: completed cleanup query for job 4 +DEBUG: completed cleanup query for job 5 +DEBUG: completed cleanup query for job 5 +DEBUG: CommitTransactionCommand + l_partkey | o_orderkey | count +-----------+------------+------- +(0 rows) + +-- Reset client logging level to its previous value +SET client_min_messages TO NOTICE; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +COMMIT; diff --git a/src/test/regress/expected/multi_large_table_task_assignment.out b/src/test/regress/expected/multi_large_table_task_assignment.out index 49c75fb6f..8eaf83f14 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment.out +++ b/src/test/regress/expected/multi_large_table_task_assignment.out @@ -6,17 +6,18 @@ -- from a sql task to its depended tasks. Note that we set the executor type to task -- tracker executor here, as we cannot run repartition jobs with real time executor. ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 710000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 10 +(1 row) + BEGIN; SET client_min_messages TO DEBUG3; -DEBUG: CommitTransactionCommand SET citus.large_table_shard_count TO 2; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand SET citus.task_executor_type TO 'task-tracker'; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand -- Single range repartition join to test anchor-shard based task assignment and -- assignment propagation to merge and data-fetch tasks. SELECT @@ -25,7 +26,6 @@ FROM orders, customer WHERE o_custkey = c_custkey; -DEBUG: StartTransactionCommand DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] @@ -43,7 +43,6 @@ DETAIL: Creating dependency on merge taskId 11 DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 3 to node localhost:57637 -DEBUG: CommitTransactionCommand count ------- 2984 @@ -54,9 +53,6 @@ DEBUG: CommitTransactionCommand -- the same merge task, and tests our constraint group creation and assignment -- propagation. Here 'orders' is considered the small table. SET citus.large_table_shard_count TO 3; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand SELECT count(*) FROM @@ -64,7 +60,6 @@ FROM WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; -DEBUG: StartTransactionCommand DEBUG: assigned task 9 to node localhost:57637 DEBUG: assigned task 15 to node localhost:57638 DEBUG: assigned task 12 to node localhost:57637 @@ -175,16 +170,12 @@ DEBUG: propagating assignment from merge task 54 to constrained sql task 45 DEBUG: propagating assignment from merge task 61 to constrained sql task 51 DEBUG: propagating assignment from merge task 61 to constrained sql task 54 DEBUG: propagating assignment from merge task 68 to constrained sql task 60 -DEBUG: CommitTransactionCommand count ------- 11998 (1 row) SET citus.large_table_shard_count TO 2; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand -- Dual hash repartition join which tests the separate hash repartition join -- task assignment algorithm. SELECT @@ -193,7 +184,6 @@ FROM lineitem, customer WHERE l_partkey = c_nationkey; -DEBUG: StartTransactionCommand DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 DEBUG: assigned task 8 to node localhost:57637 @@ -237,7 +227,6 @@ DEBUG: assigned task 3 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 12 to node localhost:57637 -DEBUG: CommitTransactionCommand count ------- 125 @@ -245,6 +234,4 @@ DEBUG: CommitTransactionCommand -- Reset client logging level to its previous value SET client_min_messages TO NOTICE; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility COMMIT; diff --git a/src/test/regress/expected/multi_large_table_task_assignment_0.out b/src/test/regress/expected/multi_large_table_task_assignment_0.out new file mode 100644 index 000000000..c752d3cb7 --- /dev/null +++ b/src/test/regress/expected/multi_large_table_task_assignment_0.out @@ -0,0 +1,258 @@ +-- +-- MULTI_LARGE_TABLE_TASK_ASSIGNMENT +-- +-- Tests which cover task assignment for MapMerge jobs for single range repartition +-- and dual hash repartition joins. The tests also cover task assignment propagation +-- from a sql task to its depended tasks. Note that we set the executor type to task +-- tracker executor here, as we cannot run repartition jobs with real time executor. +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 710000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 9 +(1 row) + +BEGIN; +SET client_min_messages TO DEBUG3; +DEBUG: CommitTransactionCommand +SET citus.large_table_shard_count TO 2; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +SET citus.task_executor_type TO 'task-tracker'; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +-- Single range repartition join to test anchor-shard based task assignment and +-- assignment propagation to merge and data-fetch tasks. +SELECT + count(*) +FROM + orders, customer +WHERE + o_custkey = c_custkey; +DEBUG: StartTransactionCommand +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: join prunable for intervals [1,1000] and [1001,2000] +DEBUG: join prunable for intervals [1,1000] and [6001,7000] +DEBUG: join prunable for intervals [1001,2000] and [1,1000] +DEBUG: join prunable for intervals [1001,2000] and [6001,7000] +DEBUG: join prunable for intervals [6001,7000] and [1,1000] +DEBUG: join prunable for intervals [6001,7000] and [1001,2000] +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 8 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 11 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: CommitTransactionCommand + count +------- + 2984 +(1 row) + +-- Single range repartition join, along with a join with a small table containing +-- more than one shard. This situation results in multiple sql tasks depending on +-- the same merge task, and tests our constraint group creation and assignment +-- propagation. Here 'orders' is considered the small table. +SET citus.large_table_shard_count TO 3; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +SELECT + count(*) +FROM + orders, customer, lineitem +WHERE + o_custkey = c_custkey AND + o_orderkey = l_orderkey; +DEBUG: StartTransactionCommand +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 15 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: join prunable for intervals [1,1509] and [2951,4455] +DEBUG: join prunable for intervals [1,1509] and [4480,5986] +DEBUG: join prunable for intervals [1,1509] and [8997,10560] +DEBUG: join prunable for intervals [1,1509] and [10560,12036] +DEBUG: join prunable for intervals [1,1509] and [12036,13473] +DEBUG: join prunable for intervals [1,1509] and [13473,14947] +DEBUG: join prunable for intervals [1509,4964] and [8997,10560] +DEBUG: join prunable for intervals [1509,4964] and [10560,12036] +DEBUG: join prunable for intervals [1509,4964] and [12036,13473] +DEBUG: join prunable for intervals [1509,4964] and [13473,14947] +DEBUG: join prunable for intervals [2951,4455] and [1,1509] +DEBUG: join prunable for intervals [2951,4455] and [4480,5986] +DEBUG: join prunable for intervals [2951,4455] and [8997,10560] +DEBUG: join prunable for intervals [2951,4455] and [10560,12036] +DEBUG: join prunable for intervals [2951,4455] and [12036,13473] +DEBUG: join prunable for intervals [2951,4455] and [13473,14947] +DEBUG: join prunable for intervals [4480,5986] and [1,1509] +DEBUG: join prunable for intervals [4480,5986] and [2951,4455] +DEBUG: join prunable for intervals [4480,5986] and [8997,10560] +DEBUG: join prunable for intervals [4480,5986] and [10560,12036] +DEBUG: join prunable for intervals [4480,5986] and [12036,13473] +DEBUG: join prunable for intervals [4480,5986] and [13473,14947] +DEBUG: join prunable for intervals [8997,10560] and [1,1509] +DEBUG: join prunable for intervals [8997,10560] and [1509,4964] +DEBUG: join prunable for intervals [8997,10560] and [2951,4455] +DEBUG: join prunable for intervals [8997,10560] and [4480,5986] +DEBUG: join prunable for intervals [8997,10560] and [12036,13473] +DEBUG: join prunable for intervals [8997,10560] and [13473,14947] +DEBUG: join prunable for intervals [10560,12036] and [1,1509] +DEBUG: join prunable for intervals [10560,12036] and [1509,4964] +DEBUG: join prunable for intervals [10560,12036] and [2951,4455] +DEBUG: join prunable for intervals [10560,12036] and [4480,5986] +DEBUG: join prunable for intervals [10560,12036] and [13473,14947] +DEBUG: join prunable for intervals [12036,13473] and [1,1509] +DEBUG: join prunable for intervals [12036,13473] and [1509,4964] +DEBUG: join prunable for intervals [12036,13473] and [2951,4455] +DEBUG: join prunable for intervals [12036,13473] and [4480,5986] +DEBUG: join prunable for intervals [12036,13473] and [8997,10560] +DEBUG: join prunable for intervals [13473,14947] and [1,1509] +DEBUG: join prunable for intervals [13473,14947] and [1509,4964] +DEBUG: join prunable for intervals [13473,14947] and [2951,4455] +DEBUG: join prunable for intervals [13473,14947] and [4480,5986] +DEBUG: join prunable for intervals [13473,14947] and [8997,10560] +DEBUG: join prunable for intervals [13473,14947] and [10560,12036] +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 19 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 19 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 13 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 16 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 19 +DETAIL: Creating dependency on merge taskId 33 +DEBUG: pruning merge fetch taskId 22 +DETAIL: Creating dependency on merge taskId 33 +DEBUG: pruning merge fetch taskId 25 +DETAIL: Creating dependency on merge taskId 40 +DEBUG: pruning merge fetch taskId 28 +DETAIL: Creating dependency on merge taskId 40 +DEBUG: pruning merge fetch taskId 31 +DETAIL: Creating dependency on merge taskId 47 +DEBUG: pruning merge fetch taskId 34 +DETAIL: Creating dependency on merge taskId 47 +DEBUG: pruning merge fetch taskId 37 +DETAIL: Creating dependency on merge taskId 54 +DEBUG: pruning merge fetch taskId 40 +DETAIL: Creating dependency on merge taskId 54 +DEBUG: pruning merge fetch taskId 43 +DETAIL: Creating dependency on merge taskId 54 +DEBUG: pruning merge fetch taskId 46 +DETAIL: Creating dependency on merge taskId 61 +DEBUG: pruning merge fetch taskId 49 +DETAIL: Creating dependency on merge taskId 61 +DEBUG: pruning merge fetch taskId 52 +DETAIL: Creating dependency on merge taskId 61 +DEBUG: pruning merge fetch taskId 55 +DETAIL: Creating dependency on merge taskId 68 +DEBUG: pruning merge fetch taskId 58 +DETAIL: Creating dependency on merge taskId 68 +DEBUG: assigned task 21 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 27 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 48 to node localhost:57637 +DEBUG: assigned task 33 to node localhost:57638 +DEBUG: assigned task 39 to node localhost:57637 +DEBUG: assigned task 57 to node localhost:57638 +DEBUG: propagating assignment from merge task 19 to constrained sql task 6 +DEBUG: propagating assignment from merge task 26 to constrained sql task 12 +DEBUG: propagating assignment from merge task 26 to constrained sql task 15 +DEBUG: propagating assignment from merge task 26 to constrained sql task 18 +DEBUG: propagating assignment from merge task 33 to constrained sql task 24 +DEBUG: propagating assignment from merge task 40 to constrained sql task 30 +DEBUG: propagating assignment from merge task 47 to constrained sql task 36 +DEBUG: propagating assignment from merge task 54 to constrained sql task 42 +DEBUG: propagating assignment from merge task 54 to constrained sql task 45 +DEBUG: propagating assignment from merge task 61 to constrained sql task 51 +DEBUG: propagating assignment from merge task 61 to constrained sql task 54 +DEBUG: propagating assignment from merge task 68 to constrained sql task 60 +DEBUG: CommitTransactionCommand + count +------- + 11998 +(1 row) + +SET citus.large_table_shard_count TO 2; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +-- Dual hash repartition join which tests the separate hash repartition join +-- task assignment algorithm. +SELECT + count(*) +FROM + lineitem, customer +WHERE + l_partkey = c_nationkey; +DEBUG: StartTransactionCommand +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 10 to node localhost:57638 +DEBUG: assigned task 16 to node localhost:57637 +DEBUG: assigned task 14 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: join prunable for task partitionId 0 and 1 +DEBUG: join prunable for task partitionId 0 and 2 +DEBUG: join prunable for task partitionId 0 and 3 +DEBUG: join prunable for task partitionId 1 and 0 +DEBUG: join prunable for task partitionId 1 and 2 +DEBUG: join prunable for task partitionId 1 and 3 +DEBUG: join prunable for task partitionId 2 and 0 +DEBUG: join prunable for task partitionId 2 and 1 +DEBUG: join prunable for task partitionId 2 and 3 +DEBUG: join prunable for task partitionId 3 and 0 +DEBUG: join prunable for task partitionId 3 and 1 +DEBUG: join prunable for task partitionId 3 and 2 +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 17 +DEBUG: pruning merge fetch taskId 2 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 26 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 11 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 35 +DEBUG: pruning merge fetch taskId 8 +DETAIL: Creating dependency on merge taskId 15 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 44 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 19 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: CommitTransactionCommand + count +------- + 125 +(1 row) + +-- Reset client logging level to its previous value +SET client_min_messages TO NOTICE; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +COMMIT; diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out index 358915664..b7a980009 100644 --- a/src/test/regress/expected/multi_metadata_sync.out +++ b/src/test/regress/expected/multi_metadata_sync.out @@ -240,16 +240,27 @@ SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; 1310007 | 1 | 0 | localhost | 57638 | 100007 (8 rows) -\d mx_testing_schema.mx_test_table - Table "mx_testing_schema.mx_test_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; Column | Type | Modifiers --------+---------+--------------------------------------------------------------------------------- col_1 | integer | col_2 | text | not null col_3 | bigint | not null default nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) -Indexes: - "mx_test_table_col_1_key" UNIQUE CONSTRAINT, btree (col_1) - "mx_index" btree (col_2) +(3 rows) + +\d mx_testing_schema.mx_test_table_col_1_key +Index "mx_testing_schema.mx_test_table_col_1_key" + Column | Type | Definition +--------+---------+------------ + col_1 | integer | col_1 +unique, btree, for table "mx_testing_schema.mx_test_table" + +\d mx_testing_schema.mx_index +Index "mx_testing_schema.mx_index" + Column | Type | Definition +--------+------+------------ + col_2 | text | col_2 +btree, for table "mx_testing_schema.mx_test_table" -- Check that pg_dist_colocation is not synced SELECT * FROM pg_dist_colocation ORDER BY colocationid; @@ -295,15 +306,11 @@ SELECT start_metadata_sync_to_node('localhost', :worker_1_port); -- Check that foreign key metadata exists on the worker \c - - - :worker_1_port -\d mx_testing_schema_2.fk_test_2 -Table "mx_testing_schema_2.fk_test_2" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | integer | - col3 | text | -Foreign-key constraints: - "fk_test_2_col1_fkey" FOREIGN KEY (col1, col2) REFERENCES mx_testing_schema.fk_test_1(col1, col3) +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_testing_schema_2.fk_test_2'::regclass; + Constraint | Definition +---------------------+----------------------------------------------------------------------------- + fk_test_2_col1_fkey | FOREIGN KEY (col1, col2) REFERENCES mx_testing_schema.fk_test_1(col1, col3) +(1 row) \c - - - :master_port DROP TABLE mx_testing_schema_2.fk_test_2; @@ -370,16 +377,27 @@ SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; 1310007 | 1 | 0 | localhost | 57638 | 100007 (8 rows) -\d mx_testing_schema.mx_test_table - Table "mx_testing_schema.mx_test_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; Column | Type | Modifiers --------+---------+--------------------------------------------------------------------------------- col_1 | integer | col_2 | text | not null col_3 | bigint | not null default nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) -Indexes: - "mx_test_table_col_1_key" UNIQUE CONSTRAINT, btree (col_1) - "mx_index" btree (col_2) +(3 rows) + +\d mx_testing_schema.mx_test_table_col_1_key +Index "mx_testing_schema.mx_test_table_col_1_key" + Column | Type | Definition +--------+---------+------------ + col_1 | integer | col_1 +unique, btree, for table "mx_testing_schema.mx_test_table" + +\d mx_testing_schema.mx_index +Index "mx_testing_schema.mx_index" + Column | Type | Definition +--------+------+------------ + col_2 | text | col_2 +btree, for table "mx_testing_schema.mx_test_table" SELECT count(*) FROM pg_trigger WHERE tgrelid='mx_testing_schema.mx_test_table'::regclass; count @@ -499,28 +517,46 @@ CREATE INDEX mx_index_1 ON mx_test_schema_1.mx_table_1 (col1); CREATE TABLE mx_test_schema_2.mx_table_2 (col1 int, col2 text); CREATE INDEX mx_index_2 ON mx_test_schema_2.mx_table_2 (col2); ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY(col1) REFERENCES mx_test_schema_1.mx_table_1(col1); -\d mx_test_schema_1.mx_table_1 -Table "mx_test_schema_1.mx_table_1" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; Column | Type | Modifiers --------+---------+----------- col1 | integer | col2 | text | -Indexes: - "mx_table_1_col1_key" UNIQUE CONSTRAINT, btree (col1) - "mx_index_1" btree (col1) -Referenced by: - TABLE "mx_test_schema_2.mx_table_2" CONSTRAINT "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +(2 rows) -\d mx_test_schema_2.mx_table_2 -Table "mx_test_schema_2.mx_table_2" +\d mx_test_schema_1.mx_table_1_col1_key +Index "mx_test_schema_1.mx_table_1_col1_key" + Column | Type | Definition +--------+---------+------------ + col1 | integer | col1 +unique, btree, for table "mx_test_schema_1.mx_table_1" + +\d mx_test_schema_1.mx_index_1 +Index "mx_test_schema_1.mx_index_1" + Column | Type | Definition +--------+---------+------------ + col1 | integer | col1 +btree, for table "mx_test_schema_1.mx_table_1" + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_2.mx_table_2'::regclass; Column | Type | Modifiers --------+---------+----------- col1 | integer | col2 | text | -Indexes: - "mx_index_2" btree (col2) -Foreign-key constraints: - "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +(2 rows) + +\d mx_test_schema_2.mx_index_2 +Index "mx_test_schema_2.mx_index_2" + Column | Type | Definition +--------+------+------------ + col2 | text | col2 +btree, for table "mx_test_schema_2.mx_table_2" + +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_2.mx_table_2'::regclass; + Constraint | Definition +------------------+----------------------------------------------------------------- + mx_fk_constraint | FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +(1 row) SELECT create_distributed_table('mx_test_schema_1.mx_table_1', 'col1'); create_distributed_table @@ -578,28 +614,13 @@ ORDER BY -- Check that metadata of MX tables exist on the metadata worker \c - - - :worker_1_port -- Check that tables are created -\d mx_test_schema_1.mx_table_1 -Table "mx_test_schema_1.mx_table_1" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | text | -Indexes: - "mx_table_1_col1_key" UNIQUE CONSTRAINT, btree (col1) - "mx_index_1" btree (col1) -Referenced by: - TABLE "mx_test_schema_2.mx_table_2" CONSTRAINT "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) - -\d mx_test_schema_2.mx_table_2 -Table "mx_test_schema_2.mx_table_2" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | text | -Indexes: - "mx_index_2" btree (col2) -Foreign-key constraints: - "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +\dt mx_test_schema_?.mx_table_? + List of relations + Schema | Name | Type | Owner +------------------+------------+-------+---------- + mx_test_schema_1 | mx_table_1 | table | postgres + mx_test_schema_2 | mx_table_2 | table | postgres +(2 rows) -- Check that table metadata are created SELECT @@ -663,38 +684,28 @@ SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; SET citus.multi_shard_commit_protocol TO '2pc'; SET client_min_messages TO 'ERROR'; CREATE INDEX mx_index_3 ON mx_test_schema_2.mx_table_2 USING hash (col1); -CREATE UNIQUE INDEX mx_index_4 ON mx_test_schema_2.mx_table_2(col1); +ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_table_2_col1_key UNIQUE (col1); \c - - - :worker_1_port -\d mx_test_schema_2.mx_table_2 -Table "mx_test_schema_2.mx_table_2" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | text | -Indexes: - "mx_index_4" UNIQUE, btree (col1) - "mx_index_2" btree (col2) - "mx_index_3" hash (col1) -Foreign-key constraints: - "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +\d mx_test_schema_2.mx_index_3 +Index "mx_test_schema_2.mx_index_3" + Column | Type | Definition +--------+---------+------------ + col1 | integer | col1 +hash, for table "mx_test_schema_2.mx_table_2" + +\d mx_test_schema_2.mx_table_2_col1_key +Index "mx_test_schema_2.mx_table_2_col1_key" + Column | Type | Definition +--------+---------+------------ + col1 | integer | col1 +unique, btree, for table "mx_test_schema_2.mx_table_2" -- Check that DROP INDEX statement is propagated \c - - - :master_port SET citus.multi_shard_commit_protocol TO '2pc'; DROP INDEX mx_test_schema_2.mx_index_3; \c - - - :worker_1_port -\d mx_test_schema_2.mx_table_2 -Table "mx_test_schema_2.mx_table_2" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | text | -Indexes: - "mx_index_4" UNIQUE, btree (col1) - "mx_index_2" btree (col2) -Foreign-key constraints: - "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) - +\d mx_test_schema_2.mx_index_3 -- Check that ALTER TABLE statements are propagated \c - - - :master_port SET citus.multi_shard_commit_protocol TO '2pc'; @@ -709,20 +720,19 @@ FOREIGN KEY REFERENCES mx_test_schema_2.mx_table_2(col1); \c - - - :worker_1_port -\d mx_test_schema_1.mx_table_1 -Table "mx_test_schema_1.mx_table_1" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; Column | Type | Modifiers --------+---------+----------- col1 | integer | col2 | text | col3 | integer | -Indexes: - "mx_table_1_col1_key" UNIQUE CONSTRAINT, btree (col1) - "mx_index_1" btree (col1) -Foreign-key constraints: - "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) -Referenced by: - TABLE "mx_test_schema_2.mx_table_2" CONSTRAINT "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +(3 rows) + +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; + Constraint | Definition +------------------+----------------------------------------------------------------- + mx_fk_constraint | FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) +(1 row) -- Check that foreign key constraint with NOT VALID works as well \c - - - :master_port @@ -738,20 +748,11 @@ REFERENCES mx_test_schema_2.mx_table_2(col1) NOT VALID; \c - - - :worker_1_port -\d mx_test_schema_1.mx_table_1 -Table "mx_test_schema_1.mx_table_1" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | - col2 | text | - col3 | integer | -Indexes: - "mx_table_1_col1_key" UNIQUE CONSTRAINT, btree (col1) - "mx_index_1" btree (col1) -Foreign-key constraints: - "mx_fk_constraint_2" FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) NOT VALID -Referenced by: - TABLE "mx_test_schema_2.mx_table_2" CONSTRAINT "mx_fk_constraint" FOREIGN KEY (col1) REFERENCES mx_test_schema_1.mx_table_1(col1) +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; + Constraint | Definition +--------------------+----------------------------------------------------------------- + mx_fk_constraint_2 | FOREIGN KEY (col1) REFERENCES mx_test_schema_2.mx_table_2(col1) +(1 row) -- Check that mark_tables_colocated call propagates the changes to the workers \c - - - :master_port @@ -932,13 +933,13 @@ SELECT create_distributed_table('mx_table_with_sequence', 'a'); (1 row) -\d mx_table_with_sequence - Table "public.mx_table_with_sequence" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; Column | Type | Modifiers --------+---------+-------------------------------------------------------------------- a | integer | b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) +(3 rows) \ds mx_table_with_sequence_b_seq List of relations @@ -956,13 +957,13 @@ SELECT create_distributed_table('mx_table_with_sequence', 'a'); -- Check that the sequences created on the metadata worker as well \c - - - :worker_1_port -\d mx_table_with_sequence - Table "public.mx_table_with_sequence" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; Column | Type | Modifiers --------+---------+-------------------------------------------------------------------- a | integer | b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) +(3 rows) \ds mx_table_with_sequence_b_seq List of relations @@ -1006,13 +1007,13 @@ SELECT groupid FROM pg_dist_local_group; 2 (1 row) -\d mx_table_with_sequence - Table "public.mx_table_with_sequence" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; Column | Type | Modifiers --------+---------+-------------------------------------------------------------------- a | integer | b | bigint | not null default nextval('mx_table_with_sequence_b_seq'::regclass) c | bigint | not null default nextval('mx_table_with_sequence_c_seq'::regclass) +(3 rows) \ds mx_table_with_sequence_b_seq List of relations @@ -1204,20 +1205,20 @@ SELECT create_reference_table('mx_ref'); (1 row) -\d mx_ref - Table "public.mx_ref" - Column | Type | Modifiers ---------+---------+----------- - col_1 | integer | - col_2 | text | +\dt mx_ref + List of relations + Schema | Name | Type | Owner +--------+--------+-------+---------- + public | mx_ref | table | postgres +(1 row) \c - - - :worker_1_port -\d mx_ref - Table "public.mx_ref" - Column | Type | Modifiers ---------+---------+----------- - col_1 | integer | - col_2 | text | +\dt mx_ref + List of relations + Schema | Name | Type | Owner +--------+--------+-------+---------- + public | mx_ref | table | postgres +(1 row) SELECT logicalrelid, partmethod, repmodel, shardid, placementid, nodename, nodeport @@ -1243,26 +1244,36 @@ ALTER TABLE mx_ref ADD COLUMN col_3 NUMERIC DEFAULT 0; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' CREATE INDEX mx_ref_index ON mx_ref(col_1); -\d mx_ref - Table "public.mx_ref" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; Column | Type | Modifiers --------+---------+----------- col_1 | integer | col_2 | text | col_3 | numeric | default 0 -Indexes: - "mx_ref_index" btree (col_1) +(3 rows) + +\d mx_ref_index + Index "public.mx_ref_index" + Column | Type | Definition +--------+---------+------------ + col_1 | integer | col_1 +btree, for table "public.mx_ref" \c - - - :worker_1_port -\d mx_ref - Table "public.mx_ref" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; Column | Type | Modifiers --------+---------+----------- col_1 | integer | col_2 | text | col_3 | numeric | default 0 -Indexes: - "mx_ref_index" btree (col_1) +(3 rows) + +\d mx_ref_index + Index "public.mx_ref_index" + Column | Type | Definition +--------+---------+------------ + col_1 | integer | col_1 +btree, for table "public.mx_ref" -- Check that metada is cleaned successfully upon drop table diff --git a/src/test/regress/expected/multi_modifying_xacts.out b/src/test/regress/expected/multi_modifying_xacts.out index f4d953c66..a03010787 100644 --- a/src/test/regress/expected/multi_modifying_xacts.out +++ b/src/test/regress/expected/multi_modifying_xacts.out @@ -185,12 +185,12 @@ INSERT INTO labs VALUES (6, 'Bell Labs'); ERROR: single-shard DML commands must not appear in transaction blocks which contain multi-shard data modifications COMMIT; -- but the DDL should correctly roll back -\d labs - Table "public.labs" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.labs'::regclass; Column | Type | Modifiers --------+--------+----------- id | bigint | not null name | text | not null +(2 rows) SELECT * FROM labs WHERE id = 6; id | name diff --git a/src/test/regress/expected/multi_mx_ddl.out b/src/test/regress/expected/multi_mx_ddl.out index 1f347e16a..9ea180d98 100644 --- a/src/test/regress/expected/multi_mx_ddl.out +++ b/src/test/regress/expected/multi_mx_ddl.out @@ -30,67 +30,112 @@ SELECT master_modify_multiple_shards('UPDATE mx_ddl_table SET version=0.1 WHERE -- SET NOT NULL ALTER TABLE mx_ddl_table ALTER COLUMN version SET NOT NULL; -- See that the changes are applied on coordinator, worker tables and shards -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers ---------+---------+-------------------- key | integer | not null value | integer | version | integer | not null default 1 -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) - "ddl_test_concurrent_index" btree (value) - "ddl_test_index" btree (value) +(3 rows) + +\d ddl_test*_index +Index "public.ddl_test_concurrent_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" + + Index "public.ddl_test_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" \c - - - :worker_1_port -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers ---------+---------+-------------------- key | integer | not null value | integer | version | integer | not null default 1 -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) - "ddl_test_concurrent_index" btree (value) - "ddl_test_index" btree (value) +(3 rows) -\d mx_ddl_table_1220088 - Table "public.mx_ddl_table_1220088" +\d ddl_test*_index +Index "public.ddl_test_concurrent_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" + + Index "public.ddl_test_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220088'::regclass; Column | Type | Modifiers ---------+---------+-------------------- key | integer | not null value | integer | version | integer | not null default 1 -Indexes: - "mx_ddl_table_pkey_1220088" PRIMARY KEY, btree (key) - "ddl_test_concurrent_index_1220088" btree (value) - "ddl_test_index_1220088" btree (value) +(3 rows) + +\d ddl_test*_index_1220088 +Index "public.ddl_test_concurrent_index_1220088" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table_1220088" + +Index "public.ddl_test_index_1220088" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table_1220088" \c - - - :worker_2_port -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers ---------+---------+-------------------- key | integer | not null value | integer | version | integer | not null default 1 -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) - "ddl_test_concurrent_index" btree (value) - "ddl_test_index" btree (value) +(3 rows) -\d mx_ddl_table_1220089 - Table "public.mx_ddl_table_1220089" +\d ddl_test*_index +Index "public.ddl_test_concurrent_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" + + Index "public.ddl_test_index" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table" + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220089'::regclass; Column | Type | Modifiers ---------+---------+-------------------- key | integer | not null value | integer | version | integer | not null default 1 -Indexes: - "mx_ddl_table_pkey_1220089" PRIMARY KEY, btree (key) - "ddl_test_concurrent_index_1220089" btree (value) - "ddl_test_index_1220089" btree (value) +(3 rows) + +\d ddl_test*_index_1220089 +Index "public.ddl_test_concurrent_index_1220089" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table_1220089" + +Index "public.ddl_test_index_1220089" + Column | Type | Definition +--------+---------+------------ + value | integer | value +btree, for table "public.mx_ddl_table_1220089" INSERT INTO mx_ddl_table VALUES (37, 78, 2); INSERT INTO mx_ddl_table VALUES (38, 78); @@ -132,52 +177,72 @@ ALTER TABLE mx_ddl_table ALTER COLUMN version DROP NOT NULL; -- DROP COLUMN ALTER TABLE mx_ddl_table DROP COLUMN version; -- See that the changes are applied on coordinator, worker tables and shards -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers --------+---------+----------- key | integer | not null value | integer | -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) +(2 rows) + +\di ddl_test*_index + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) \c - - - :worker_1_port -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers --------+---------+----------- key | integer | not null value | integer | -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) +(2 rows) -\d mx_ddl_table_1220088 -Table "public.mx_ddl_table_1220088" +\di ddl_test*_index + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220088'::regclass; Column | Type | Modifiers --------+---------+----------- key | integer | not null value | integer | -Indexes: - "mx_ddl_table_pkey_1220088" PRIMARY KEY, btree (key) +(2 rows) + +\di ddl_test*_index_1220088 + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) \c - - - :worker_2_port -\d mx_ddl_table - Table "public.mx_ddl_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; Column | Type | Modifiers --------+---------+----------- key | integer | not null value | integer | -Indexes: - "mx_ddl_table_pkey" PRIMARY KEY, btree (key) +(2 rows) -\d mx_ddl_table_1220089 -Table "public.mx_ddl_table_1220089" +\di ddl_test*_index + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220089'::regclass; Column | Type | Modifiers --------+---------+----------- key | integer | not null value | integer | -Indexes: - "mx_ddl_table_pkey_1220089" PRIMARY KEY, btree (key) +(2 rows) + +\di ddl_test*_index_1220089 + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) -- Show that DDL commands are done within a two-phase commit transaction \c - - - :master_port diff --git a/src/test/regress/expected/multi_mx_metadata.out b/src/test/regress/expected/multi_mx_metadata.out index 5798cb557..5b990d03e 100644 --- a/src/test/regress/expected/multi_mx_metadata.out +++ b/src/test/regress/expected/multi_mx_metadata.out @@ -43,15 +43,26 @@ SELECT count(*) FROM pg_dist_transaction; (1 row) \c - - - :worker_1_port -\d distributed_mx_table -Table "public.distributed_mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='distributed_mx_table'::regclass; Column | Type | Modifiers --------+-------+----------- key | text | not null value | jsonb | -Indexes: - "distributed_mx_table_pkey" PRIMARY KEY, btree (key) - "distributed_mx_table_value_idx" gin (value) +(2 rows) + +\d distributed_mx_table_pkey +Index "public.distributed_mx_table_pkey" + Column | Type | Definition +--------+------+------------ + key | text | key +primary key, btree, for table "public.distributed_mx_table" + +\d distributed_mx_table_value_idx +Index "public.distributed_mx_table_value_idx" + Column | Type | Definition +--------+------+------------ + value | text | value +gin, for table "public.distributed_mx_table" SELECT repmodel FROM pg_dist_partition WHERE logicalrelid = 'distributed_mx_table'::regclass; @@ -68,15 +79,26 @@ WHERE logicalrelid = 'distributed_mx_table'::regclass; (1 row) \c - - - :worker_2_port -\d distributed_mx_table -Table "public.distributed_mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='distributed_mx_table'::regclass; Column | Type | Modifiers --------+-------+----------- key | text | not null value | jsonb | -Indexes: - "distributed_mx_table_pkey" PRIMARY KEY, btree (key) - "distributed_mx_table_value_idx" gin (value) +(2 rows) + +\d distributed_mx_table_pkey +Index "public.distributed_mx_table_pkey" + Column | Type | Definition +--------+------+------------ + key | text | key +primary key, btree, for table "public.distributed_mx_table" + +\d distributed_mx_table_value_idx +Index "public.distributed_mx_table_value_idx" + Column | Type | Definition +--------+------+------------ + value | text | value +gin, for table "public.distributed_mx_table" SELECT repmodel FROM pg_dist_partition WHERE logicalrelid = 'distributed_mx_table'::regclass; diff --git a/src/test/regress/expected/multi_name_lengths.out b/src/test/regress/expected/multi_name_lengths.out index 4dc78c236..e6fd08504 100644 --- a/src/test/regress/expected/multi_name_lengths.out +++ b/src/test/regress/expected/multi_name_lengths.out @@ -20,18 +20,13 @@ SELECT master_create_worker_shards('too_long_12345678901234567890123456789012345 (1 row) \c - - - :worker_1_port -\d too_long_* -Table "public.too_long_12345678901234567890123456789012345678_e0119164_225000" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null - -Table "public.too_long_12345678901234567890123456789012345678_e0119164_225001" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null +\dt too_long_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------------------------------------+-------+---------- + public | too_long_12345678901234567890123456789012345678_e0119164_225000 | table | postgres + public | too_long_12345678901234567890123456789012345678_e0119164_225001 | table | postgres +(2 rows) \c - - - :master_port -- Verify that the UDF works and rejects bad arguments. @@ -83,8 +78,7 @@ ERROR: cannot create constraint without a name on a distributed table ALTER TABLE name_lengths ADD CHECK (date_col_12345678901234567890123456789012345678901234567890 > '2014-01-01'::date); ERROR: cannot create constraint without a name on a distributed table \c - - - :worker_1_port -\d name_lengths_* - Table "public.name_lengths_225002" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.name_lengths_225002'::regclass; Column | Type | Modifiers --------------------------------------------------------------+------------------+----------- col1 | integer | not null @@ -92,19 +86,7 @@ ERROR: cannot create constraint without a name on a distributed table float_col_12345678901234567890123456789012345678901234567890 | double precision | date_col_12345678901234567890123456789012345678901234567890 | date | int_col_12345678901234567890123456789012345678901234567890 | integer | default 1 -Indexes: - "constraint_a_225002" UNIQUE CONSTRAINT, btree (col1) - - Table "public.name_lengths_225003" - Column | Type | Modifiers ---------------------------------------------------------------+------------------+----------- - col1 | integer | not null - col2 | integer | not null - float_col_12345678901234567890123456789012345678901234567890 | double precision | - date_col_12345678901234567890123456789012345678901234567890 | date | - int_col_12345678901234567890123456789012345678901234567890 | integer | default 1 -Indexes: - "constraint_a_225003" UNIQUE CONSTRAINT, btree (col1) +(5 rows) \c - - - :master_port -- Placeholders for unsupported add constraints with EXPLICIT names that are too long @@ -118,7 +100,12 @@ ALTER TABLE name_lengths ADD CONSTRAINT nl_checky_123456789012345678901234567890 NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' \c - - - :worker_1_port -\d nl_* +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.name_lengths_225002'::regclass; + Constraint | Definition +-----------------------------------------------------------------+------------------------------------------------------------------------------------------- + nl_checky_1234567890123456789012345678901234567_b16df46d_225002 | CHECK (date_col_12345678901234567890123456789012345678901234567890 >= '01-01-2014'::date) +(1 row) + \c - - - :master_port -- Placeholders for RENAME operations ALTER TABLE name_lengths RENAME TO name_len_12345678901234567890123456789012345678901234567890; @@ -203,22 +190,18 @@ CREATE TABLE sneaky_name_lengths ( col2 integer not null, CONSTRAINT checky_12345678901234567890123456789012345678901234567890 CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) ); -\d sneaky_name_lengths* - Table "public.sneaky_name_lengths" - Column | Type | Modifiers -----------------------------------------------------------------+---------+----------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | not null - col2 | integer | not null -Indexes: - "sneaky_name_lengths_int_col_1234567890123456789012345678901_key" UNIQUE CONSTRAINT, btree (int_col_123456789012345678901234567890123456789012345678901234) -Check constraints: - "checky_12345678901234567890123456789012345678901234567890" CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) +\di public.sneaky_name_lengths* + List of relations + Schema | Name | Type | Owner | Table +--------+-----------------------------------------------------------------+-------+----------+--------------------- + public | sneaky_name_lengths_int_col_1234567890123456789012345678901_key | index | postgres | sneaky_name_lengths +(1 row) - Index "public.sneaky_name_lengths_int_col_1234567890123456789012345678901_key" - Column | Type | Definition -----------------------------------------------------------------+---------+---------------------------------------------------------------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | int_col_123456789012345678901234567890123456789012345678901234 -unique, btree, for table "public.sneaky_name_lengths" +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths'::regclass; + Constraint | Definition +-----------------------------------------------------------+------------------------------------------------------------------------------ + checky_12345678901234567890123456789012345678901234567890 | CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) +(1 row) SELECT master_create_distributed_table('sneaky_name_lengths', 'int_col_123456789012345678901234567890123456789012345678901234', 'hash'); master_create_distributed_table @@ -233,38 +216,18 @@ SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2'); (1 row) \c - - - :worker_1_port -\d sneaky_name_lengths* - Table "public.sneaky_name_lengths_225006" - Column | Type | Modifiers -----------------------------------------------------------------+---------+----------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | not null - col2 | integer | not null -Indexes: - "sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225006" UNIQUE CONSTRAINT, btree (int_col_123456789012345678901234567890123456789012345678901234) -Check constraints: - "checky_12345678901234567890123456789012345678901234567890" CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) +\di public.sneaky*225006 + List of relations + Schema | Name | Type | Owner | Table +--------+-----------------------------------------------------------------+-------+----------+---------------------------- + public | sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225006 | index | postgres | sneaky_name_lengths_225006 +(1 row) - Table "public.sneaky_name_lengths_225007" - Column | Type | Modifiers -----------------------------------------------------------------+---------+----------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | not null - col2 | integer | not null -Indexes: - "sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225007" UNIQUE CONSTRAINT, btree (int_col_123456789012345678901234567890123456789012345678901234) -Check constraints: - "checky_12345678901234567890123456789012345678901234567890" CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) - - Index "public.sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225006" - Column | Type | Definition -----------------------------------------------------------------+---------+---------------------------------------------------------------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | int_col_123456789012345678901234567890123456789012345678901234 -unique, btree, for table "public.sneaky_name_lengths_225006" - - Index "public.sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225007" - Column | Type | Definition -----------------------------------------------------------------+---------+---------------------------------------------------------------- - int_col_123456789012345678901234567890123456789012345678901234 | integer | int_col_123456789012345678901234567890123456789012345678901234 -unique, btree, for table "public.sneaky_name_lengths_225007" +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths_225006'::regclass; + Constraint | Definition +-----------------------------------------------------------+------------------------------------------------------------------------------ + checky_12345678901234567890123456789012345678901234567890 | CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) +(1 row) \c - - - :master_port DROP TABLE sneaky_name_lengths CASCADE; @@ -288,24 +251,12 @@ SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2'); (1 row) \c - - - :worker_1_port -\d sneaky_name_lengths* - Table "public.sneaky_name_lengths_225008" - Column | Type | Modifiers -------------------------------------------------------------+---------+----------- - col1 | integer | not null - col2 | integer | not null - int_col_12345678901234567890123456789012345678901234567890 | integer | not null -Indexes: - "unique_1234567890123456789012345678901234567890_a5986f27_225008" UNIQUE CONSTRAINT, btree (col1) - - Table "public.sneaky_name_lengths_225009" - Column | Type | Modifiers -------------------------------------------------------------+---------+----------- - col1 | integer | not null - col2 | integer | not null - int_col_12345678901234567890123456789012345678901234567890 | integer | not null -Indexes: - "unique_1234567890123456789012345678901234567890_a5986f27_225009" UNIQUE CONSTRAINT, btree (col1) +\di unique*225008 + List of relations + Schema | Name | Type | Owner | Table +--------+-----------------------------------------------------------------+-------+----------+---------------------------- + public | unique_1234567890123456789012345678901234567890_a5986f27_225008 | index | postgres | sneaky_name_lengths_225008 +(1 row) \c - - - :master_port DROP TABLE sneaky_name_lengths CASCADE; @@ -327,18 +278,13 @@ SELECT master_create_worker_shards('too_long_12345678901234567890123456789012345 (1 row) \c - - - :worker_1_port -\d too_long_* -Table "public.too_long_1234567890123456789012345678901_e0119164_2250000000000" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null - -Table "public.too_long_1234567890123456789012345678901_e0119164_2250000000001" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null +\dt *225000000000* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------------------------------------+-------+---------- + public | too_long_1234567890123456789012345678901_e0119164_2250000000000 | table | postgres + public | too_long_1234567890123456789012345678901_e0119164_2250000000001 | table | postgres +(2 rows) \c - - - :master_port DROP TABLE too_long_12345678901234567890123456789012345678901234567890 CASCADE; @@ -359,34 +305,21 @@ SELECT master_create_worker_shards(U&'elephant_!0441!043B!043E!043D!0441!043B!04 (1 row) \c - - - :worker_1_port -\d elephant_* -Index "public.elephant_слонслонслонсло_14d34928_2250000000002" - Column | Type | Definition ---------+---------+------------ - col1 | integer | col1 -primary key, btree, for table "public.elephant_слонслонслонсло_c8b737c2_2250000000002" +\dt public.elephant_* + List of relations + Schema | Name | Type | Owner +--------+-------------------------------------------------+-------+---------- + public | elephant_слонслонслонсло_c8b737c2_2250000000002 | table | postgres + public | elephant_слонслонслонсло_c8b737c2_2250000000003 | table | postgres +(2 rows) -Index "public.elephant_слонслонслонсло_14d34928_2250000000003" - Column | Type | Definition ---------+---------+------------ - col1 | integer | col1 -primary key, btree, for table "public.elephant_слонслонслонсло_c8b737c2_2250000000003" - -Table "public.elephant_слонслонслонсло_c8b737c2_2250000000002" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null -Indexes: - "elephant_слонслонслонсло_14d34928_2250000000002" PRIMARY KEY, btree (col1) - -Table "public.elephant_слонслонслонсло_c8b737c2_2250000000003" - Column | Type | Modifiers ---------+---------+----------- - col1 | integer | not null - col2 | integer | not null -Indexes: - "elephant_слонслонслонсло_14d34928_2250000000003" PRIMARY KEY, btree (col1) +\di public.elephant_* + List of relations + Schema | Name | Type | Owner | Table +--------+-------------------------------------------------+-------+----------+------------------------------------------------- + public | elephant_слонслонслонсло_14d34928_2250000000002 | index | postgres | elephant_слонслонслонсло_c8b737c2_2250000000002 + public | elephant_слонслонслонсло_14d34928_2250000000003 | index | postgres | elephant_слонслонслонсло_c8b737c2_2250000000003 +(2 rows) \c - - - :master_port -- Clean up. diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning.out b/src/test/regress/expected/multi_null_minmax_value_pruning.out index fd5491287..8abddb3d3 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning.out @@ -4,6 +4,14 @@ -- This test checks that we can handle null min/max values in shard statistics -- and that we don't partition or join prune shards that have null values. ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 760000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 10 +(1 row) + SET client_min_messages TO DEBUG2; SET citus.explain_all_tasks TO on; -- to avoid differing explain output - executor doesn't matter, @@ -73,9 +81,9 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate @@ -87,44 +95,44 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem - -> Task - Node: host=localhost port=57638 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders (60 rows) -- Now set the minimum value for a shard to null. Then check that we don't apply @@ -167,9 +175,9 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate @@ -181,51 +189,51 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem - -> Task - Node: host=localhost port=57638 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders - -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - -> Task - Node: host=localhost port=57637 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders - -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem - -> Task - Node: host=localhost port=57638 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) -> Index Only Scan using orders_pkey_290008 on orders_290008 orders - -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem - -> Task - Node: host=localhost port=57637 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders - -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem -> Index Only Scan using orders_pkey_290009 on orders_290009 orders - -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders (67 rows) -- Next, set the maximum value for another shard to null. Then check that we @@ -271,9 +279,9 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate @@ -285,58 +293,58 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem - -> Task - Node: host=localhost port=57638 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders (74 rows) -- Last, set the minimum value to 0 and check that we don't treat it as null. We @@ -379,9 +387,9 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate @@ -393,51 +401,51 @@ DEBUG: join prunable for intervals [13473,14947] and [1,5986] Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem - -> Task - Node: host=localhost port=57637 dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57638 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders -> Task Node: host=localhost port=57637 dbname=regression -> Aggregate -> Merge Join - Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) - -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders (67 rows) -- Set minimum and maximum values for two shards back to their original values diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning_0.out b/src/test/regress/expected/multi_null_minmax_value_pruning_0.out new file mode 100644 index 000000000..fd1ba6dc3 --- /dev/null +++ b/src/test/regress/expected/multi_null_minmax_value_pruning_0.out @@ -0,0 +1,454 @@ +-- +-- MULTI_NULL_MINMAX_VALUE_PRUNING +-- +-- This test checks that we can handle null min/max values in shard statistics +-- and that we don't partition or join prune shards that have null values. +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 760000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 9 +(1 row) + +SET client_min_messages TO DEBUG2; +SET citus.explain_all_tasks TO on; +-- to avoid differing explain output - executor doesn't matter, +-- because were testing pruning here. +SET citus.task_executor_type TO 'real-time'; +-- Change configuration to treat lineitem and orders tables as large +SET citus.large_table_shard_count TO 2; +SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290000; + shardminvalue | shardmaxvalue +---------------+--------------- + 1 | 1509 +(1 row) + +SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290001; + shardminvalue | shardmaxvalue +---------------+--------------- + 1509 | 2951 +(1 row) + +-- Check that partition and join pruning works when min/max values exist +-- Adding l_orderkey = 1 to make the query not router executable +EXPLAIN (COSTS FALSE) +SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030 or l_orderkey = 1; + QUERY PLAN +----------------------------------------------------------------------- + Custom Scan (Citus Real-Time) + Task Count: 2 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Bitmap Heap Scan on lineitem_290000 lineitem + Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) + -> BitmapOr + -> Bitmap Index Scan on lineitem_pkey_290000 + Index Cond: (l_orderkey = 9030) + -> Bitmap Index Scan on lineitem_pkey_290000 + Index Cond: (l_orderkey = 1) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Bitmap Heap Scan on lineitem_290004 lineitem + Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) + -> BitmapOr + -> Bitmap Index Scan on lineitem_pkey_290004 + Index Cond: (l_orderkey = 9030) + -> Bitmap Index Scan on lineitem_pkey_290004 + Index Cond: (l_orderkey = 1) +(21 rows) + +EXPLAIN (COSTS FALSE) +SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders + WHERE l_orderkey = o_orderkey; +DEBUG: join prunable for intervals [1,1509] and [8997,14946] +DEBUG: join prunable for intervals [1509,2951] and [8997,14946] +DEBUG: join prunable for intervals [2951,4455] and [8997,14946] +DEBUG: join prunable for intervals [4480,5986] and [8997,14946] +DEBUG: join prunable for intervals [8997,10560] and [1,5986] +DEBUG: join prunable for intervals [10560,12036] and [1,5986] +DEBUG: join prunable for intervals [12036,13473] and [1,5986] +DEBUG: join prunable for intervals [13473,14947] and [1,5986] + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 8 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem +(60 rows) + +-- Now set the minimum value for a shard to null. Then check that we don't apply +-- partition or join pruning for the shard with null min value. +UPDATE pg_dist_shard SET shardminvalue = NULL WHERE shardid = 290000; +EXPLAIN (COSTS FALSE) +SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; + QUERY PLAN +------------------------------------------------------------------------------- + Custom Scan (Citus Real-Time) + Task Count: 2 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + Index Cond: (l_orderkey = 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Index Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + Index Cond: (l_orderkey = 9030) +(11 rows) + +EXPLAIN (COSTS FALSE) +SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders + WHERE l_orderkey = o_orderkey; +DEBUG: join prunable for intervals [1509,2951] and [8997,14946] +DEBUG: join prunable for intervals [2951,4455] and [8997,14946] +DEBUG: join prunable for intervals [4480,5986] and [8997,14946] +DEBUG: join prunable for intervals [8997,10560] and [1,5986] +DEBUG: join prunable for intervals [10560,12036] and [1,5986] +DEBUG: join prunable for intervals [12036,13473] and [1,5986] +DEBUG: join prunable for intervals [13473,14947] and [1,5986] + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 9 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem +(67 rows) + +-- Next, set the maximum value for another shard to null. Then check that we +-- don't apply partition or join pruning for this other shard either. +UPDATE pg_dist_shard SET shardmaxvalue = NULL WHERE shardid = 290001; +EXPLAIN (COSTS FALSE) +SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; + QUERY PLAN +------------------------------------------------------------------------------- + Custom Scan (Citus Real-Time) + Task Count: 3 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + Index Cond: (l_orderkey = 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + Index Cond: (l_orderkey = 9030) + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + Index Cond: (l_orderkey = 9030) +(15 rows) + +EXPLAIN (COSTS FALSE) +SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders + WHERE l_orderkey = o_orderkey; +DEBUG: join prunable for intervals [2951,4455] and [8997,14946] +DEBUG: join prunable for intervals [4480,5986] and [8997,14946] +DEBUG: join prunable for intervals [8997,10560] and [1,5986] +DEBUG: join prunable for intervals [10560,12036] and [1,5986] +DEBUG: join prunable for intervals [12036,13473] and [1,5986] +DEBUG: join prunable for intervals [13473,14947] and [1,5986] + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 10 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem +(74 rows) + +-- Last, set the minimum value to 0 and check that we don't treat it as null. We +-- should apply partition and join pruning for this shard now. +UPDATE pg_dist_shard SET shardminvalue = '0' WHERE shardid = 290000; +EXPLAIN (COSTS FALSE) +SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; + QUERY PLAN +------------------------------------------------------------------------------- + Custom Scan (Citus Real-Time) + Task Count: 2 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + Index Cond: (l_orderkey = 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Index Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + Index Cond: (l_orderkey = 9030) +(11 rows) + +EXPLAIN (COSTS FALSE) +SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders + WHERE l_orderkey = o_orderkey; +DEBUG: join prunable for intervals [0,1509] and [8997,14946] +DEBUG: join prunable for intervals [2951,4455] and [8997,14946] +DEBUG: join prunable for intervals [4480,5986] and [8997,14946] +DEBUG: join prunable for intervals [8997,10560] and [1,5986] +DEBUG: join prunable for intervals [10560,12036] and [1,5986] +DEBUG: join prunable for intervals [12036,13473] and [1,5986] +DEBUG: join prunable for intervals [13473,14947] and [1,5986] + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Aggregate + -> Custom Scan (Citus Real-Time) + Task Count: 9 + Tasks Shown: All + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (lineitem.l_orderkey = orders.o_orderkey) + -> Index Only Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290001 on lineitem_290001 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290002 on lineitem_290002 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290008 on orders_290008 orders + -> Index Only Scan using lineitem_pkey_290003 on lineitem_290003 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290004 on lineitem_290004 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290005 on lineitem_290005 lineitem + -> Task + Node: host=localhost port=57638 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290006 on lineitem_290006 lineitem + -> Task + Node: host=localhost port=57637 dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (orders.o_orderkey = lineitem.l_orderkey) + -> Index Only Scan using orders_pkey_290009 on orders_290009 orders + -> Index Only Scan using lineitem_pkey_290007 on lineitem_290007 lineitem +(67 rows) + +-- Set minimum and maximum values for two shards back to their original values +UPDATE pg_dist_shard SET shardminvalue = '1' WHERE shardid = 290000; +UPDATE pg_dist_shard SET shardmaxvalue = '4964' WHERE shardid = 290001; +SET client_min_messages TO NOTICE; diff --git a/src/test/regress/expected/multi_reference_table.out b/src/test/regress/expected/multi_reference_table.out index 855d96a0b..e54e4428e 100644 --- a/src/test/regress/expected/multi_reference_table.out +++ b/src/test/regress/expected/multi_reference_table.out @@ -1331,43 +1331,61 @@ ALTER TABLE reference_table_ddl DROP COLUMN value_1; ALTER TABLE reference_table_ddl ALTER COLUMN value_2 SET DEFAULT 25.0; ALTER TABLE reference_table_ddl ALTER COLUMN value_3 SET NOT NULL; -- see that Citus applied all DDLs to the table -\d reference_table_ddl - Table "public.reference_table_ddl" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl'::regclass; Column | Type | Modifiers ---------+-----------------------------+-------------- value_2 | double precision | default 25.0 value_3 | text | not null value_4 | timestamp without time zone | value_5 | double precision | -Indexes: - "reference_index_2" btree (value_2, value_3) +(4 rows) + +\d reference_index_2 + Index "public.reference_index_2" + Column | Type | Definition +---------+------------------+------------ + value_2 | double precision | value_2 + value_3 | text | value_3 +btree, for table "public.reference_table_ddl" -- also to the shard placements \c - - - :worker_1_port -\d reference_table_ddl* - Table "public.reference_table_ddl_1250019" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl_1250019'::regclass; Column | Type | Modifiers ---------+-----------------------------+-------------- value_2 | double precision | default 25.0 value_3 | text | not null value_4 | timestamp without time zone | value_5 | double precision | -Indexes: - "reference_index_2_1250019" btree (value_2, value_3) +(4 rows) + +\d reference_index_2_1250019 +Index "public.reference_index_2_1250019" + Column | Type | Definition +---------+------------------+------------ + value_2 | double precision | value_2 + value_3 | text | value_3 +btree, for table "public.reference_table_ddl_1250019" \c - - - :master_port DROP INDEX reference_index_2; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' \c - - - :worker_1_port -\d reference_table_ddl* - Table "public.reference_table_ddl_1250019" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl_1250019'::regclass; Column | Type | Modifiers ---------+-----------------------------+-------------- value_2 | double precision | default 25.0 value_3 | text | not null value_4 | timestamp without time zone | value_5 | double precision | +(4 rows) + +\di reference_index_2* + List of relations + Schema | Name | Type | Owner | Table +--------+------+------+-------+------- +(0 rows) \c - - - :master_port -- as we expect, renaming and setting WITH OIDS does not work for reference tables diff --git a/src/test/regress/expected/multi_remove_node_reference_table.out b/src/test/regress/expected/multi_remove_node_reference_table.out index 477e6a880..55cd66e0a 100644 --- a/src/test/regress/expected/multi_remove_node_reference_table.out +++ b/src/test/regress/expected/multi_remove_node_reference_table.out @@ -631,12 +631,12 @@ WHERE \c - - - :master_port -- verify table structure is changed -\d remove_node_reference_table -Table "public.remove_node_reference_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.remove_node_reference_table'::regclass; Column | Type | Modifiers ---------+---------+----------- column1 | integer | column2 | integer | +(2 rows) -- re-add the node for next tests SELECT master_add_node('localhost', :worker_2_port); diff --git a/src/test/regress/expected/multi_schema_support.out b/src/test/regress/expected/multi_schema_support.out index 44dd95066..51c5bb76e 100644 --- a/src/test/regress/expected/multi_schema_support.out +++ b/src/test/regress/expected/multi_schema_support.out @@ -586,8 +586,7 @@ ALTER TABLE test_schema_support.nation_hash ADD COLUMN new_col INT; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' -- verify column is added -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null @@ -595,10 +594,10 @@ HINT: You can enable two-phase commit for extra safety with: SET citus.multi_sh n_regionkey | integer | not null n_comment | character varying(152) | new_col | integer | +(5 rows) \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null @@ -606,6 +605,7 @@ HINT: You can enable two-phase commit for extra safety with: SET citus.multi_sh n_regionkey | integer | not null n_comment | character varying(152) | new_col | integer | +(5 rows) \c - - - :master_port ALTER TABLE test_schema_support.nation_hash DROP COLUMN IF EXISTS non_existent_column; @@ -614,24 +614,24 @@ NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' ALTER TABLE test_schema_support.nation_hash DROP COLUMN IF EXISTS new_col; -- verify column is dropped -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null n_name | character(25) | not null n_regionkey | integer | not null n_comment | character varying(152) | +(4 rows) \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null n_name | character(25) | not null n_regionkey | integer | not null n_comment | character varying(152) | +(4 rows) \c - - - :master_port --test with search_path is set @@ -640,8 +640,7 @@ ALTER TABLE nation_hash ADD COLUMN new_col INT; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' -- verify column is added -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null @@ -649,10 +648,10 @@ HINT: You can enable two-phase commit for extra safety with: SET citus.multi_sh n_regionkey | integer | not null n_comment | character varying(152) | new_col | integer | +(5 rows) \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null @@ -660,6 +659,7 @@ HINT: You can enable two-phase commit for extra safety with: SET citus.multi_sh n_regionkey | integer | not null n_comment | character varying(152) | new_col | integer | +(5 rows) \c - - - :master_port SET search_path TO test_schema_support; @@ -669,24 +669,24 @@ NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' ALTER TABLE nation_hash DROP COLUMN IF EXISTS new_col; -- verify column is dropped -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null n_name | character(25) | not null n_regionkey | integer | not null n_comment | character varying(152) | +(4 rows) \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; Column | Type | Modifiers -------------+------------------------+----------- n_nationkey | integer | not null n_name | character(25) | not null n_regionkey | integer | not null n_comment | character varying(152) | +(4 rows) \c - - - :master_port -- test CREATE/DROP INDEX with schemas @@ -696,28 +696,20 @@ CREATE INDEX index1 ON test_schema_support.nation_hash(n_name); NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' --verify INDEX is created -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | -Indexes: - "index1" btree (n_name) +\d test_schema_support.index1 + Index "test_schema_support.index1" + Column | Type | Definition +--------+---------------+------------ + n_name | character(25) | n_name +btree, for table "test_schema_support.nation_hash" \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | -Indexes: - "index1_1190003" btree (n_name) +\d test_schema_support.index1_1190003 +Index "test_schema_support.index1_1190003" + Column | Type | Definition +--------+---------------+------------ + n_name | character(25) | n_name +btree, for table "test_schema_support.nation_hash_1190003" \c - - - :master_port -- DROP index @@ -725,25 +717,9 @@ DROP INDEX test_schema_support.index1; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' --verify INDEX is dropped -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | - +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | - +\d test_schema_support.index1_1190003 \c - - - :master_port --test with search_path is set SET search_path TO test_schema_support; @@ -752,28 +728,20 @@ CREATE INDEX index1 ON nation_hash(n_name); NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' --verify INDEX is created -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | -Indexes: - "index1" btree (n_name) +\d test_schema_support.index1 + Index "test_schema_support.index1" + Column | Type | Definition +--------+---------------+------------ + n_name | character(25) | n_name +btree, for table "test_schema_support.nation_hash" \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | -Indexes: - "index1_1190003" btree (n_name) +\d test_schema_support.index1_1190003 +Index "test_schema_support.index1_1190003" + Column | Type | Definition +--------+---------------+------------ + n_name | character(25) | n_name +btree, for table "test_schema_support.nation_hash_1190003" \c - - - :master_port -- DROP index @@ -782,25 +750,9 @@ DROP INDEX index1; NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' --verify INDEX is dropped -\d test_schema_support.nation_hash; - Table "test_schema_support.nation_hash" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | - +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; - Table "test_schema_support.nation_hash_1190003" - Column | Type | Modifiers --------------+------------------------+----------- - n_nationkey | integer | not null - n_name | character(25) | not null - n_regionkey | integer | not null - n_comment | character varying(152) | - +\d test_schema_support.index1_1190003 \c - - - :master_port -- test master_copy_shard_placement with schemas SET search_path TO public; diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index 0468d42e2..edae227c0 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -2,6 +2,14 @@ -- MULTI_TASK_ASSIGNMENT -- ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 880000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 10 +(1 row) + SET citus.explain_distributed_queries TO off; -- Check that our policies for assigning tasks to worker nodes run as expected. -- To test this, we first create a shell table, and then manually insert shard @@ -46,19 +54,12 @@ BEGIN; -- the following log messages print node name and port numbers; and node numbers -- in regression tests depend upon PG_VERSION_NUM. SET client_min_messages TO DEBUG3; -DEBUG: CommitTransactionCommand -- First test the default greedy task assignment policy SET citus.task_assignment_policy TO 'greedy'; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57637 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -67,12 +68,9 @@ DEBUG: CommitTransactionCommand (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57637 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -82,16 +80,10 @@ DEBUG: CommitTransactionCommand -- Next test the first-replica task assignment policy SET citus.task_assignment_policy TO 'first-replica'; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -100,12 +92,9 @@ DEBUG: CommitTransactionCommand (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -115,16 +104,10 @@ DEBUG: CommitTransactionCommand -- Finally test the round-robin task assignment policy SET citus.task_assignment_policy TO 'round-robin'; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -133,12 +116,9 @@ DEBUG: CommitTransactionCommand (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57638 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -147,12 +127,9 @@ DEBUG: CommitTransactionCommand (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 -DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -161,10 +138,5 @@ DEBUG: CommitTransactionCommand (3 rows) RESET citus.task_assignment_policy; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: CommitTransactionCommand RESET client_min_messages; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility COMMIT; diff --git a/src/test/regress/expected/multi_task_assignment_policy_0.out b/src/test/regress/expected/multi_task_assignment_policy_0.out new file mode 100644 index 000000000..642e05a6d --- /dev/null +++ b/src/test/regress/expected/multi_task_assignment_policy_0.out @@ -0,0 +1,178 @@ +-- +-- MULTI_TASK_ASSIGNMENT +-- +ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 880000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + major_version +--------------- + 9 +(1 row) + +SET citus.explain_distributed_queries TO off; +-- Check that our policies for assigning tasks to worker nodes run as expected. +-- To test this, we first create a shell table, and then manually insert shard +-- and shard placement data into system catalogs. We next run Explain command, +-- and check that tasks are assigned to worker nodes as expected. +CREATE TABLE task_assignment_test_table (test_id integer); +SELECT master_create_distributed_table('task_assignment_test_table', 'test_id', 'append'); + master_create_distributed_table +--------------------------------- + +(1 row) + +-- Create logical shards with shardids 200, 201, and 202 +INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) + SELECT pg_class.oid, series.index, 'r', 1, 1000 + FROM pg_class, generate_series(200, 202) AS series(index) + WHERE pg_class.relname = 'task_assignment_test_table'; +-- Create shard placements for shard 200 and 201 +INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport) + SELECT 200, 1, 1, nodename, nodeport + FROM pg_dist_shard_placement + GROUP BY nodename, nodeport + ORDER BY nodename, nodeport ASC + LIMIT 2; +INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport) + SELECT 201, 1, 1, nodename, nodeport + FROM pg_dist_shard_placement + GROUP BY nodename, nodeport + ORDER BY nodename, nodeport ASC + LIMIT 2; +-- Create shard placements for shard 202 +INSERT INTO pg_dist_shard_placement (shardid, shardstate, shardlength, nodename, nodeport) + SELECT 202, 1, 1, nodename, nodeport + FROM pg_dist_shard_placement + GROUP BY nodename, nodeport + ORDER BY nodename, nodeport DESC + LIMIT 2; +-- Start transaction block to avoid auto commits. This avoids additional debug +-- messages from getting printed at real transaction starts and commits. +BEGIN; +-- Increase log level to see which worker nodes tasks are assigned to. Note that +-- the following log messages print node name and port numbers; and node numbers +-- in regression tests depend upon PG_VERSION_NUM. +SET client_min_messages TO DEBUG3; +DEBUG: CommitTransactionCommand +-- First test the default greedy task assignment policy +SET citus.task_assignment_policy TO 'greedy'; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Next test the first-replica task assignment policy +SET citus.task_assignment_policy TO 'first-replica'; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +-- Finally test the round-robin task assignment policy +SET citus.task_assignment_policy TO 'round-robin'; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +RESET citus.task_assignment_policy; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: CommitTransactionCommand +RESET client_min_messages; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +COMMIT; diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out new file mode 100644 index 000000000..7f8b92503 --- /dev/null +++ b/src/test/regress/expected/multi_test_helpers.out @@ -0,0 +1,86 @@ +-- File to create functions and helpers needed for subsequent tests +-- create a helper function to create objects on each node +CREATE FUNCTION run_command_on_master_and_workers(p_sql text) +RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE p_sql; + PERFORM run_command_on_workers(p_sql); +END;$$; +-- The following views are intended as alternatives to \d commands, whose +-- output changed in PostgreSQL 10. In particular, they must be used any time +-- a test wishes to print out the structure of a relation, which previously +-- was safely accomplished by a \d invocation. +SELECT run_command_on_master_and_workers( +$desc_views$ +CREATE VIEW table_fkey_cols AS +SELECT rc.constraint_name AS "name", + kcu.column_name AS "column_name", + uc_kcu.column_name AS "refd_column_name", + format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, + format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid +FROM information_schema.referential_constraints rc, + information_schema.key_column_usage kcu, + information_schema.key_column_usage uc_kcu +WHERE rc.constraint_schema = kcu.constraint_schema AND + rc.constraint_name = kcu.constraint_name AND + rc.unique_constraint_schema = uc_kcu.constraint_schema AND + rc.unique_constraint_name = uc_kcu.constraint_name; + +CREATE VIEW table_fkeys AS +SELECT name AS "Constraint", + format('FOREIGN KEY (%s) REFERENCES %s(%s)', + string_agg(DISTINCT quote_ident(column_name), ', '), + string_agg(DISTINCT refd_relid::regclass::text, ', '), + string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", + "relid" +FROM table_fkey_cols +GROUP BY (name, relid); + +CREATE VIEW table_attrs AS +SELECT c.column_name AS "name", + c.data_type AS "type", + CASE + WHEN character_maximum_length IS NOT NULL THEN + format('(%s)', character_maximum_length) + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN + format('(%s,%s)', numeric_precision, numeric_scale) + ELSE '' + END AS "modifier", + c.column_default AS "default", + (NOT c.is_nullable::boolean) AS "notnull", + format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" +FROM information_schema.columns AS c +ORDER BY ordinal_position; + +CREATE VIEW table_desc AS +SELECT "name" AS "Column", + "type" || "modifier" AS "Type", + rtrim(( + CASE "notnull" + WHEN true THEN 'not null ' + ELSE '' + END + ) || ( + CASE WHEN "default" IS NULL THEN '' + ELSE 'default ' || "default" + END + )) AS "Modifiers", + "relid" +FROM table_attrs; + +CREATE VIEW table_checks AS +SELECT cc.constraint_name AS "Constraint", + ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", + format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid +FROM information_schema.check_constraints cc, + information_schema.constraint_column_usage ccu +WHERE cc.constraint_schema = ccu.constraint_schema AND + cc.constraint_name = ccu.constraint_name +ORDER BY cc.constraint_name ASC; +$desc_views$ +); + run_command_on_master_and_workers +----------------------------------- + +(1 row) + diff --git a/src/test/regress/expected/multi_transactional_drop_shards.out b/src/test/regress/expected/multi_transactional_drop_shards.out index c59b3134c..e5cf653be 100644 --- a/src/test/regress/expected/multi_transactional_drop_shards.out +++ b/src/test/regress/expected/multi_transactional_drop_shards.out @@ -46,34 +46,24 @@ ORDER BY (8 rows) -- verify table is not dropped -\d transactional_drop_shards; -Table "public.transactional_drop_shards" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards + List of relations + Schema | Name | Type | Owner +--------+---------------------------+-------+---------- + public | transactional_drop_shards | table | postgres +(1 row) -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410000" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | - -Table "public.transactional_drop_shards_1410001" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | - -Table "public.transactional_drop_shards_1410002" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | - -Table "public.transactional_drop_shards_1410003" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410000 | table | postgres + public | transactional_drop_shards_1410001 | table | postgres + public | transactional_drop_shards_1410002 | table | postgres + public | transactional_drop_shards_1410003 | table | postgres +(4 rows) \c - - - :master_port -- test DROP TABLE(ergo master_drop_all_shards) in transaction, then COMMIT @@ -99,10 +89,20 @@ ORDER BY (0 rows) -- verify table is dropped -\d transactional_drop_shards; +\dt transactional_drop_shards + List of relations + Schema | Name | Type | Owner +--------+------+------+------- +(0 rows) + -- verify shards are dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+------+------+------- +(0 rows) + \c - - - :master_port -- test master_delete_protocol in transaction, then ROLLBACK CREATE TABLE transactional_drop_shards(column1 int); @@ -149,11 +149,12 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410004" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410004 | table | postgres +(1 row) \c - - - :master_port -- test master_delete_protocol in transaction, then COMMIT @@ -185,7 +186,12 @@ ORDER BY -- verify shards are dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+------+------+------- +(0 rows) + \c - - - :master_port -- test DROP table in a transaction after insertion SELECT master_create_empty_shard('transactional_drop_shards'); @@ -220,19 +226,21 @@ ORDER BY (2 rows) -- verify table is not dropped -\d transactional_drop_shards; -Table "public.transactional_drop_shards" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards + List of relations + Schema | Name | Type | Owner +--------+---------------------------+-------+---------- + public | transactional_drop_shards | table | postgres +(1 row) -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410005" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410005 | table | postgres +(1 row) \c - - - :master_port -- test master_apply_delete_command in a transaction after insertion @@ -268,11 +276,12 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410005" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410005 | table | postgres +(1 row) -- test DROP table with failing worker CREATE FUNCTION fail_drop_table() RETURNS event_trigger AS $fdt$ @@ -308,19 +317,21 @@ ORDER BY (2 rows) -- verify table is not dropped -\d transactional_drop_shards; -Table "public.transactional_drop_shards" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards + List of relations + Schema | Name | Type | Owner +--------+---------------------------+-------+---------- + public | transactional_drop_shards | table | postgres +(1 row) -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410005" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410005 | table | postgres +(1 row) \c - - - :master_port -- test DROP reference table with failing worker @@ -357,19 +368,21 @@ ORDER BY (2 rows) -- verify table is not dropped -\d transactional_drop_reference; -Table "public.transactional_drop_reference" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_reference + List of relations + Schema | Name | Type | Owner +--------+------------------------------+-------+---------- + public | transactional_drop_reference | table | postgres +(1 row) -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_reference*; -Table "public.transactional_drop_reference_1410006" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_reference* + List of relations + Schema | Name | Type | Owner +--------+--------------------------------------+-------+---------- + public | transactional_drop_reference_1410006 | table | postgres +(1 row) \c - - - :master_port -- test master_apply_delete_command table with failing worker @@ -400,11 +413,12 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; -Table "public.transactional_drop_shards_1410005" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt transactional_drop_shards_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_shards_1410005 | table | postgres +(1 row) DROP EVENT TRIGGER fail_drop_table; \c - - - :master_port @@ -464,16 +478,29 @@ ORDER BY (16 rows) -- verify table is not dropped -\d transactional_drop_serial; - Table "public.transactional_drop_serial" - Column | Type | Modifiers ----------+---------+----------------------------------------------------------------------------- - column1 | integer | - column2 | integer | not null default nextval('transactional_drop_serial_column2_seq'::regclass) +\dt transactional_drop_serial + List of relations + Schema | Name | Type | Owner +--------+---------------------------+-------+---------- + public | transactional_drop_serial | table | postgres +(1 row) -- verify shards and sequence are not dropped \c - - - :worker_1_port -\d transactional_drop_serial_1410006; +\dt transactional_drop_serial_* + List of relations + Schema | Name | Type | Owner +--------+-----------------------------------+-------+---------- + public | transactional_drop_serial_1410007 | table | postgres + public | transactional_drop_serial_1410008 | table | postgres + public | transactional_drop_serial_1410009 | table | postgres + public | transactional_drop_serial_1410010 | table | postgres + public | transactional_drop_serial_1410011 | table | postgres + public | transactional_drop_serial_1410012 | table | postgres + public | transactional_drop_serial_1410013 | table | postgres + public | transactional_drop_serial_1410014 | table | postgres +(8 rows) + \ds transactional_drop_serial_column2_seq List of relations Schema | Name | Type | Owner @@ -505,10 +532,20 @@ ORDER BY (0 rows) -- verify table is dropped -\d transactional_drop_serial; +\dt transactional_drop_serial + List of relations + Schema | Name | Type | Owner +--------+------+------+------- +(0 rows) + -- verify shards and sequence are dropped \c - - - :worker_1_port -\d transactional_drop_serial_1410006; +\dt transactional_drop_serial_* + List of relations + Schema | Name | Type | Owner +--------+------+------+------- +(0 rows) + \ds transactional_drop_serial_column2_seq List of relations Schema | Name | Type | Owner diff --git a/src/test/regress/expected/multi_unsupported_worker_operations.out b/src/test/regress/expected/multi_unsupported_worker_operations.out index a8101b0d5..4d3e578a1 100644 --- a/src/test/regress/expected/multi_unsupported_worker_operations.out +++ b/src/test/regress/expected/multi_unsupported_worker_operations.out @@ -144,15 +144,13 @@ NOTICE: using one-phase commit for distributed DDL commands HINT: You can enable two-phase commit for extra safety with: SET citus.multi_shard_commit_protocol TO '2pc' \c - - - :worker_1_port -- DDL commands -\d mx_table - Table "public.mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; Column | Type | Modifiers --------+---------+---------------------------------------------------------- col_1 | integer | col_2 | text | col_3 | bigint | not null default nextval('mx_table_col_3_seq'::regclass) -Indexes: - "mx_test_uniq_index" UNIQUE, btree (col_1) +(3 rows) CREATE INDEX mx_test_index ON mx_table(col_2); ERROR: operation is not allowed on this node @@ -163,16 +161,15 @@ HINT: Connect to the coordinator and run it again. ALTER TABLE mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY(col_1) REFERENCES mx_table(col_1); ERROR: operation is not allowed on this node HINT: Connect to the coordinator and run it again. -\d mx_table - Table "public.mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; Column | Type | Modifiers --------+---------+---------------------------------------------------------- col_1 | integer | col_2 | text | col_3 | bigint | not null default nextval('mx_table_col_3_seq'::regclass) -Indexes: - "mx_test_uniq_index" UNIQUE, btree (col_1) +(3 rows) +\d mx_test_index -- master_modify_multiple_shards SELECT master_modify_multiple_shards('UPDATE mx_table SET col_2=''none'''); ERROR: operation is not allowed on this node @@ -402,23 +399,23 @@ CREATE SEQUENCE some_sequence; DROP SEQUENCE some_sequence; -- Show that dropping the sequence of an MX table with cascade harms the table and shards BEGIN; -\d mx_table - Table "public.mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; Column | Type | Modifiers --------+---------+---------------------------------------------------------- col_1 | integer | col_2 | text | col_3 | bigint | not null default nextval('mx_table_col_3_seq'::regclass) +(3 rows) DROP SEQUENCE mx_table_col_3_seq CASCADE; NOTICE: drop cascades to default for table mx_table column col_3 -\d mx_table - Table "public.mx_table" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; Column | Type | Modifiers --------+---------+----------- col_1 | integer | col_2 | text | col_3 | bigint | not null +(3 rows) ROLLBACK; -- Cleanup diff --git a/src/test/regress/expected/multi_upgrade_reference_table.out b/src/test/regress/expected/multi_upgrade_reference_table.out index 80c2cde7a..f50284088 100644 --- a/src/test/regress/expected/multi_upgrade_reference_table.out +++ b/src/test/regress/expected/multi_upgrade_reference_table.out @@ -794,11 +794,12 @@ ORDER BY -- verify that shard is replicated to other worker \c - - - :worker_2_port -\d upgrade_reference_table_transaction_commit_* -Table "public.upgrade_reference_table_transaction_commit_1360014" - Column | Type | Modifiers ----------+---------+----------- - column1 | integer | +\dt upgrade_reference_table_transaction_commit_* + List of relations + Schema | Name | Type | Owner +--------+----------------------------------------------------+-------+---------- + public | upgrade_reference_table_transaction_commit_1360014 | table | postgres +(1 row) \c - - - :master_port DROP TABLE upgrade_reference_table_transaction_commit; diff --git a/src/test/regress/input/multi_alter_table_statements.source b/src/test/regress/input/multi_alter_table_statements.source index 0ca6877d6..58113c28f 100644 --- a/src/test/regress/input/multi_alter_table_statements.source +++ b/src/test/regress/input/multi_alter_table_statements.source @@ -47,7 +47,7 @@ FROM ORDER BY attnum; \c - - - :master_port -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; SELECT float_column, count(*) FROM lineitem_alter GROUP BY float_column; SELECT int_column1, count(*) FROM lineitem_alter GROUP BY int_column1; @@ -65,7 +65,7 @@ SELECT int_column1, count(*) FROM lineitem_alter GROUP BY int_column1; -- Verify that SET NOT NULL works ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET NOT NULL; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; -- Drop default so that NULLs will be inserted for this column ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP DEFAULT; @@ -77,7 +77,7 @@ ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP DEFAULT; -- Verify that DROP NOT NULL works ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP NOT NULL; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; -- \copy should succeed now \copy lineitem_alter (l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment) FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' @@ -88,7 +88,7 @@ SELECT count(*) from lineitem_alter; SELECT int_column2, pg_typeof(int_column2), count(*) from lineitem_alter GROUP BY int_column2; ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET DATA TYPE FLOAT; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; SELECT int_column2, pg_typeof(int_column2), count(*) from lineitem_alter GROUP BY int_column2; @@ -116,19 +116,19 @@ ALTER TABLE lineitem_alter DROP COLUMN IF EXISTS int_column2; ALTER TABLE IF EXISTS lineitem_alter RENAME COLUMN l_orderkey_renamed TO l_orderkey; SELECT SUM(l_orderkey) FROM lineitem_alter; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; -- Verify that we can execute commands with multiple subcommands ALTER TABLE lineitem_alter ADD COLUMN int_column1 INTEGER, ADD COLUMN int_column2 INTEGER; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; ALTER TABLE lineitem_alter ADD COLUMN int_column3 INTEGER, ALTER COLUMN int_column1 SET STATISTICS 10; ALTER TABLE lineitem_alter DROP COLUMN int_column1, DROP COLUMN int_column2; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; -- Verify that we cannot execute alter commands on the distribution column @@ -161,7 +161,7 @@ ALTER TABLE IF EXISTS non_existent_table RENAME COLUMN column1 TO column2; -- Verify that none of the failed alter table commands took effect on the master -- node -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; -- verify that non-propagated ddl commands are allowed inside a transaction block SET citus.enable_ddl_propagation to false; @@ -185,7 +185,8 @@ CREATE INDEX temp_index_2 ON lineitem_alter(l_orderkey); ALTER TABLE lineitem_alter ADD COLUMN first integer; COMMIT; -\d lineitem_alter +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; +\d temp_index_2 ALTER TABLE lineitem_alter DROP COLUMN first; DROP INDEX temp_index_2; diff --git a/src/test/regress/input/multi_behavioral_analytics_create_table.source b/src/test/regress/input/multi_behavioral_analytics_create_table.source index b246564db..fcb1ade20 100644 --- a/src/test/regress/input/multi_behavioral_analytics_create_table.source +++ b/src/test/regress/input/multi_behavioral_analytics_create_table.source @@ -41,14 +41,6 @@ CREATE INDEX is_index4 ON events_table(event_type); CREATE INDEX is_index5 ON users_table(value_2); CREATE INDEX is_index6 ON events_table(value_2); --- create a helper function to create types/functions on each node -CREATE FUNCTION run_command_on_master_and_workers(p_sql text) -RETURNS void LANGUAGE plpgsql AS $$ -BEGIN - EXECUTE p_sql; - PERFORM run_command_on_workers(p_sql); -END;$$; - -- Create composite type to use in subquery pushdown SELECT run_command_on_master_and_workers($f$ diff --git a/src/test/regress/multi_binary_schedule b/src/test/regress/multi_binary_schedule index 698eb69a2..44b277067 100644 --- a/src/test/regress/multi_binary_schedule +++ b/src/test/regress/multi_binary_schedule @@ -12,6 +12,7 @@ # --- test: multi_extension test: multi_cluster_management +test: multi_test_helpers test: multi_table_ddl # ---------- diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule index 36182b000..c8ae84212 100644 --- a/src/test/regress/multi_mx_schedule +++ b/src/test/regress/multi_mx_schedule @@ -15,6 +15,7 @@ # --- test: multi_extension test: multi_cluster_management +test: multi_test_helpers test: multi_mx_create_table test: multi_mx_copy_data multi_mx_router_planner diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index bc7d0d977..2fdd9b3d5 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -17,6 +17,7 @@ # --- test: multi_extension test: multi_cluster_management +test: multi_test_helpers test: multi_table_ddl test: multi_name_lengths test: multi_metadata_access diff --git a/src/test/regress/multi_task_tracker_extra_schedule b/src/test/regress/multi_task_tracker_extra_schedule index b56ba506f..bdccc840b 100644 --- a/src/test/regress/multi_task_tracker_extra_schedule +++ b/src/test/regress/multi_task_tracker_extra_schedule @@ -15,6 +15,7 @@ # --- test: multi_extension test: multi_cluster_management +test: multi_test_helpers test: multi_table_ddl # ---------- diff --git a/src/test/regress/output/multi_alter_table_statements.source b/src/test/regress/output/multi_alter_table_statements.source index 6c89e17ea..7c9d29510 100644 --- a/src/test/regress/output/multi_alter_table_statements.source +++ b/src/test/regress/output/multi_alter_table_statements.source @@ -77,8 +77,7 @@ ORDER BY attnum; (27 rows) \c - - - :master_port -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -102,6 +101,7 @@ ORDER BY attnum; int_column1 | integer | default 1 int_column2 | integer | default 2 null_column | integer | +(21 rows) SELECT float_column, count(*) FROM lineitem_alter GROUP BY float_column; float_column | count @@ -138,8 +138,7 @@ SELECT int_column1, count(*) FROM lineitem_alter GROUP BY int_column1; -- Verify that SET NOT NULL works ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET NOT NULL; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+-------------------- l_orderkey | bigint | not null @@ -163,6 +162,7 @@ ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET NOT NULL; int_column1 | integer | int_column2 | integer | not null default 2 null_column | integer | +(21 rows) -- Drop default so that NULLs will be inserted for this column ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP DEFAULT; @@ -173,8 +173,7 @@ ERROR: null value in column "int_column2" violates not-null constraint DETAIL: Failing row contains (1, 155190, 7706, 1, 17.00, 21168.23, 0.04, 0.02, N, O, 1996-03-13, 1996-02-12, 1996-03-22, DELIVER IN PERSON , TRUCK , egular courts above the, 1, null, null, null, null). -- Verify that DROP NOT NULL works ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP NOT NULL; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -198,6 +197,7 @@ ALTER TABLE lineitem_alter ALTER COLUMN int_column2 DROP NOT NULL; int_column1 | integer | int_column2 | integer | null_column | integer | +(21 rows) -- \copy should succeed now \copy lineitem_alter (l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment) FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' @@ -216,8 +216,7 @@ SELECT int_column2, pg_typeof(int_column2), count(*) from lineitem_alter GROUP B (2 rows) ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET DATA TYPE FLOAT; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -241,6 +240,7 @@ ALTER TABLE lineitem_alter ALTER COLUMN int_column2 SET DATA TYPE FLOAT; int_column1 | integer | int_column2 | double precision | null_column | integer | +(21 rows) SELECT int_column2, pg_typeof(int_column2), count(*) from lineitem_alter GROUP BY int_column2; int_column2 | pg_typeof | count @@ -280,8 +280,7 @@ SELECT SUM(l_orderkey) FROM lineitem_alter; 53620791 (1 row) -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -301,12 +300,12 @@ SELECT SUM(l_orderkey) FROM lineitem_alter; l_shipmode | character(10) | not null l_comment | character varying(44) | not null null_column | integer | +(17 rows) -- Verify that we can execute commands with multiple subcommands ALTER TABLE lineitem_alter ADD COLUMN int_column1 INTEGER, ADD COLUMN int_column2 INTEGER; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -328,14 +327,14 @@ ALTER TABLE lineitem_alter ADD COLUMN int_column1 INTEGER, null_column | integer | int_column1 | integer | int_column2 | integer | +(19 rows) ALTER TABLE lineitem_alter ADD COLUMN int_column3 INTEGER, ALTER COLUMN int_column1 SET STATISTICS 10; ERROR: alter table command is currently unsupported DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP CONSTRAINT and TYPE subcommands are supported. ALTER TABLE lineitem_alter DROP COLUMN int_column1, DROP COLUMN int_column2; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -355,6 +354,7 @@ ALTER TABLE lineitem_alter DROP COLUMN int_column1, DROP COLUMN int_column2; l_shipmode | character(10) | not null l_comment | character varying(44) | not null null_column | integer | +(17 rows) -- Verify that we cannot execute alter commands on the distribution column ALTER TABLE lineitem_alter ALTER COLUMN l_orderkey DROP NOT NULL; @@ -395,8 +395,7 @@ ALTER TABLE IF EXISTS non_existent_table RENAME COLUMN column1 TO column2; NOTICE: relation "non_existent_table" does not exist, skipping -- Verify that none of the failed alter table commands took effect on the master -- node -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -416,6 +415,7 @@ NOTICE: relation "non_existent_table" does not exist, skipping l_shipmode | character(10) | not null l_comment | character varying(44) | not null null_column | integer | +(17 rows) -- verify that non-propagated ddl commands are allowed inside a transaction block SET citus.enable_ddl_propagation to false; @@ -446,8 +446,7 @@ BEGIN; CREATE INDEX temp_index_2 ON lineitem_alter(l_orderkey); ALTER TABLE lineitem_alter ADD COLUMN first integer; COMMIT; -\d lineitem_alter - Table "public.lineitem_alter" +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.lineitem_alter'::regclass; Column | Type | Modifiers -----------------+-----------------------+----------- l_orderkey | bigint | not null @@ -468,8 +467,14 @@ COMMIT; l_comment | character varying(44) | not null null_column | integer | first | integer | -Indexes: - "temp_index_2" btree (l_orderkey) +(18 rows) + +\d temp_index_2 + Index "public.temp_index_2" + Column | Type | Definition +------------+--------+------------ + l_orderkey | bigint | l_orderkey +btree, for table "public.lineitem_alter" ALTER TABLE lineitem_alter DROP COLUMN first; DROP INDEX temp_index_2; diff --git a/src/test/regress/output/multi_behavioral_analytics_create_table.source b/src/test/regress/output/multi_behavioral_analytics_create_table.source index c07e21cad..093bc1d15 100644 --- a/src/test/regress/output/multi_behavioral_analytics_create_table.source +++ b/src/test/regress/output/multi_behavioral_analytics_create_table.source @@ -62,13 +62,6 @@ CREATE INDEX is_index3 ON users_table(value_1); CREATE INDEX is_index4 ON events_table(event_type); CREATE INDEX is_index5 ON users_table(value_2); CREATE INDEX is_index6 ON events_table(value_2); --- create a helper function to create types/functions on each node -CREATE FUNCTION run_command_on_master_and_workers(p_sql text) -RETURNS void LANGUAGE plpgsql AS $$ -BEGIN - EXECUTE p_sql; - PERFORM run_command_on_workers(p_sql); -END;$$; -- Create composite type to use in subquery pushdown SELECT run_command_on_master_and_workers($f$ diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 1e31dd04d..7af3f2ec6 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -439,7 +439,7 @@ my @arguments = ( '--user', $user ); -if ($majorversion eq '9.5' || $majorversion eq '9.6') +if ($majorversion eq '9.5' || $majorversion eq '9.6' || $majorversion eq '10') { push(@arguments, '--bindir', "tmp_check/tmp-bin"); } diff --git a/src/test/regress/sql/multi_alter_table_add_constraints.sql b/src/test/regress/sql/multi_alter_table_add_constraints.sql index 1daa4e9e8..58c420b49 100644 --- a/src/test/regress/sql/multi_alter_table_add_constraints.sql +++ b/src/test/regress/sql/multi_alter_table_add_constraints.sql @@ -398,11 +398,11 @@ INSERT INTO products VALUES(1,'product_1', 10, 8); ROLLBACK; -- There should be no constraint on master and worker(s) -\d products +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='products'::regclass; \c - - - :worker_1_port -\d products_1450199 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.products_1450202'::regclass; \c - - - :master_port @@ -415,11 +415,11 @@ ALTER TABLE products ADD CONSTRAINT p_key_product PRIMARY KEY(product_no); ROLLBACK; -- There should be no constraint on master and worker(s) -\d products +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='products'::regclass; \c - - - :worker_1_port -\d products_1450199 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.products_1450202'::regclass; \c - - - :master_port DROP TABLE products; diff --git a/src/test/regress/sql/multi_colocation_utils.sql b/src/test/regress/sql/multi_colocation_utils.sql index b08d217d8..1957910b1 100644 --- a/src/test/regress/sql/multi_colocation_utils.sql +++ b/src/test/regress/sql/multi_colocation_utils.sql @@ -293,8 +293,8 @@ SELECT create_distributed_table('table_bigint', 'id', colocate_with => 'table1_g -- check worker table schemas \c - - - :worker_1_port -\d table3_groupE_1300050 -\d schema_collocation.table4_groupE_1300052 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.table3_groupE_1300050'::regclass; +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='schema_collocation.table4_groupE_1300052'::regclass; \c - - - :master_port diff --git a/src/test/regress/sql/multi_create_table_constraints.sql b/src/test/regress/sql/multi_create_table_constraints.sql index dde288bab..aafb644dc 100644 --- a/src/test/regress/sql/multi_create_table_constraints.sql +++ b/src/test/regress/sql/multi_create_table_constraints.sql @@ -237,9 +237,8 @@ SELECT master_create_distributed_table('check_example', 'partition_col', 'hash') SELECT master_create_worker_shards('check_example', '2', '2'); \c - - - :worker_1_port -\d check_example* -\c - - - :worker_2_port -\d check_example* +\d check_example_partition_col_key_365040 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.check_example_365040'::regclass; \c - - - :master_port -- drop unnecessary tables @@ -260,7 +259,7 @@ CREATE TABLE raw_table_2 (user_id int REFERENCES raw_table_1(user_id), UNIQUE(us SELECT create_distributed_table('raw_table_2', 'user_id'); -- see that the constraint exists -\d raw_table_2 +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='raw_table_2'::regclass; -- should be prevented by the foreign key DROP TABLE raw_table_1; @@ -269,7 +268,7 @@ DROP TABLE raw_table_1; DROP TABLE raw_table_1 CASCADE; -- see that the constraint also dropped -\d raw_table_2 +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='raw_table_2'::regclass; -- drop the table as well DROP TABLE raw_table_2; diff --git a/src/test/regress/sql/multi_create_table_new_features.sql b/src/test/regress/sql/multi_create_table_new_features.sql new file mode 100644 index 000000000..7e5676021 --- /dev/null +++ b/src/test/regress/sql/multi_create_table_new_features.sql @@ -0,0 +1,21 @@ +-- +-- MULTI_CREATE_TABLE_NEW_FEATURES +-- + +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + +-- Verify that the GENERATED ... AS IDENTITY feature in PostgreSQL 10 +-- is forbidden in distributed tables. + +CREATE TABLE table_identity_col ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + payload text ); + +SELECT master_create_distributed_table('table_identity_col', 'id', 'append'); + +SELECT create_distributed_table('table_identity_col', 'id'); +SELECT create_distributed_table('table_identity_col', 'text'); + +SELECT create_reference_table('table_identity_col'); diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index b6099e389..ff83499ed 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -426,6 +426,7 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) SET parallel_setup_cost=0; SET parallel_tuple_cost=0; SET min_parallel_relation_size=0; +SET min_parallel_table_scan_size=0; SET max_parallel_workers_per_gather=4; -- ensure local plans display correctly diff --git a/src/test/regress/sql/multi_join_order_additional.sql b/src/test/regress/sql/multi_join_order_additional.sql index 42dc4b97a..87469c14e 100644 --- a/src/test/regress/sql/multi_join_order_additional.sql +++ b/src/test/regress/sql/multi_join_order_additional.sql @@ -5,6 +5,8 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 650000; +-- print whether we're running on 9.5 to make version-specific tests clear +SELECT substring(version(), '\d+(?:\.\d+)?') = '9.5' AS is_95; -- Set configuration to print table join order and pruned shards diff --git a/src/test/regress/sql/multi_large_table_join_planning.sql b/src/test/regress/sql/multi_large_table_join_planning.sql index 3a10c5f76..d8da3454f 100644 --- a/src/test/regress/sql/multi_large_table_join_planning.sql +++ b/src/test/regress/sql/multi_large_table_join_planning.sql @@ -10,6 +10,9 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 690000; SET citus.enable_unique_job_ids TO off; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; BEGIN; SET client_min_messages TO DEBUG4; diff --git a/src/test/regress/sql/multi_large_table_task_assignment.sql b/src/test/regress/sql/multi_large_table_task_assignment.sql index bd4de27d7..9a69189fa 100644 --- a/src/test/regress/sql/multi_large_table_task_assignment.sql +++ b/src/test/regress/sql/multi_large_table_task_assignment.sql @@ -9,6 +9,10 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 710000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + BEGIN; SET client_min_messages TO DEBUG3; diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql index d698e8b08..47dee0e22 100644 --- a/src/test/regress/sql/multi_metadata_sync.sql +++ b/src/test/regress/sql/multi_metadata_sync.sql @@ -78,7 +78,9 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition ORDER BY logicalrelid; SELECT * FROM pg_dist_shard ORDER BY shardid; SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; -\d mx_testing_schema.mx_test_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; +\d mx_testing_schema.mx_test_table_col_1_key +\d mx_testing_schema.mx_index -- Check that pg_dist_colocation is not synced SELECT * FROM pg_dist_colocation ORDER BY colocationid; @@ -107,7 +109,7 @@ SELECT start_metadata_sync_to_node('localhost', :worker_1_port); -- Check that foreign key metadata exists on the worker \c - - - :worker_1_port -\d mx_testing_schema_2.fk_test_2 +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_testing_schema_2.fk_test_2'::regclass; \c - - - :master_port DROP TABLE mx_testing_schema_2.fk_test_2; @@ -126,7 +128,9 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; SELECT * FROM pg_dist_partition ORDER BY logicalrelid; SELECT * FROM pg_dist_shard ORDER BY shardid; SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; -\d mx_testing_schema.mx_test_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_testing_schema.mx_test_table'::regclass; +\d mx_testing_schema.mx_test_table_col_1_key +\d mx_testing_schema.mx_index SELECT count(*) FROM pg_trigger WHERE tgrelid='mx_testing_schema.mx_test_table'::regclass; -- Make sure that start_metadata_sync_to_node cannot be called inside a transaction @@ -190,8 +194,13 @@ CREATE TABLE mx_test_schema_2.mx_table_2 (col1 int, col2 text); CREATE INDEX mx_index_2 ON mx_test_schema_2.mx_table_2 (col2); ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY(col1) REFERENCES mx_test_schema_1.mx_table_1(col1); -\d mx_test_schema_1.mx_table_1 -\d mx_test_schema_2.mx_table_2 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; +\d mx_test_schema_1.mx_table_1_col1_key +\d mx_test_schema_1.mx_index_1 + +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_2.mx_table_2'::regclass; +\d mx_test_schema_2.mx_index_2 +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_2.mx_table_2'::regclass; SELECT create_distributed_table('mx_test_schema_1.mx_table_1', 'col1'); SELECT create_distributed_table('mx_test_schema_2.mx_table_2', 'col1'); @@ -222,8 +231,7 @@ ORDER BY \c - - - :worker_1_port -- Check that tables are created -\d mx_test_schema_1.mx_table_1 -\d mx_test_schema_2.mx_table_2 +\dt mx_test_schema_?.mx_table_? -- Check that table metadata are created SELECT @@ -260,16 +268,17 @@ SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodename, nodeport; SET citus.multi_shard_commit_protocol TO '2pc'; SET client_min_messages TO 'ERROR'; CREATE INDEX mx_index_3 ON mx_test_schema_2.mx_table_2 USING hash (col1); -CREATE UNIQUE INDEX mx_index_4 ON mx_test_schema_2.mx_table_2(col1); +ALTER TABLE mx_test_schema_2.mx_table_2 ADD CONSTRAINT mx_table_2_col1_key UNIQUE (col1); \c - - - :worker_1_port -\d mx_test_schema_2.mx_table_2 +\d mx_test_schema_2.mx_index_3 +\d mx_test_schema_2.mx_table_2_col1_key -- Check that DROP INDEX statement is propagated \c - - - :master_port SET citus.multi_shard_commit_protocol TO '2pc'; DROP INDEX mx_test_schema_2.mx_index_3; \c - - - :worker_1_port -\d mx_test_schema_2.mx_table_2 +\d mx_test_schema_2.mx_index_3 -- Check that ALTER TABLE statements are propagated \c - - - :master_port @@ -285,7 +294,8 @@ FOREIGN KEY REFERENCES mx_test_schema_2.mx_table_2(col1); \c - - - :worker_1_port -\d mx_test_schema_1.mx_table_1 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_test_schema_1.mx_table_1'::regclass; +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; -- Check that foreign key constraint with NOT VALID works as well \c - - - :master_port @@ -301,7 +311,7 @@ REFERENCES mx_test_schema_2.mx_table_2(col1) NOT VALID; \c - - - :worker_1_port -\d mx_test_schema_1.mx_table_1 +SELECT "Constraint", "Definition" FROM table_fkeys WHERE relid='mx_test_schema_1.mx_table_1'::regclass; -- Check that mark_tables_colocated call propagates the changes to the workers \c - - - :master_port @@ -417,13 +427,13 @@ DROP TABLE mx_table_with_small_sequence; -- Create an MX table with (BIGSERIAL) sequences CREATE TABLE mx_table_with_sequence(a int, b BIGSERIAL, c BIGSERIAL); SELECT create_distributed_table('mx_table_with_sequence', 'a'); -\d mx_table_with_sequence +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; \ds mx_table_with_sequence_b_seq \ds mx_table_with_sequence_c_seq -- Check that the sequences created on the metadata worker as well \c - - - :worker_1_port -\d mx_table_with_sequence +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; \ds mx_table_with_sequence_b_seq \ds mx_table_with_sequence_c_seq @@ -437,7 +447,7 @@ SELECT start_metadata_sync_to_node('localhost', :worker_2_port); \c - - - :worker_2_port SELECT groupid FROM pg_dist_local_group; -\d mx_table_with_sequence +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_table_with_sequence'::regclass; \ds mx_table_with_sequence_b_seq \ds mx_table_with_sequence_c_seq SELECT nextval('mx_table_with_sequence_b_seq'); @@ -525,10 +535,10 @@ DROP USER mx_user; \c - - - :master_port CREATE TABLE mx_ref (col_1 int, col_2 text); SELECT create_reference_table('mx_ref'); -\d mx_ref +\dt mx_ref \c - - - :worker_1_port -\d mx_ref +\dt mx_ref SELECT logicalrelid, partmethod, repmodel, shardid, placementid, nodename, nodeport FROM @@ -546,10 +556,12 @@ SELECT shardid AS ref_table_shardid FROM pg_dist_shard WHERE logicalrelid='mx_re \c - - - :master_port ALTER TABLE mx_ref ADD COLUMN col_3 NUMERIC DEFAULT 0; CREATE INDEX mx_ref_index ON mx_ref(col_1); -\d mx_ref +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; +\d mx_ref_index \c - - - :worker_1_port -\d mx_ref +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ref'::regclass; +\d mx_ref_index -- Check that metada is cleaned successfully upon drop table \c - - - :master_port diff --git a/src/test/regress/sql/multi_modifying_xacts.sql b/src/test/regress/sql/multi_modifying_xacts.sql index e5d9ddc71..233a64be7 100644 --- a/src/test/regress/sql/multi_modifying_xacts.sql +++ b/src/test/regress/sql/multi_modifying_xacts.sql @@ -151,7 +151,7 @@ INSERT INTO labs VALUES (6, 'Bell Labs'); COMMIT; -- but the DDL should correctly roll back -\d labs +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.labs'::regclass; SELECT * FROM labs WHERE id = 6; -- COPY can happen after single row INSERT diff --git a/src/test/regress/sql/multi_mx_ddl.sql b/src/test/regress/sql/multi_mx_ddl.sql index 3c4301dc1..cde000c50 100644 --- a/src/test/regress/sql/multi_mx_ddl.sql +++ b/src/test/regress/sql/multi_mx_ddl.sql @@ -20,19 +20,22 @@ ALTER TABLE mx_ddl_table ALTER COLUMN version SET NOT NULL; -- See that the changes are applied on coordinator, worker tables and shards -\d mx_ddl_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\d ddl_test*_index \c - - - :worker_1_port -\d mx_ddl_table - -\d mx_ddl_table_1220088 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\d ddl_test*_index +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220088'::regclass; +\d ddl_test*_index_1220088 \c - - - :worker_2_port -\d mx_ddl_table - -\d mx_ddl_table_1220089 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\d ddl_test*_index +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220089'::regclass; +\d ddl_test*_index_1220089 INSERT INTO mx_ddl_table VALUES (37, 78, 2); INSERT INTO mx_ddl_table VALUES (38, 78); @@ -68,19 +71,22 @@ ALTER TABLE mx_ddl_table DROP COLUMN version; -- See that the changes are applied on coordinator, worker tables and shards -\d mx_ddl_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\di ddl_test*_index \c - - - :worker_1_port -\d mx_ddl_table - -\d mx_ddl_table_1220088 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\di ddl_test*_index +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220088'::regclass; +\di ddl_test*_index_1220088 \c - - - :worker_2_port -\d mx_ddl_table - -\d mx_ddl_table_1220089 +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table'::regclass; +\di ddl_test*_index +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='mx_ddl_table_1220089'::regclass; +\di ddl_test*_index_1220089 -- Show that DDL commands are done within a two-phase commit transaction \c - - - :master_port diff --git a/src/test/regress/sql/multi_mx_metadata.sql b/src/test/regress/sql/multi_mx_metadata.sql index f86834beb..d804d4307 100644 --- a/src/test/regress/sql/multi_mx_metadata.sql +++ b/src/test/regress/sql/multi_mx_metadata.sql @@ -27,7 +27,9 @@ SELECT count(*) FROM pg_dist_transaction; \c - - - :worker_1_port -\d distributed_mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='distributed_mx_table'::regclass; +\d distributed_mx_table_pkey +\d distributed_mx_table_value_idx SELECT repmodel FROM pg_dist_partition WHERE logicalrelid = 'distributed_mx_table'::regclass; @@ -37,7 +39,9 @@ WHERE logicalrelid = 'distributed_mx_table'::regclass; \c - - - :worker_2_port -\d distributed_mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='distributed_mx_table'::regclass; +\d distributed_mx_table_pkey +\d distributed_mx_table_value_idx SELECT repmodel FROM pg_dist_partition WHERE logicalrelid = 'distributed_mx_table'::regclass; diff --git a/src/test/regress/sql/multi_name_lengths.sql b/src/test/regress/sql/multi_name_lengths.sql index 5f992b0d7..b434d1d72 100644 --- a/src/test/regress/sql/multi_name_lengths.sql +++ b/src/test/regress/sql/multi_name_lengths.sql @@ -14,7 +14,7 @@ SELECT master_create_distributed_table('too_long_1234567890123456789012345678901 SELECT master_create_worker_shards('too_long_12345678901234567890123456789012345678901234567890', '2', '2'); \c - - - :worker_1_port -\d too_long_* +\dt too_long_* \c - - - :master_port -- Verify that the UDF works and rejects bad arguments. @@ -50,7 +50,7 @@ ALTER TABLE name_lengths ADD EXCLUDE (int_col_1234567890123456789012345678901234 ALTER TABLE name_lengths ADD CHECK (date_col_12345678901234567890123456789012345678901234567890 > '2014-01-01'::date); \c - - - :worker_1_port -\d name_lengths_* +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.name_lengths_225002'::regclass; \c - - - :master_port -- Placeholders for unsupported add constraints with EXPLICIT names that are too long @@ -59,7 +59,7 @@ ALTER TABLE name_lengths ADD CONSTRAINT nl_exclude_12345678901234567890123456789 ALTER TABLE name_lengths ADD CONSTRAINT nl_checky_12345678901234567890123456789012345678901234567890 CHECK (date_col_12345678901234567890123456789012345678901234567890 >= '2014-01-01'::date); \c - - - :worker_1_port -\d nl_* +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.name_lengths_225002'::regclass; \c - - - :master_port -- Placeholders for RENAME operations @@ -99,13 +99,16 @@ CREATE TABLE sneaky_name_lengths ( col2 integer not null, CONSTRAINT checky_12345678901234567890123456789012345678901234567890 CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100) ); -\d sneaky_name_lengths* + +\di public.sneaky_name_lengths* +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths'::regclass; SELECT master_create_distributed_table('sneaky_name_lengths', 'int_col_123456789012345678901234567890123456789012345678901234', 'hash'); SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2'); \c - - - :worker_1_port -\d sneaky_name_lengths* +\di public.sneaky*225006 +SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths_225006'::regclass; \c - - - :master_port DROP TABLE sneaky_name_lengths CASCADE; @@ -121,7 +124,7 @@ SELECT master_create_distributed_table('sneaky_name_lengths', 'col1', 'hash'); SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2'); \c - - - :worker_1_port -\d sneaky_name_lengths* +\di unique*225008 \c - - - :master_port DROP TABLE sneaky_name_lengths CASCADE; @@ -135,7 +138,7 @@ SELECT master_create_distributed_table('too_long_1234567890123456789012345678901 SELECT master_create_worker_shards('too_long_12345678901234567890123456789012345678901234567890', '2', '2'); \c - - - :worker_1_port -\d too_long_* +\dt *225000000000* \c - - - :master_port DROP TABLE too_long_12345678901234567890123456789012345678901234567890 CASCADE; @@ -148,7 +151,8 @@ SELECT master_create_distributed_table(U&'elephant_!0441!043B!043E!043D!0441!043 SELECT master_create_worker_shards(U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D' UESCAPE '!', '2', '2'); \c - - - :worker_1_port -\d elephant_* +\dt public.elephant_* +\di public.elephant_* \c - - - :master_port -- Clean up. diff --git a/src/test/regress/sql/multi_null_minmax_value_pruning.sql b/src/test/regress/sql/multi_null_minmax_value_pruning.sql index 60ebd51cf..03dbb77f1 100644 --- a/src/test/regress/sql/multi_null_minmax_value_pruning.sql +++ b/src/test/regress/sql/multi_null_minmax_value_pruning.sql @@ -8,6 +8,10 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 760000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + SET client_min_messages TO DEBUG2; SET citus.explain_all_tasks TO on; -- to avoid differing explain output - executor doesn't matter, diff --git a/src/test/regress/sql/multi_reference_table.sql b/src/test/regress/sql/multi_reference_table.sql index e16c2d271..9c22ce6c1 100644 --- a/src/test/regress/sql/multi_reference_table.sql +++ b/src/test/regress/sql/multi_reference_table.sql @@ -859,15 +859,18 @@ ALTER TABLE reference_table_ddl ALTER COLUMN value_2 SET DEFAULT 25.0; ALTER TABLE reference_table_ddl ALTER COLUMN value_3 SET NOT NULL; -- see that Citus applied all DDLs to the table -\d reference_table_ddl +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl'::regclass; +\d reference_index_2 -- also to the shard placements \c - - - :worker_1_port -\d reference_table_ddl* +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl_1250019'::regclass; +\d reference_index_2_1250019 \c - - - :master_port DROP INDEX reference_index_2; \c - - - :worker_1_port -\d reference_table_ddl* +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.reference_table_ddl_1250019'::regclass; +\di reference_index_2* \c - - - :master_port -- as we expect, renaming and setting WITH OIDS does not work for reference tables diff --git a/src/test/regress/sql/multi_remove_node_reference_table.sql b/src/test/regress/sql/multi_remove_node_reference_table.sql index 34bcaf7e5..60f3584da 100644 --- a/src/test/regress/sql/multi_remove_node_reference_table.sql +++ b/src/test/regress/sql/multi_remove_node_reference_table.sql @@ -384,7 +384,7 @@ WHERE \c - - - :master_port -- verify table structure is changed -\d remove_node_reference_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.remove_node_reference_table'::regclass; -- re-add the node for next tests SELECT master_add_node('localhost', :worker_2_port); diff --git a/src/test/regress/sql/multi_schema_support.sql b/src/test/regress/sql/multi_schema_support.sql index a5864a562..4b0879947 100644 --- a/src/test/regress/sql/multi_schema_support.sql +++ b/src/test/regress/sql/multi_schema_support.sql @@ -420,18 +420,18 @@ SET search_path TO public; ALTER TABLE test_schema_support.nation_hash ADD COLUMN new_col INT; -- verify column is added -\d test_schema_support.nation_hash; +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash'::regclass; \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; \c - - - :master_port ALTER TABLE test_schema_support.nation_hash DROP COLUMN IF EXISTS non_existent_column; ALTER TABLE test_schema_support.nation_hash DROP COLUMN IF EXISTS new_col; -- verify column is dropped -\d test_schema_support.nation_hash; +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash'::regclass; \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; \c - - - :master_port --test with search_path is set @@ -439,9 +439,9 @@ SET search_path TO test_schema_support; ALTER TABLE nation_hash ADD COLUMN new_col INT; -- verify column is added -\d test_schema_support.nation_hash; +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash'::regclass; \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; \c - - - :master_port SET search_path TO test_schema_support; @@ -449,9 +449,9 @@ ALTER TABLE nation_hash DROP COLUMN IF EXISTS non_existent_column; ALTER TABLE nation_hash DROP COLUMN IF EXISTS new_col; -- verify column is dropped -\d test_schema_support.nation_hash; +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash'::regclass; \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +SELECT "Column", "Type", "Modifiers" FROM public.table_desc WHERE relid='test_schema_support.nation_hash_1190003'::regclass; \c - - - :master_port @@ -462,18 +462,18 @@ SET search_path TO public; CREATE INDEX index1 ON test_schema_support.nation_hash(n_name); --verify INDEX is created -\d test_schema_support.nation_hash; +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +\d test_schema_support.index1_1190003 \c - - - :master_port -- DROP index DROP INDEX test_schema_support.index1; --verify INDEX is dropped -\d test_schema_support.nation_hash; +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +\d test_schema_support.index1_1190003 \c - - - :master_port --test with search_path is set @@ -483,9 +483,9 @@ SET search_path TO test_schema_support; CREATE INDEX index1 ON nation_hash(n_name); --verify INDEX is created -\d test_schema_support.nation_hash; +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +\d test_schema_support.index1_1190003 \c - - - :master_port -- DROP index @@ -493,9 +493,9 @@ SET search_path TO test_schema_support; DROP INDEX index1; --verify INDEX is dropped -\d test_schema_support.nation_hash; +\d test_schema_support.index1 \c - - - :worker_1_port -\d test_schema_support.nation_hash_1190003; +\d test_schema_support.index1_1190003 \c - - - :master_port diff --git a/src/test/regress/sql/multi_task_assignment_policy.sql b/src/test/regress/sql/multi_task_assignment_policy.sql index 28f8c57ed..74253d36e 100644 --- a/src/test/regress/sql/multi_task_assignment_policy.sql +++ b/src/test/regress/sql/multi_task_assignment_policy.sql @@ -5,6 +5,11 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 880000; +-- print major version to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+') AS major_version; + + SET citus.explain_distributed_queries TO off; diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql new file mode 100644 index 000000000..d63ab6529 --- /dev/null +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -0,0 +1,83 @@ +-- File to create functions and helpers needed for subsequent tests + +-- create a helper function to create objects on each node +CREATE FUNCTION run_command_on_master_and_workers(p_sql text) +RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE p_sql; + PERFORM run_command_on_workers(p_sql); +END;$$; + +-- The following views are intended as alternatives to \d commands, whose +-- output changed in PostgreSQL 10. In particular, they must be used any time +-- a test wishes to print out the structure of a relation, which previously +-- was safely accomplished by a \d invocation. +SELECT run_command_on_master_and_workers( +$desc_views$ +CREATE VIEW table_fkey_cols AS +SELECT rc.constraint_name AS "name", + kcu.column_name AS "column_name", + uc_kcu.column_name AS "refd_column_name", + format('%I.%I', kcu.table_schema, kcu.table_name)::regclass::oid AS relid, + format('%I.%I', uc_kcu.table_schema, uc_kcu.table_name)::regclass::oid AS refd_relid +FROM information_schema.referential_constraints rc, + information_schema.key_column_usage kcu, + information_schema.key_column_usage uc_kcu +WHERE rc.constraint_schema = kcu.constraint_schema AND + rc.constraint_name = kcu.constraint_name AND + rc.unique_constraint_schema = uc_kcu.constraint_schema AND + rc.unique_constraint_name = uc_kcu.constraint_name; + +CREATE VIEW table_fkeys AS +SELECT name AS "Constraint", + format('FOREIGN KEY (%s) REFERENCES %s(%s)', + string_agg(DISTINCT quote_ident(column_name), ', '), + string_agg(DISTINCT refd_relid::regclass::text, ', '), + string_agg(DISTINCT quote_ident(refd_column_name), ', ')) AS "Definition", + "relid" +FROM table_fkey_cols +GROUP BY (name, relid); + +CREATE VIEW table_attrs AS +SELECT c.column_name AS "name", + c.data_type AS "type", + CASE + WHEN character_maximum_length IS NOT NULL THEN + format('(%s)', character_maximum_length) + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL THEN + format('(%s,%s)', numeric_precision, numeric_scale) + ELSE '' + END AS "modifier", + c.column_default AS "default", + (NOT c.is_nullable::boolean) AS "notnull", + format('%I.%I', c.table_schema, c.table_name)::regclass::oid AS "relid" +FROM information_schema.columns AS c +ORDER BY ordinal_position; + +CREATE VIEW table_desc AS +SELECT "name" AS "Column", + "type" || "modifier" AS "Type", + rtrim(( + CASE "notnull" + WHEN true THEN 'not null ' + ELSE '' + END + ) || ( + CASE WHEN "default" IS NULL THEN '' + ELSE 'default ' || "default" + END + )) AS "Modifiers", + "relid" +FROM table_attrs; + +CREATE VIEW table_checks AS +SELECT cc.constraint_name AS "Constraint", + ('CHECK ' || regexp_replace(check_clause, '^\((.*)\)$', '\1')) AS "Definition", + format('%I.%I', ccu.table_schema, ccu.table_name)::regclass::oid AS relid +FROM information_schema.check_constraints cc, + information_schema.constraint_column_usage ccu +WHERE cc.constraint_schema = ccu.constraint_schema AND + cc.constraint_name = ccu.constraint_name +ORDER BY cc.constraint_name ASC; +$desc_views$ +); diff --git a/src/test/regress/sql/multi_transactional_drop_shards.sql b/src/test/regress/sql/multi_transactional_drop_shards.sql index 193d46fad..43912640a 100644 --- a/src/test/regress/sql/multi_transactional_drop_shards.sql +++ b/src/test/regress/sql/multi_transactional_drop_shards.sql @@ -28,11 +28,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is not dropped -\d transactional_drop_shards; +\dt transactional_drop_shards -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -53,11 +53,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is dropped -\d transactional_drop_shards; +\dt transactional_drop_shards -- verify shards are dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -83,7 +83,7 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -105,7 +105,7 @@ ORDER BY -- verify shards are dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -129,11 +129,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is not dropped -\d transactional_drop_shards; +\dt transactional_drop_shards -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -156,7 +156,7 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* -- test DROP table with failing worker @@ -186,11 +186,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is not dropped -\d transactional_drop_shards; +\dt transactional_drop_shards -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* \c - - - :master_port @@ -214,11 +214,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is not dropped -\d transactional_drop_reference; +\dt transactional_drop_reference -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_reference*; +\dt transactional_drop_reference* \c - - - :master_port @@ -240,7 +240,7 @@ ORDER BY -- verify shards are not dropped \c - - - :worker_1_port -\d transactional_drop_shards_*; +\dt transactional_drop_shards_* DROP EVENT TRIGGER fail_drop_table; \c - - - :master_port @@ -267,11 +267,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is not dropped -\d transactional_drop_serial; +\dt transactional_drop_serial -- verify shards and sequence are not dropped \c - - - :worker_1_port -\d transactional_drop_serial_1410006; +\dt transactional_drop_serial_* \ds transactional_drop_serial_column2_seq \c - - - :master_port @@ -293,11 +293,11 @@ ORDER BY shardid, nodename, nodeport; -- verify table is dropped -\d transactional_drop_serial; +\dt transactional_drop_serial -- verify shards and sequence are dropped \c - - - :worker_1_port -\d transactional_drop_serial_1410006; +\dt transactional_drop_serial_* \ds transactional_drop_serial_column2_seq \c - - - :master_port diff --git a/src/test/regress/sql/multi_unsupported_worker_operations.sql b/src/test/regress/sql/multi_unsupported_worker_operations.sql index c98b233cd..dcd172171 100644 --- a/src/test/regress/sql/multi_unsupported_worker_operations.sql +++ b/src/test/regress/sql/multi_unsupported_worker_operations.sql @@ -94,11 +94,12 @@ CREATE UNIQUE INDEX mx_test_uniq_index ON mx_table(col_1); \c - - - :worker_1_port -- DDL commands -\d mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; CREATE INDEX mx_test_index ON mx_table(col_2); ALTER TABLE mx_table ADD COLUMN col_4 int; ALTER TABLE mx_table_2 ADD CONSTRAINT mx_fk_constraint FOREIGN KEY(col_1) REFERENCES mx_table(col_1); -\d mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; +\d mx_test_index -- master_modify_multiple_shards SELECT master_modify_multiple_shards('UPDATE mx_table SET col_2=''none'''); @@ -217,9 +218,9 @@ DROP SEQUENCE some_sequence; -- Show that dropping the sequence of an MX table with cascade harms the table and shards BEGIN; -\d mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; DROP SEQUENCE mx_table_col_3_seq CASCADE; -\d mx_table +SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.mx_table'::regclass; ROLLBACK; -- Cleanup diff --git a/src/test/regress/sql/multi_upgrade_reference_table.sql b/src/test/regress/sql/multi_upgrade_reference_table.sql index d41b111b5..4c6d6372d 100644 --- a/src/test/regress/sql/multi_upgrade_reference_table.sql +++ b/src/test/regress/sql/multi_upgrade_reference_table.sql @@ -513,7 +513,7 @@ ORDER BY -- verify that shard is replicated to other worker \c - - - :worker_2_port -\d upgrade_reference_table_transaction_commit_* +\dt upgrade_reference_table_transaction_commit_* \c - - - :master_port DROP TABLE upgrade_reference_table_transaction_commit;