From 2eaf7bba6969475e609fcfe75fe9a1b7e60c23c3 Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Fri, 13 Mar 2020 12:18:31 +0300 Subject: [PATCH] not use local copy if we are copying into intermediate results file We have special logic to copy into intermediate results and we use a custom format for that, "result" copy format. Postgres internally does not know this format and if we use this locally it will error saying that it does not know this format. Files are visible to all transactions, which means that we can use any connection to access files. In order to use the existing logic, it makes sense that in case we have intermediate results, which means we will write the results to a file, we preserve the same behavior, which is opening connections to localhost. Therefore if we have intermediate results we return false in ShouldExecuteCopyLocally. --- .../distributed/commands/local_multi_copy.c | 2 +- src/backend/distributed/commands/multi_copy.c | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/commands/local_multi_copy.c b/src/backend/distributed/commands/local_multi_copy.c index efa8aa954..60eaef3ce 100644 --- a/src/backend/distributed/commands/local_multi_copy.c +++ b/src/backend/distributed/commands/local_multi_copy.c @@ -88,6 +88,7 @@ WriteTupleToLocalShard(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, in DoLocalCopy(localCopyOutState->fe_msgbuf, copyDest->distributedRelationId, shardId, copyDest->copyStatement, isEndOfCopy); + resetStringInfo(localCopyOutState->fe_msgbuf); } } @@ -173,7 +174,6 @@ DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStat heap_close(shard, NoLock); free_parsestate(pState); - resetStringInfo(buffer); } diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 34852becb..0c17a32e4 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -291,7 +291,7 @@ static void CitusCopyDestReceiverDestroy(DestReceiver *destReceiver); static bool ContainsLocalPlacement(int64 shardId); static void FinishLocalCopy(CitusCopyDestReceiver *copyDest); static void CloneCopyOutStateForLocalCopy(CopyOutState from, CopyOutState to); -static bool ShouldExecuteCopyLocally(void); +static bool ShouldExecuteCopyLocally(bool isIntermediateResult); static void LogLocalCopyExecution(uint64 shardId); @@ -1984,8 +1984,6 @@ CreateCitusCopyDestReceiver(Oid tableId, List *columnNameList, int partitionColu CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) palloc0( sizeof(CitusCopyDestReceiver)); - copyDest->shouldUseLocalCopy = ShouldExecuteCopyLocally(); - /* set up the DestReceiver function pointers */ copyDest->pub.receiveSlot = CitusCopyDestReceiverReceive; copyDest->pub.rStartup = CitusCopyDestReceiverStartup; @@ -2011,13 +2009,23 @@ CreateCitusCopyDestReceiver(Oid tableId, List *columnNameList, int partitionColu * operation should be done locally for local placements. */ static bool -ShouldExecuteCopyLocally() +ShouldExecuteCopyLocally(bool isIntermediateResult) { if (!EnableLocalExecution) { return false; } + /* + * Intermediate files are written to a file, and files are visible to all + * transactions, and we use a custom copy format for copy therefore we will + * use the existing logic for that. + */ + if (isIntermediateResult) + { + return false; + } + if (TransactionAccessedLocalPlacement) { /* @@ -2056,6 +2064,8 @@ CitusCopyDestReceiverStartup(DestReceiver *dest, int operation, { CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) dest; + bool isIntermediateResult = copyDest->intermediateResultIdPrefix != NULL; + copyDest->shouldUseLocalCopy = ShouldExecuteCopyLocally(isIntermediateResult); Oid tableId = copyDest->distributedRelationId; char *relationName = get_rel_name(tableId);