mirror of https://github.com/citusdata/citus.git
Merge pull request #327 from citusdata/apply_formatting_tool
Apply Citus Data coding conventions cr: @sumedhpathak @anarazelpull/335/head
commit
2fcfa011ae
|
@ -20,3 +20,23 @@ src/test/regress/output/*.source -whitespace
|
||||||
|
|
||||||
# These files are maintained or generated elsewhere. We take them as is.
|
# These files are maintained or generated elsewhere. We take them as is.
|
||||||
configure -whitespace
|
configure -whitespace
|
||||||
|
|
||||||
|
# all C files (implementation and header) use our style...
|
||||||
|
*.[ch] citus-style
|
||||||
|
|
||||||
|
# except these exceptions...
|
||||||
|
src/backend/distributed/utils/citus_outfuncs.c -citus-style
|
||||||
|
src/backend/distributed/utils/citus_read.c -citus-style
|
||||||
|
src/backend/distributed/utils/citus_readfuncs_94.c -citus-style
|
||||||
|
src/backend/distributed/utils/citus_readfuncs_95.c -citus-style
|
||||||
|
src/backend/distributed/utils/ruleutils_94.c -citus-style
|
||||||
|
src/backend/distributed/utils/ruleutils_95.c -citus-style
|
||||||
|
src/include/distributed/citus_nodes.h -citus-style
|
||||||
|
src/include/dumputils.h -citus-style
|
||||||
|
|
||||||
|
# all csql files use PostgreSQL style...
|
||||||
|
src/bin/csql/*.[ch] -citus-style
|
||||||
|
|
||||||
|
# except these exceptions
|
||||||
|
src/bin/csql/copy_options.c citus-style
|
||||||
|
src/bin/csql/stage.[ch] citus-style
|
||||||
|
|
7
Makefile
7
Makefile
|
@ -42,6 +42,13 @@ clean-csql:
|
||||||
install: install-csql
|
install: install-csql
|
||||||
clean: clean-csql
|
clean: clean-csql
|
||||||
|
|
||||||
|
# apply or check style
|
||||||
|
reindent:
|
||||||
|
cd ${citusdb_abs_top_srcdir} && citus_indent --quiet
|
||||||
|
check-style:
|
||||||
|
cd ${citusdb_abs_top_srcdir} && citus_indent --quiet --check
|
||||||
|
.PHONY: reindent check-style
|
||||||
|
|
||||||
# depend on install for now
|
# depend on install for now
|
||||||
check: all install
|
check: all install
|
||||||
$(MAKE) -C src/test/regress check-full
|
$(MAKE) -C src/test/regress check-full
|
||||||
|
|
|
@ -271,17 +271,27 @@ ReceiveCopyData(StringInfo copyData)
|
||||||
switch (messageType)
|
switch (messageType)
|
||||||
{
|
{
|
||||||
case 'd': /* CopyData */
|
case 'd': /* CopyData */
|
||||||
|
{
|
||||||
copyDone = false;
|
copyDone = false;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'c': /* CopyDone */
|
case 'c': /* CopyDone */
|
||||||
|
{
|
||||||
copyDone = true;
|
copyDone = true;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'f': /* CopyFail */
|
case 'f': /* CopyFail */
|
||||||
|
{
|
||||||
ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED),
|
ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED),
|
||||||
errmsg("COPY data failed: %s", pq_getmsgstring(copyData))));
|
errmsg("COPY data failed: %s", pq_getmsgstring(copyData))));
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'H': /* Flush */
|
case 'H': /* Flush */
|
||||||
case 'S': /* Sync */
|
case 'S': /* Sync */
|
||||||
|
{
|
||||||
/*
|
/*
|
||||||
* Ignore Flush/Sync for the convenience of client libraries (such
|
* Ignore Flush/Sync for the convenience of client libraries (such
|
||||||
* as libpq) that may send those without noticing that the command
|
* as libpq) that may send those without noticing that the command
|
||||||
|
@ -289,12 +299,16 @@ ReceiveCopyData(StringInfo copyData)
|
||||||
*/
|
*/
|
||||||
copyDone = false;
|
copyDone = false;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
{
|
||||||
ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION),
|
ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION),
|
||||||
errmsg("unexpected message type 0x%02X during COPY data",
|
errmsg("unexpected message type 0x%02X during COPY data",
|
||||||
messageType)));
|
messageType)));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return copyDone;
|
return copyDone;
|
||||||
}
|
}
|
||||||
|
|
|
@ -706,7 +706,7 @@ ClientConnectionReady(PGconn *connection, PostgresPollingStatusType pollingStatu
|
||||||
fd_set readFileDescriptorSet;
|
fd_set readFileDescriptorSet;
|
||||||
fd_set writeFileDescriptorSet;
|
fd_set writeFileDescriptorSet;
|
||||||
fd_set exceptionFileDescriptorSet;
|
fd_set exceptionFileDescriptorSet;
|
||||||
struct timeval immediateTimeout = {0, 0};
|
struct timeval immediateTimeout = { 0, 0 };
|
||||||
int connectionFileDescriptor = PQsocket(connection);
|
int connectionFileDescriptor = PQsocket(connection);
|
||||||
|
|
||||||
FD_ZERO(&readFileDescriptorSet);
|
FD_ZERO(&readFileDescriptorSet);
|
||||||
|
|
|
@ -157,7 +157,6 @@ multi_ExecutorStart(QueryDesc *queryDesc, int eflags)
|
||||||
queryDesc->plannedstmt = masterSelectPlan;
|
queryDesc->plannedstmt = masterSelectPlan;
|
||||||
eflags |= EXEC_FLAG_CITUS_MASTER_SELECT;
|
eflags |= EXEC_FLAG_CITUS_MASTER_SELECT;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* if the execution is not done for router executor, drop into standard executor */
|
/* if the execution is not done for router executor, drop into standard executor */
|
||||||
|
@ -253,7 +252,7 @@ multi_ExecutorEnd(QueryDesc *queryDesc)
|
||||||
RangeTblEntry *rangeTableEntry = linitial(planStatement->rtable);
|
RangeTblEntry *rangeTableEntry = linitial(planStatement->rtable);
|
||||||
Oid masterTableRelid = rangeTableEntry->relid;
|
Oid masterTableRelid = rangeTableEntry->relid;
|
||||||
|
|
||||||
ObjectAddress masterTableObject = {InvalidOid, InvalidOid, 0};
|
ObjectAddress masterTableObject = { InvalidOid, InvalidOid, 0 };
|
||||||
|
|
||||||
masterTableObject.classId = RelationRelationId;
|
masterTableObject.classId = RelationRelationId;
|
||||||
masterTableObject.objectId = masterTableRelid;
|
masterTableObject.objectId = masterTableRelid;
|
||||||
|
|
|
@ -89,7 +89,7 @@ MultiRealTimeExecute(Job *job)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* loop around until all tasks complete, one task fails, or user cancels */
|
/* loop around until all tasks complete, one task fails, or user cancels */
|
||||||
while ( !(allTasksCompleted || taskFailed || QueryCancelPending) )
|
while (!(allTasksCompleted || taskFailed || QueryCancelPending))
|
||||||
{
|
{
|
||||||
uint32 taskCount = list_length(taskList);
|
uint32 taskCount = list_length(taskList);
|
||||||
uint32 completedTaskCount = 0;
|
uint32 completedTaskCount = 0;
|
||||||
|
@ -287,8 +287,9 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution)
|
||||||
uint32 currentCount = taskExecution->connectPollCount;
|
uint32 currentCount = taskExecution->connectPollCount;
|
||||||
if (currentCount >= maxCount)
|
if (currentCount >= maxCount)
|
||||||
{
|
{
|
||||||
ereport(WARNING, (errmsg("could not establish asynchronous connection "
|
ereport(WARNING, (errmsg("could not establish asynchronous "
|
||||||
"after %u ms", REMOTE_NODE_CONNECT_TIMEOUT)));
|
"connection after %u ms",
|
||||||
|
REMOTE_NODE_CONNECT_TIMEOUT)));
|
||||||
|
|
||||||
taskStatusArray[currentIndex] = EXEC_TASK_FAILED;
|
taskStatusArray[currentIndex] = EXEC_TASK_FAILED;
|
||||||
}
|
}
|
||||||
|
@ -342,7 +343,8 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution)
|
||||||
{
|
{
|
||||||
List *dataFetchTaskList = task->dependedTaskList;
|
List *dataFetchTaskList = task->dependedTaskList;
|
||||||
int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex;
|
int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex;
|
||||||
Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, dataFetchTaskIndex);
|
Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList,
|
||||||
|
dataFetchTaskIndex);
|
||||||
|
|
||||||
char *dataFetchQuery = dataFetchTask->queryString;
|
char *dataFetchQuery = dataFetchTask->queryString;
|
||||||
int32 connectionId = connectionIdArray[currentIndex];
|
int32 connectionId = connectionIdArray[currentIndex];
|
||||||
|
@ -411,11 +413,13 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution)
|
||||||
StringInfo computeTaskQuery = makeStringInfo();
|
StringInfo computeTaskQuery = makeStringInfo();
|
||||||
if (BinaryMasterCopyFormat)
|
if (BinaryMasterCopyFormat)
|
||||||
{
|
{
|
||||||
appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY, queryString);
|
appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY,
|
||||||
|
queryString);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT, queryString);
|
appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT,
|
||||||
|
queryString);
|
||||||
}
|
}
|
||||||
|
|
||||||
querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data);
|
querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data);
|
||||||
|
@ -475,7 +479,8 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution)
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ereport(WARNING, (errcode_for_file_access(),
|
ereport(WARNING, (errcode_for_file_access(),
|
||||||
errmsg("could not open file \"%s\": %m", filename)));
|
errmsg("could not open file \"%s\": %m",
|
||||||
|
filename)));
|
||||||
|
|
||||||
taskStatusArray[currentIndex] = EXEC_TASK_FAILED;
|
taskStatusArray[currentIndex] = EXEC_TASK_FAILED;
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,6 +80,7 @@ RouterExecutorStart(QueryDesc *queryDesc, int eflags, Task *task)
|
||||||
queryDesc->estate = executorState;
|
queryDesc->estate = executorState;
|
||||||
|
|
||||||
#if (PG_VERSION_NUM < 90500)
|
#if (PG_VERSION_NUM < 90500)
|
||||||
|
|
||||||
/* make sure that upsertQuery is false for versions that UPSERT is not available */
|
/* make sure that upsertQuery is false for versions that UPSERT is not available */
|
||||||
Assert(task->upsertQuery == false);
|
Assert(task->upsertQuery == false);
|
||||||
#endif
|
#endif
|
||||||
|
@ -219,9 +220,9 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas
|
||||||
}
|
}
|
||||||
|
|
||||||
MemoryContextSwitchTo(oldcontext);
|
MemoryContextSwitchTo(oldcontext);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* ExecuteDistributedModify is the main entry point for modifying distributed
|
* ExecuteDistributedModify is the main entry point for modifying distributed
|
||||||
* tables. A distributed modification is successful if any placement of the
|
* tables. A distributed modification is successful if any placement of the
|
||||||
|
@ -532,9 +533,10 @@ StoreQueryResult(PGconn *connection, TupleDesc tupleDescriptor,
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* RouterExecutorFinish cleans up after a distributed execution.
|
* RouterExecutorFinish cleans up after a distributed execution.
|
||||||
*/
|
*/
|
||||||
void
|
void
|
||||||
RouterExecutorFinish(QueryDesc *queryDesc)
|
RouterExecutorFinish(QueryDesc *queryDesc)
|
||||||
{
|
{
|
||||||
|
|
|
@ -41,7 +41,6 @@ typedef struct TaskMapKey
|
||||||
TaskType taskType;
|
TaskType taskType;
|
||||||
uint64 jobId;
|
uint64 jobId;
|
||||||
uint32 taskId;
|
uint32 taskId;
|
||||||
|
|
||||||
} TaskMapKey;
|
} TaskMapKey;
|
||||||
|
|
||||||
|
|
||||||
|
@ -53,7 +52,6 @@ typedef struct TaskMapEntry
|
||||||
{
|
{
|
||||||
TaskMapKey key;
|
TaskMapKey key;
|
||||||
Task *task;
|
Task *task;
|
||||||
|
|
||||||
} TaskMapEntry;
|
} TaskMapEntry;
|
||||||
|
|
||||||
|
|
||||||
|
@ -83,7 +81,8 @@ static TaskTracker * TrackerHashLookup(HTAB *trackerHash, const char *nodeName,
|
||||||
static TaskExecStatus ManageTaskExecution(TaskTracker *taskTracker,
|
static TaskExecStatus ManageTaskExecution(TaskTracker *taskTracker,
|
||||||
TaskTracker *sourceTaskTracker,
|
TaskTracker *sourceTaskTracker,
|
||||||
Task *task, TaskExecution *taskExecution);
|
Task *task, TaskExecution *taskExecution);
|
||||||
static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker, Task *task,
|
static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker,
|
||||||
|
Task *task,
|
||||||
TaskExecution *taskExecution);
|
TaskExecution *taskExecution);
|
||||||
static bool TaskExecutionsCompleted(List *taskList);
|
static bool TaskExecutionsCompleted(List *taskList);
|
||||||
static StringInfo MapFetchTaskQueryString(Task *mapFetchTask, Task *mapTask);
|
static StringInfo MapFetchTaskQueryString(Task *mapFetchTask, Task *mapTask);
|
||||||
|
@ -194,8 +193,8 @@ MultiTaskTrackerExecute(Job *job)
|
||||||
TrackerHashConnect(transmitTrackerHash);
|
TrackerHashConnect(transmitTrackerHash);
|
||||||
|
|
||||||
/* loop around until all tasks complete, one task fails, or user cancels */
|
/* loop around until all tasks complete, one task fails, or user cancels */
|
||||||
while ( !(allTasksCompleted || taskFailed || taskTransmitFailed ||
|
while (!(allTasksCompleted || taskFailed || taskTransmitFailed ||
|
||||||
clusterFailed || QueryCancelPending) )
|
clusterFailed || QueryCancelPending))
|
||||||
{
|
{
|
||||||
TaskTracker *taskTracker = NULL;
|
TaskTracker *taskTracker = NULL;
|
||||||
TaskTracker *transmitTracker = NULL;
|
TaskTracker *transmitTracker = NULL;
|
||||||
|
@ -826,7 +825,8 @@ TrackerConnectPoll(TaskTracker *taskTracker)
|
||||||
uint32 nodePort = taskTracker->workerPort;
|
uint32 nodePort = taskTracker->workerPort;
|
||||||
char *nodeDatabase = get_database_name(MyDatabaseId);
|
char *nodeDatabase = get_database_name(MyDatabaseId);
|
||||||
|
|
||||||
int32 connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase);
|
int32 connectionId = MultiClientConnectStart(nodeName, nodePort,
|
||||||
|
nodeDatabase);
|
||||||
if (connectionId != INVALID_CONNECTION_ID)
|
if (connectionId != INVALID_CONNECTION_ID)
|
||||||
{
|
{
|
||||||
taskTracker->connectionId = connectionId;
|
taskTracker->connectionId = connectionId;
|
||||||
|
@ -869,8 +869,9 @@ TrackerConnectPoll(TaskTracker *taskTracker)
|
||||||
uint32 currentCount = taskTracker->connectPollCount;
|
uint32 currentCount = taskTracker->connectPollCount;
|
||||||
if (currentCount >= maxCount)
|
if (currentCount >= maxCount)
|
||||||
{
|
{
|
||||||
ereport(WARNING, (errmsg("could not establish asynchronous connection "
|
ereport(WARNING, (errmsg("could not establish asynchronous "
|
||||||
"after %u ms", REMOTE_NODE_CONNECT_TIMEOUT)));
|
"connection after %u ms",
|
||||||
|
REMOTE_NODE_CONNECT_TIMEOUT)));
|
||||||
|
|
||||||
taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED;
|
taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED;
|
||||||
|
|
||||||
|
@ -1212,7 +1213,8 @@ ManageTaskExecution(TaskTracker *taskTracker, TaskTracker *sourceTaskTracker,
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
/* we fatal here to avoid leaking client-side resources */
|
/* we fatal here to avoid leaking client-side resources */
|
||||||
ereport(FATAL, (errmsg("invalid execution status: %d", currentExecutionStatus)));
|
ereport(FATAL, (errmsg("invalid execution status: %d",
|
||||||
|
currentExecutionStatus)));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1327,7 +1329,8 @@ ManageTransmitExecution(TaskTracker *transmitTracker,
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ereport(WARNING, (errcode_for_file_access(),
|
ereport(WARNING, (errcode_for_file_access(),
|
||||||
errmsg("could not open file \"%s\": %m", filename)));
|
errmsg("could not open file \"%s\": %m",
|
||||||
|
filename)));
|
||||||
|
|
||||||
nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY;
|
nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY;
|
||||||
}
|
}
|
||||||
|
@ -1463,7 +1466,8 @@ ManageTransmitExecution(TaskTracker *transmitTracker,
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
/* we fatal here to avoid leaking client-side resources */
|
/* we fatal here to avoid leaking client-side resources */
|
||||||
ereport(FATAL, (errmsg("invalid transmit status: %d", currentTransmitStatus)));
|
ereport(FATAL, (errmsg("invalid transmit status: %d",
|
||||||
|
currentTransmitStatus)));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -2317,7 +2321,7 @@ AssignQueuedTasks(TaskTracker *taskTracker)
|
||||||
{
|
{
|
||||||
StringInfo taskAssignmentQuery = taskState->taskAssignmentQuery;
|
StringInfo taskAssignmentQuery = taskState->taskAssignmentQuery;
|
||||||
|
|
||||||
if(taskAssignmentCount > 0)
|
if (taskAssignmentCount > 0)
|
||||||
{
|
{
|
||||||
appendStringInfo(multiAssignQuery, ";");
|
appendStringInfo(multiAssignQuery, ";");
|
||||||
}
|
}
|
||||||
|
@ -2336,7 +2340,7 @@ AssignQueuedTasks(TaskTracker *taskTracker)
|
||||||
taskState = (TrackerTaskState *) hash_seq_search(&status);
|
taskState = (TrackerTaskState *) hash_seq_search(&status);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(taskAssignmentCount > 0)
|
if (taskAssignmentCount > 0)
|
||||||
{
|
{
|
||||||
void *queryResult = NULL;
|
void *queryResult = NULL;
|
||||||
int rowCount = 0;
|
int rowCount = 0;
|
||||||
|
@ -2833,7 +2837,8 @@ TrackerHashCleanupJob(HTAB *taskTrackerHash, Task *jobCleanupTask)
|
||||||
if (queryStatus == CLIENT_QUERY_DONE)
|
if (queryStatus == CLIENT_QUERY_DONE)
|
||||||
{
|
{
|
||||||
ereport(DEBUG4, (errmsg("completed cleanup query for job " UINT64_FORMAT
|
ereport(DEBUG4, (errmsg("completed cleanup query for job " UINT64_FORMAT
|
||||||
" on node \"%s:%u\"", jobId, nodeName, nodePort)));
|
" on node \"%s:%u\"", jobId, nodeName,
|
||||||
|
nodePort)));
|
||||||
|
|
||||||
/* clear connection for future cleanup queries */
|
/* clear connection for future cleanup queries */
|
||||||
taskTracker->connectionBusy = false;
|
taskTracker->connectionBusy = false;
|
||||||
|
|
|
@ -757,7 +757,7 @@ IsAlterTableRenameStmt(RenameStmt *renameStmt)
|
||||||
isAlterTableRenameStmt = true;
|
isAlterTableRenameStmt = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if (PG_VERSION_NUM >=90500)
|
#if (PG_VERSION_NUM >= 90500)
|
||||||
else if (renameStmt->renameType == OBJECT_TABCONSTRAINT)
|
else if (renameStmt->renameType == OBJECT_TABCONSTRAINT)
|
||||||
{
|
{
|
||||||
isAlterTableRenameStmt = true;
|
isAlterTableRenameStmt = true;
|
||||||
|
@ -905,8 +905,9 @@ ExecuteCommandOnWorkerShards(Oid relationId, const char *commandString,
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT " on "
|
ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT
|
||||||
"node %s:%d", shardId, workerName, workerPort)));
|
" on node %s:%d", shardId, workerName,
|
||||||
|
workerPort)));
|
||||||
}
|
}
|
||||||
|
|
||||||
isFirstPlacement = false;
|
isFirstPlacement = false;
|
||||||
|
@ -988,6 +989,7 @@ AllFinalizedPlacementsAccessible(Oid relationId)
|
||||||
static void
|
static void
|
||||||
RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg)
|
RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg)
|
||||||
{
|
{
|
||||||
|
/* *INDENT-OFF* */
|
||||||
HeapTuple tuple;
|
HeapTuple tuple;
|
||||||
struct DropRelationCallbackState *state;
|
struct DropRelationCallbackState *state;
|
||||||
char relkind;
|
char relkind;
|
||||||
|
@ -1022,10 +1024,8 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi
|
||||||
classform = (Form_pg_class) GETSTRUCT(tuple);
|
classform = (Form_pg_class) GETSTRUCT(tuple);
|
||||||
|
|
||||||
if (classform->relkind != relkind)
|
if (classform->relkind != relkind)
|
||||||
{
|
|
||||||
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
|
||||||
errmsg("\"%s\" is not an index", rel->relname)));
|
errmsg("\"%s\" is not an index", rel->relname)));
|
||||||
}
|
|
||||||
|
|
||||||
/* Allow DROP to either table owner or schema owner */
|
/* Allow DROP to either table owner or schema owner */
|
||||||
if (!pg_class_ownercheck(relOid, GetUserId()) &&
|
if (!pg_class_ownercheck(relOid, GetUserId()) &&
|
||||||
|
@ -1054,4 +1054,5 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi
|
||||||
if (OidIsValid(state->heapOid))
|
if (OidIsValid(state->heapOid))
|
||||||
LockRelationOid(state->heapOid, heap_lockmode);
|
LockRelationOid(state->heapOid, heap_lockmode);
|
||||||
}
|
}
|
||||||
|
/* *INDENT-ON* */
|
||||||
}
|
}
|
||||||
|
|
|
@ -146,7 +146,7 @@ master_apply_delete_command(PG_FUNCTION_ARGS)
|
||||||
{
|
{
|
||||||
List *shardPlacementList = NIL;
|
List *shardPlacementList = NIL;
|
||||||
List *droppedPlacementList = NIL;
|
List *droppedPlacementList = NIL;
|
||||||
List *lingeringPlacementList= NIL;
|
List *lingeringPlacementList = NIL;
|
||||||
ListCell *shardPlacementCell = NULL;
|
ListCell *shardPlacementCell = NULL;
|
||||||
ListCell *droppedPlacementCell = NULL;
|
ListCell *droppedPlacementCell = NULL;
|
||||||
ListCell *lingeringPlacementCell = NULL;
|
ListCell *lingeringPlacementCell = NULL;
|
||||||
|
@ -167,7 +167,8 @@ master_apply_delete_command(PG_FUNCTION_ARGS)
|
||||||
shardPlacementList = ShardPlacementList(shardId);
|
shardPlacementList = ShardPlacementList(shardId);
|
||||||
foreach(shardPlacementCell, shardPlacementList)
|
foreach(shardPlacementCell, shardPlacementList)
|
||||||
{
|
{
|
||||||
ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell);
|
ShardPlacement *shardPlacement =
|
||||||
|
(ShardPlacement *) lfirst(shardPlacementCell);
|
||||||
char *workerName = shardPlacement->nodeName;
|
char *workerName = shardPlacement->nodeName;
|
||||||
uint32 workerPort = shardPlacement->nodePort;
|
uint32 workerPort = shardPlacement->nodePort;
|
||||||
bool dropSuccessful = false;
|
bool dropSuccessful = false;
|
||||||
|
@ -176,14 +177,17 @@ master_apply_delete_command(PG_FUNCTION_ARGS)
|
||||||
char tableType = get_rel_relkind(relationId);
|
char tableType = get_rel_relkind(relationId);
|
||||||
if (tableType == RELKIND_RELATION)
|
if (tableType == RELKIND_RELATION)
|
||||||
{
|
{
|
||||||
appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, quotedShardName);
|
appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND,
|
||||||
|
quotedShardName);
|
||||||
}
|
}
|
||||||
else if (tableType == RELKIND_FOREIGN_TABLE)
|
else if (tableType == RELKIND_FOREIGN_TABLE)
|
||||||
{
|
{
|
||||||
appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, quotedShardName);
|
appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND,
|
||||||
|
quotedShardName);
|
||||||
}
|
}
|
||||||
|
|
||||||
dropSuccessful = ExecuteRemoteCommand(workerName, workerPort, workerDropQuery);
|
dropSuccessful = ExecuteRemoteCommand(workerName, workerPort,
|
||||||
|
workerDropQuery);
|
||||||
if (dropSuccessful)
|
if (dropSuccessful)
|
||||||
{
|
{
|
||||||
droppedPlacementList = lappend(droppedPlacementList, shardPlacement);
|
droppedPlacementList = lappend(droppedPlacementList, shardPlacement);
|
||||||
|
@ -227,7 +231,8 @@ master_apply_delete_command(PG_FUNCTION_ARGS)
|
||||||
|
|
||||||
if (QueryCancelPending)
|
if (QueryCancelPending)
|
||||||
{
|
{
|
||||||
ereport(WARNING, (errmsg("cancel requests are ignored during shard deletion")));
|
ereport(WARNING, (errmsg("cancel requests are ignored during shard "
|
||||||
|
"deletion")));
|
||||||
QueryCancelPending = false;
|
QueryCancelPending = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -298,7 +303,7 @@ CheckDeleteCriteria(Node *deleteCriteria)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* CheckPartitionColumn checks that the given where clause is based only on the
|
* CheckPartitionColumn checks that the given where clause is based only on the
|
||||||
* partition key of the given relation id.
|
* partition key of the given relation id.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -709,11 +709,16 @@ hostname_client_addr(void)
|
||||||
#ifdef HAVE_IPV6
|
#ifdef HAVE_IPV6
|
||||||
case AF_INET6:
|
case AF_INET6:
|
||||||
#endif
|
#endif
|
||||||
|
{
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
{
|
||||||
ereport(ERROR, (errmsg("invalid address family in connection")));
|
ereport(ERROR, (errmsg("invalid address family in connection")));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
remoteHost = palloc0(remoteHostLen);
|
remoteHost = palloc0(remoteHostLen);
|
||||||
|
|
||||||
|
|
|
@ -393,6 +393,7 @@ DistributedModifyTask(Query *query)
|
||||||
query->onConflict = RebuildOnConflict(relationId, query->onConflict);
|
query->onConflict = RebuildOnConflict(relationId, query->onConflict);
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
|
|
||||||
/* always set to false for PG_VERSION_NUM < 90500 */
|
/* always set to false for PG_VERSION_NUM < 90500 */
|
||||||
upsertQuery = false;
|
upsertQuery = false;
|
||||||
#endif
|
#endif
|
||||||
|
@ -414,6 +415,7 @@ DistributedModifyTask(Query *query)
|
||||||
|
|
||||||
|
|
||||||
#if (PG_VERSION_NUM >= 90500)
|
#if (PG_VERSION_NUM >= 90500)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* RebuildOnConflict rebuilds OnConflictExpr for correct deparsing. The function
|
* RebuildOnConflict rebuilds OnConflictExpr for correct deparsing. The function
|
||||||
* makes WHERE clause elements explicit and filters dropped columns
|
* makes WHERE clause elements explicit and filters dropped columns
|
||||||
|
@ -448,7 +450,7 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict)
|
||||||
foreach(targetEntryCell, onConflictSet)
|
foreach(targetEntryCell, onConflictSet)
|
||||||
{
|
{
|
||||||
TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell);
|
TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell);
|
||||||
FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno -1];
|
FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno - 1];
|
||||||
|
|
||||||
/* skip dropped columns */
|
/* skip dropped columns */
|
||||||
if (tableAttribute->attisdropped)
|
if (tableAttribute->attisdropped)
|
||||||
|
@ -468,6 +470,8 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict)
|
||||||
|
|
||||||
return updatedOnConflict;
|
return updatedOnConflict;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -37,14 +37,14 @@ int LargeTableShardCount = 4; /* shard counts for a large table */
|
||||||
bool LogMultiJoinOrder = false; /* print join order as a debugging aid */
|
bool LogMultiJoinOrder = false; /* print join order as a debugging aid */
|
||||||
|
|
||||||
/* Function pointer type definition for join rule evaluation functions */
|
/* Function pointer type definition for join rule evaluation functions */
|
||||||
typedef JoinOrderNode * (*RuleEvalFunction) (JoinOrderNode *currentJoinNode,
|
typedef JoinOrderNode *(*RuleEvalFunction) (JoinOrderNode *currentJoinNode,
|
||||||
TableEntry *candidateTable,
|
TableEntry *candidateTable,
|
||||||
List *candidateShardList,
|
List *candidateShardList,
|
||||||
List *applicableJoinClauses,
|
List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
|
|
||||||
static char * RuleNameArray[JOIN_RULE_LAST] = {0}; /* ordered join rule names */
|
static char *RuleNameArray[JOIN_RULE_LAST] = { 0 }; /* ordered join rule names */
|
||||||
static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */
|
static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */
|
||||||
|
|
||||||
|
|
||||||
/* Local functions forward declarations */
|
/* Local functions forward declarations */
|
||||||
|
@ -54,7 +54,8 @@ static bool JoinExprListWalker(Node *node, List **joinList);
|
||||||
static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex);
|
static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex);
|
||||||
static List * MergeShardIntervals(List *leftShardIntervalList,
|
static List * MergeShardIntervals(List *leftShardIntervalList,
|
||||||
List *rightShardIntervalList, JoinType joinType);
|
List *rightShardIntervalList, JoinType joinType);
|
||||||
static bool ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList);
|
static bool ShardIntervalsMatch(List *leftShardIntervalList,
|
||||||
|
List *rightShardIntervalList);
|
||||||
static List * LoadSortedShardIntervalList(Oid relationId);
|
static List * LoadSortedShardIntervalList(Oid relationId);
|
||||||
static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList,
|
static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList,
|
||||||
List *joinClauseList);
|
List *joinClauseList);
|
||||||
|
@ -68,31 +69,41 @@ static List * TableEntryListDifference(List *lhsTableList, List *rhsTableList);
|
||||||
static TableEntry * FindTableEntry(List *tableEntryList, uint32 tableId);
|
static TableEntry * FindTableEntry(List *tableEntryList, uint32 tableId);
|
||||||
|
|
||||||
/* Local functions forward declarations for join evaluations */
|
/* Local functions forward declarations for join evaluations */
|
||||||
static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode,
|
static JoinOrderNode * EvaluateJoinRules(List *joinedTableList,
|
||||||
TableEntry *candidateTable, List *candidateShardList,
|
JoinOrderNode *currentJoinNode,
|
||||||
|
TableEntry *candidateTable,
|
||||||
|
List *candidateShardList,
|
||||||
List *joinClauseList, JoinType joinType);
|
List *joinClauseList, JoinType joinType);
|
||||||
static List * RangeTableIdList(List *tableList);
|
static List * RangeTableIdList(List *tableList);
|
||||||
static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType);
|
static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType);
|
||||||
static char * JoinRuleName(JoinRuleType ruleType);
|
static char * JoinRuleName(JoinRuleType ruleType);
|
||||||
static JoinOrderNode * BroadcastJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
static JoinOrderNode * BroadcastJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
||||||
List *candidateShardList, List *applicableJoinClauses,
|
List *candidateShardList,
|
||||||
|
List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
static JoinOrderNode * LocalJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
static JoinOrderNode * LocalJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
||||||
List *candidateShardList, List *applicableJoinClauses,
|
List *candidateShardList, List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
static bool JoinOnColumns(Var *currentPartitioncolumn, Var *candidatePartitionColumn,
|
static bool JoinOnColumns(Var *currentPartitioncolumn, Var *candidatePartitionColumn,
|
||||||
List *joinClauseList);
|
List *joinClauseList);
|
||||||
static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode,
|
||||||
List *candidateShardList, List *applicableJoinClauses,
|
TableEntry *candidateTable,
|
||||||
|
List *candidateShardList,
|
||||||
|
List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode,
|
||||||
List *candidateShardList, List *applicableJoinClauses,
|
TableEntry *candidateTable,
|
||||||
|
List *candidateShardList,
|
||||||
|
List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, TableEntry *candidateTable,
|
static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode,
|
||||||
List *candidateShardList, List *applicableJoinClauses,
|
TableEntry *candidateTable,
|
||||||
|
List *candidateShardList,
|
||||||
|
List *applicableJoinClauses,
|
||||||
JoinType joinType);
|
JoinType joinType);
|
||||||
static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType joinRuleType,
|
static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType
|
||||||
Var *partitionColumn, char partitionMethod);
|
joinRuleType, Var *partitionColumn,
|
||||||
|
char partitionMethod);
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -106,7 +117,7 @@ List *
|
||||||
FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList)
|
FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList)
|
||||||
{
|
{
|
||||||
List *joinList = NIL;
|
List *joinList = NIL;
|
||||||
ListCell * joinCell = NULL;
|
ListCell *joinCell = NULL;
|
||||||
List *joinWhereClauseList = NIL;
|
List *joinWhereClauseList = NIL;
|
||||||
List *joinOrderList = NIL;
|
List *joinOrderList = NIL;
|
||||||
List *joinedTableList = NIL;
|
List *joinedTableList = NIL;
|
||||||
|
@ -199,7 +210,6 @@ FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList)
|
||||||
"query"),
|
"query"),
|
||||||
errdetail("Shards of relations in outer join queries "
|
errdetail("Shards of relations in outer join queries "
|
||||||
"must have 1-to-1 shard partitioning")));
|
"must have 1-to-1 shard partitioning")));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -439,7 +449,7 @@ MergeShardIntervals(List *leftShardIntervalList, List *rightShardIntervalList,
|
||||||
bool nextMaxSmaller = comparisonResult > 0;
|
bool nextMaxSmaller = comparisonResult > 0;
|
||||||
|
|
||||||
if ((shardUnion && nextMaxLarger) ||
|
if ((shardUnion && nextMaxLarger) ||
|
||||||
(!shardUnion && nextMaxSmaller) )
|
(!shardUnion && nextMaxSmaller))
|
||||||
{
|
{
|
||||||
newShardInterval->maxValue = datumCopy(nextMax, typeByValue, typeLen);
|
newShardInterval->maxValue = datumCopy(nextMax, typeByValue, typeLen);
|
||||||
}
|
}
|
||||||
|
@ -586,7 +596,8 @@ ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList)
|
||||||
nextRightIntervalCell = lnext(rightShardIntervalCell);
|
nextRightIntervalCell = lnext(rightShardIntervalCell);
|
||||||
if (nextRightIntervalCell != NULL)
|
if (nextRightIntervalCell != NULL)
|
||||||
{
|
{
|
||||||
ShardInterval *nextRightInterval = (ShardInterval *) lfirst(nextRightIntervalCell);
|
ShardInterval *nextRightInterval =
|
||||||
|
(ShardInterval *) lfirst(nextRightIntervalCell);
|
||||||
shardIntervalsIntersect = ShardIntervalsOverlap(leftInterval,
|
shardIntervalsIntersect = ShardIntervalsOverlap(leftInterval,
|
||||||
nextRightInterval);
|
nextRightInterval);
|
||||||
if (shardIntervalsIntersect)
|
if (shardIntervalsIntersect)
|
||||||
|
@ -1028,7 +1039,7 @@ EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode,
|
||||||
JoinRuleType ruleType = (JoinRuleType) ruleIndex;
|
JoinRuleType ruleType = (JoinRuleType) ruleIndex;
|
||||||
RuleEvalFunction ruleEvalFunction = JoinRuleEvalFunction(ruleType);
|
RuleEvalFunction ruleEvalFunction = JoinRuleEvalFunction(ruleType);
|
||||||
|
|
||||||
nextJoinNode = (*ruleEvalFunction) (currentJoinNode,
|
nextJoinNode = (*ruleEvalFunction)(currentJoinNode,
|
||||||
candidateTable,
|
candidateTable,
|
||||||
candidateShardList,
|
candidateShardList,
|
||||||
applicableJoinClauses,
|
applicableJoinClauses,
|
||||||
|
|
|
@ -91,7 +91,8 @@ static void ParentSetNewChild(MultiNode *parentNode, MultiNode *oldChildNode,
|
||||||
|
|
||||||
/* Local functions forward declarations for aggregate expressions */
|
/* Local functions forward declarations for aggregate expressions */
|
||||||
static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode,
|
static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode,
|
||||||
MultiExtendedOp *masterNode, MultiExtendedOp *workerNode);
|
MultiExtendedOp *masterNode,
|
||||||
|
MultiExtendedOp *workerNode);
|
||||||
static void TransformSubqueryNode(MultiTable *subqueryNode);
|
static void TransformSubqueryNode(MultiTable *subqueryNode);
|
||||||
static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode);
|
static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode);
|
||||||
static Node * MasterAggregateMutator(Node *originalNode, AttrNumber *columnId);
|
static Node * MasterAggregateMutator(Node *originalNode, AttrNumber *columnId);
|
||||||
|
@ -117,7 +118,8 @@ static void ErrorIfUnsupportedArrayAggregate(Aggref *arrayAggregateExpression);
|
||||||
static void ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression,
|
static void ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression,
|
||||||
MultiNode *logicalPlanNode);
|
MultiNode *logicalPlanNode);
|
||||||
static Var * AggregateDistinctColumn(Aggref *aggregateExpression);
|
static Var * AggregateDistinctColumn(Aggref *aggregateExpression);
|
||||||
static bool TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode,
|
static bool TablePartitioningSupportsDistinct(List *tableNodeList,
|
||||||
|
MultiExtendedOp *opNode,
|
||||||
Var *distinctColumn);
|
Var *distinctColumn);
|
||||||
static bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column);
|
static bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column);
|
||||||
|
|
||||||
|
@ -1220,7 +1222,7 @@ MasterExtendedOpNode(MultiExtendedOp *originalOpNode)
|
||||||
bool hasAggregates = contain_agg_clause((Node *) originalExpression);
|
bool hasAggregates = contain_agg_clause((Node *) originalExpression);
|
||||||
if (hasAggregates)
|
if (hasAggregates)
|
||||||
{
|
{
|
||||||
Node *newNode = MasterAggregateMutator((Node*) originalExpression,
|
Node *newNode = MasterAggregateMutator((Node *) originalExpression,
|
||||||
&columnId);
|
&columnId);
|
||||||
newExpression = (Expr *) newNode;
|
newExpression = (Expr *) newNode;
|
||||||
}
|
}
|
||||||
|
@ -1980,23 +1982,31 @@ CountDistinctHashFunctionName(Oid argumentType)
|
||||||
switch (argumentType)
|
switch (argumentType)
|
||||||
{
|
{
|
||||||
case INT4OID:
|
case INT4OID:
|
||||||
|
{
|
||||||
hashFunctionName = pstrdup(HLL_HASH_INTEGER_FUNC_NAME);
|
hashFunctionName = pstrdup(HLL_HASH_INTEGER_FUNC_NAME);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case INT8OID:
|
case INT8OID:
|
||||||
|
{
|
||||||
hashFunctionName = pstrdup(HLL_HASH_BIGINT_FUNC_NAME);
|
hashFunctionName = pstrdup(HLL_HASH_BIGINT_FUNC_NAME);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case TEXTOID:
|
case TEXTOID:
|
||||||
case BPCHAROID:
|
case BPCHAROID:
|
||||||
case VARCHAROID:
|
case VARCHAROID:
|
||||||
|
{
|
||||||
hashFunctionName = pstrdup(HLL_HASH_TEXT_FUNC_NAME);
|
hashFunctionName = pstrdup(HLL_HASH_TEXT_FUNC_NAME);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
{
|
||||||
hashFunctionName = pstrdup(HLL_HASH_ANY_FUNC_NAME);
|
hashFunctionName = pstrdup(HLL_HASH_ANY_FUNC_NAME);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return hashFunctionName;
|
return hashFunctionName;
|
||||||
}
|
}
|
||||||
|
@ -2563,7 +2573,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit)
|
||||||
List *joinTreeTableIndexList = NIL;
|
List *joinTreeTableIndexList = NIL;
|
||||||
uint32 joiningTableCount = 0;
|
uint32 joiningTableCount = 0;
|
||||||
|
|
||||||
ExtractRangeTableIndexWalker((Node*) subqueryTree->jointree,
|
ExtractRangeTableIndexWalker((Node *) subqueryTree->jointree,
|
||||||
&joinTreeTableIndexList);
|
&joinTreeTableIndexList);
|
||||||
joiningTableCount = list_length(joinTreeTableIndexList);
|
joiningTableCount = list_length(joinTreeTableIndexList);
|
||||||
|
|
||||||
|
@ -2639,7 +2649,7 @@ ErrorIfUnsupportedTableCombination(Query *queryTree)
|
||||||
* Extract all range table indexes from the join tree. Note that sub-queries
|
* Extract all range table indexes from the join tree. Note that sub-queries
|
||||||
* that get pulled up by PostgreSQL don't appear in this join tree.
|
* that get pulled up by PostgreSQL don't appear in this join tree.
|
||||||
*/
|
*/
|
||||||
ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList);
|
ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList);
|
||||||
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
|
@ -3019,7 +3029,7 @@ FullCompositeFieldList(List *compositeFieldList)
|
||||||
uint32 fieldIndex = 0;
|
uint32 fieldIndex = 0;
|
||||||
|
|
||||||
ListCell *fieldSelectCell = NULL;
|
ListCell *fieldSelectCell = NULL;
|
||||||
foreach (fieldSelectCell, compositeFieldList)
|
foreach(fieldSelectCell, compositeFieldList)
|
||||||
{
|
{
|
||||||
FieldSelect *fieldSelect = (FieldSelect *) lfirst(fieldSelectCell);
|
FieldSelect *fieldSelect = (FieldSelect *) lfirst(fieldSelectCell);
|
||||||
uint32 compositeFieldIndex = 0;
|
uint32 compositeFieldIndex = 0;
|
||||||
|
@ -3229,6 +3239,7 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery)
|
||||||
CompositeFieldRecursive(outerQueryExpression, parentQuery);
|
CompositeFieldRecursive(outerQueryExpression, parentQuery);
|
||||||
FieldSelect *localCompositeField =
|
FieldSelect *localCompositeField =
|
||||||
CompositeFieldRecursive(localQueryExpression, lateralQuery);
|
CompositeFieldRecursive(localQueryExpression, lateralQuery);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If partition colums are composite fields, add them to list to
|
* If partition colums are composite fields, add them to list to
|
||||||
* check later if all composite fields are used.
|
* check later if all composite fields are used.
|
||||||
|
@ -3251,7 +3262,7 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* check composite fields */
|
/* check composite fields */
|
||||||
if(!supportedLateralQuery)
|
if (!supportedLateralQuery)
|
||||||
{
|
{
|
||||||
bool outerFullCompositeFieldList =
|
bool outerFullCompositeFieldList =
|
||||||
FullCompositeFieldList(outerCompositeFieldList);
|
FullCompositeFieldList(outerCompositeFieldList);
|
||||||
|
@ -3309,7 +3320,7 @@ JoinOnPartitionColumn(Query *query)
|
||||||
* If partition colums are composite fields, add them to list to
|
* If partition colums are composite fields, add them to list to
|
||||||
* check later if all composite fields are used.
|
* check later if all composite fields are used.
|
||||||
*/
|
*/
|
||||||
if(leftCompositeField && rightCompositeField)
|
if (leftCompositeField && rightCompositeField)
|
||||||
{
|
{
|
||||||
leftCompositeFieldList = lappend(leftCompositeFieldList,
|
leftCompositeFieldList = lappend(leftCompositeFieldList,
|
||||||
leftCompositeField);
|
leftCompositeField);
|
||||||
|
@ -3318,7 +3329,7 @@ JoinOnPartitionColumn(Query *query)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* if both sides are not composite fields, they are normal columns */
|
/* if both sides are not composite fields, they are normal columns */
|
||||||
if(!(leftCompositeField && rightCompositeField))
|
if (!(leftCompositeField && rightCompositeField))
|
||||||
{
|
{
|
||||||
joinOnPartitionColumn = true;
|
joinOnPartitionColumn = true;
|
||||||
break;
|
break;
|
||||||
|
@ -3327,7 +3338,7 @@ JoinOnPartitionColumn(Query *query)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* check composite fields */
|
/* check composite fields */
|
||||||
if(!joinOnPartitionColumn)
|
if (!joinOnPartitionColumn)
|
||||||
{
|
{
|
||||||
bool leftFullCompositeFieldList =
|
bool leftFullCompositeFieldList =
|
||||||
FullCompositeFieldList(leftCompositeFieldList);
|
FullCompositeFieldList(leftCompositeFieldList);
|
||||||
|
@ -3641,7 +3652,7 @@ LeafQuery(Query *queryTree)
|
||||||
* Extract all range table indexes from the join tree. Note that sub-queries
|
* Extract all range table indexes from the join tree. Note that sub-queries
|
||||||
* that get pulled up by PostgreSQL don't appear in this join tree.
|
* that get pulled up by PostgreSQL don't appear in this join tree.
|
||||||
*/
|
*/
|
||||||
ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList);
|
ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList);
|
||||||
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -39,11 +39,11 @@ bool SubqueryPushdown = false; /* is subquery pushdown enabled */
|
||||||
|
|
||||||
|
|
||||||
/* Function pointer type definition for apply join rule functions */
|
/* Function pointer type definition for apply join rule functions */
|
||||||
typedef MultiNode * (*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode,
|
typedef MultiNode *(*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode,
|
||||||
Var *partitionColumn, JoinType joinType,
|
Var *partitionColumn, JoinType joinType,
|
||||||
List *joinClauses);
|
List *joinClauses);
|
||||||
|
|
||||||
static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */
|
static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */
|
||||||
|
|
||||||
/* Local functions forward declarations */
|
/* Local functions forward declarations */
|
||||||
static MultiNode * MultiPlanTree(Query *queryTree);
|
static MultiNode * MultiPlanTree(Query *queryTree);
|
||||||
|
@ -157,7 +157,7 @@ SubqueryEntryList(Query *queryTree)
|
||||||
* only walk over range table entries at this level and do not recurse into
|
* only walk over range table entries at this level and do not recurse into
|
||||||
* subqueries.
|
* subqueries.
|
||||||
*/
|
*/
|
||||||
ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList);
|
ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList);
|
||||||
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
|
@ -285,6 +285,7 @@ MultiPlanTree(Query *queryTree)
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
bool hasOuterJoin = false;
|
bool hasOuterJoin = false;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We calculate the join order using the list of tables in the query and
|
* We calculate the join order using the list of tables in the query and
|
||||||
* the join clauses between them. Note that this function owns the table
|
* the join clauses between them. Note that this function owns the table
|
||||||
|
@ -465,6 +466,7 @@ ErrorIfQueryNotSupported(Query *queryTree)
|
||||||
|
|
||||||
|
|
||||||
#if (PG_VERSION_NUM >= 90500)
|
#if (PG_VERSION_NUM >= 90500)
|
||||||
|
|
||||||
/* HasTablesample returns tree if the query contains tablesample */
|
/* HasTablesample returns tree if the query contains tablesample */
|
||||||
static bool
|
static bool
|
||||||
HasTablesample(Query *queryTree)
|
HasTablesample(Query *queryTree)
|
||||||
|
@ -485,6 +487,8 @@ HasTablesample(Query *queryTree)
|
||||||
|
|
||||||
return hasTablesample;
|
return hasTablesample;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
@ -529,7 +533,8 @@ HasUnsupportedJoinWalker(Node *node, void *context)
|
||||||
* ErrorIfSubqueryNotSupported checks that we can perform distributed planning for
|
* ErrorIfSubqueryNotSupported checks that we can perform distributed planning for
|
||||||
* the given subquery.
|
* the given subquery.
|
||||||
*/
|
*/
|
||||||
static void ErrorIfSubqueryNotSupported(Query *subqueryTree)
|
static void
|
||||||
|
ErrorIfSubqueryNotSupported(Query *subqueryTree)
|
||||||
{
|
{
|
||||||
char *errorDetail = NULL;
|
char *errorDetail = NULL;
|
||||||
bool preconditionsSatisfied = true;
|
bool preconditionsSatisfied = true;
|
||||||
|
@ -587,7 +592,6 @@ HasOuterJoin(Query *queryTree)
|
||||||
static bool
|
static bool
|
||||||
HasOuterJoinWalker(Node *node, void *context)
|
HasOuterJoinWalker(Node *node, void *context)
|
||||||
{
|
{
|
||||||
|
|
||||||
bool hasOuterJoin = false;
|
bool hasOuterJoin = false;
|
||||||
if (node == NULL)
|
if (node == NULL)
|
||||||
{
|
{
|
||||||
|
@ -657,7 +661,7 @@ HasComplexRangeTableType(Query *queryTree)
|
||||||
* Extract all range table indexes from the join tree. Note that sub-queries
|
* Extract all range table indexes from the join tree. Note that sub-queries
|
||||||
* that get pulled up by PostgreSQL don't appear in this join tree.
|
* that get pulled up by PostgreSQL don't appear in this join tree.
|
||||||
*/
|
*/
|
||||||
ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList);
|
ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList);
|
||||||
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
foreach(joinTreeTableIndexCell, joinTreeTableIndexList)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
|
@ -1178,8 +1182,8 @@ IsSelectClause(Node *clause)
|
||||||
|
|
||||||
/* we currently consider the following nodes as select clauses */
|
/* we currently consider the following nodes as select clauses */
|
||||||
NodeTag nodeTag = nodeTag(clause);
|
NodeTag nodeTag = nodeTag(clause);
|
||||||
if ( !(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr ||
|
if (!(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr ||
|
||||||
nodeTag == T_NullTest || nodeTag == T_BooleanTest) )
|
nodeTag == T_NullTest || nodeTag == T_BooleanTest))
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -1567,7 +1571,7 @@ ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType,
|
||||||
|
|
||||||
/* call the join rule application function to create the new join node */
|
/* call the join rule application function to create the new join node */
|
||||||
ruleApplyFunction = JoinRuleApplyFunction(ruleType);
|
ruleApplyFunction = JoinRuleApplyFunction(ruleType);
|
||||||
multiNode = (*ruleApplyFunction) (leftNode, rightNode, partitionColumn,
|
multiNode = (*ruleApplyFunction)(leftNode, rightNode, partitionColumn,
|
||||||
joinType, applicableJoinClauses);
|
joinType, applicableJoinClauses);
|
||||||
|
|
||||||
if (joinType != JOIN_INNER && CitusIsA(multiNode, MultiJoin))
|
if (joinType != JOIN_INNER && CitusIsA(multiNode, MultiJoin))
|
||||||
|
@ -1918,7 +1922,7 @@ ErrorIfSubqueryJoin(Query *queryTree)
|
||||||
* Extract all range table indexes from the join tree. Note that sub-queries
|
* Extract all range table indexes from the join tree. Note that sub-queries
|
||||||
* that get pulled up by PostgreSQL don't appear in this join tree.
|
* that get pulled up by PostgreSQL don't appear in this join tree.
|
||||||
*/
|
*/
|
||||||
ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList);
|
ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList);
|
||||||
joiningRangeTableCount = list_length(joinTreeTableIndexList);
|
joiningRangeTableCount = list_length(joinTreeTableIndexList);
|
||||||
|
|
||||||
if (joiningRangeTableCount > 1)
|
if (joiningRangeTableCount > 1)
|
||||||
|
|
|
@ -138,7 +138,7 @@ static OpExpr * MakeOpExpressionWithZeroConst(void);
|
||||||
static List * BuildRestrictInfoList(List *qualList);
|
static List * BuildRestrictInfoList(List *qualList);
|
||||||
static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery,
|
static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery,
|
||||||
List *dependedJobList);
|
List *dependedJobList);
|
||||||
static JoinSequenceNode * JoinSequenceArray(List * rangeTableFragmentsList,
|
static JoinSequenceNode * JoinSequenceArray(List *rangeTableFragmentsList,
|
||||||
Query *jobQuery, List *dependedJobList);
|
Query *jobQuery, List *dependedJobList);
|
||||||
static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependedJobList);
|
static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependedJobList);
|
||||||
static void CheckJoinBetweenColumns(OpExpr *joinClause);
|
static void CheckJoinBetweenColumns(OpExpr *joinClause);
|
||||||
|
@ -155,7 +155,8 @@ static StringInfo DatumArrayString(Datum *datumArray, uint32 datumCount, Oid dat
|
||||||
static Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType,
|
static Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType,
|
||||||
char *queryString);
|
char *queryString);
|
||||||
static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList);
|
static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList);
|
||||||
static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment);
|
static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry,
|
||||||
|
RangeTableFragment *fragment);
|
||||||
static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId);
|
static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId);
|
||||||
static List * PruneSqlTaskDependencies(List *sqlTaskList);
|
static List * PruneSqlTaskDependencies(List *sqlTaskList);
|
||||||
static List * AssignTaskList(List *sqlTaskList);
|
static List * AssignTaskList(List *sqlTaskList);
|
||||||
|
@ -167,7 +168,7 @@ static Task * GreedyAssignTask(WorkerNode *workerNode, List *taskList,
|
||||||
static List * RoundRobinAssignTaskList(List *taskList);
|
static List * RoundRobinAssignTaskList(List *taskList);
|
||||||
static List * RoundRobinReorder(Task *task, List *placementList);
|
static List * RoundRobinReorder(Task *task, List *placementList);
|
||||||
static List * ReorderAndAssignTaskList(List *taskList,
|
static List * ReorderAndAssignTaskList(List *taskList,
|
||||||
List * (*reorderFunction) (Task *, List *));
|
List * (*reorderFunction)(Task *, List *));
|
||||||
static int CompareTasksByShardId(const void *leftElement, const void *rightElement);
|
static int CompareTasksByShardId(const void *leftElement, const void *rightElement);
|
||||||
static List * ActiveShardPlacementLists(List *taskList);
|
static List * ActiveShardPlacementLists(List *taskList);
|
||||||
static List * ActivePlacementList(List *placementList);
|
static List * ActivePlacementList(List *placementList);
|
||||||
|
@ -309,6 +310,7 @@ BuildJobTree(MultiTreeRoot *multiTree)
|
||||||
partitionKey, partitionType,
|
partitionKey, partitionType,
|
||||||
baseRelationId,
|
baseRelationId,
|
||||||
JOIN_MAP_MERGE_JOB);
|
JOIN_MAP_MERGE_JOB);
|
||||||
|
|
||||||
/* reset depended job list */
|
/* reset depended job list */
|
||||||
loopDependedJobList = NIL;
|
loopDependedJobList = NIL;
|
||||||
loopDependedJobList = list_make1(mapMergeJob);
|
loopDependedJobList = list_make1(mapMergeJob);
|
||||||
|
@ -538,7 +540,7 @@ BuildJobQuery(MultiNode *multiNode, List *dependedJobList)
|
||||||
* If we are building this query on a repartitioned subquery job then we
|
* If we are building this query on a repartitioned subquery job then we
|
||||||
* don't need to update column attributes.
|
* don't need to update column attributes.
|
||||||
*/
|
*/
|
||||||
if(dependedJobList != NIL)
|
if (dependedJobList != NIL)
|
||||||
{
|
{
|
||||||
Job *job = (Job *) linitial(dependedJobList);
|
Job *job = (Job *) linitial(dependedJobList);
|
||||||
if (CitusIsA(job, MapMergeJob))
|
if (CitusIsA(job, MapMergeJob))
|
||||||
|
@ -870,7 +872,7 @@ TargetEntryList(List *expressionList)
|
||||||
Expr *expression = (Expr *) lfirst(expressionCell);
|
Expr *expression = (Expr *) lfirst(expressionCell);
|
||||||
|
|
||||||
TargetEntry *targetEntry = makeTargetEntry(expression,
|
TargetEntry *targetEntry = makeTargetEntry(expression,
|
||||||
list_length(targetEntryList)+1,
|
list_length(targetEntryList) + 1,
|
||||||
NULL, false);
|
NULL, false);
|
||||||
targetEntryList = lappend(targetEntryList, targetEntry);
|
targetEntryList = lappend(targetEntryList, targetEntry);
|
||||||
}
|
}
|
||||||
|
@ -1044,7 +1046,7 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList
|
||||||
|
|
||||||
/* fix the column attributes in ON (...) clauses */
|
/* fix the column attributes in ON (...) clauses */
|
||||||
columnList = pull_var_clause_default((Node *) joinNode->joinClauseList);
|
columnList = pull_var_clause_default((Node *) joinNode->joinClauseList);
|
||||||
foreach (columnCell, columnList)
|
foreach(columnCell, columnList)
|
||||||
{
|
{
|
||||||
Var *column = (Var *) lfirst(columnCell);
|
Var *column = (Var *) lfirst(columnCell);
|
||||||
UpdateColumnAttributes(column, *rangeTableList, dependedJobList);
|
UpdateColumnAttributes(column, *rangeTableList, dependedJobList);
|
||||||
|
@ -1093,7 +1095,8 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList
|
||||||
uint32 columnCount = (uint32) list_length(dependedTargetList);
|
uint32 columnCount = (uint32) list_length(dependedTargetList);
|
||||||
List *columnNameList = DerivedColumnNameList(columnCount, dependedJob->jobId);
|
List *columnNameList = DerivedColumnNameList(columnCount, dependedJob->jobId);
|
||||||
|
|
||||||
RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, columnNameList,
|
RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode,
|
||||||
|
columnNameList,
|
||||||
tableIdList);
|
tableIdList);
|
||||||
RangeTblRef *rangeTableRef = makeNode(RangeTblRef);
|
RangeTblRef *rangeTableRef = makeNode(RangeTblRef);
|
||||||
|
|
||||||
|
@ -1864,6 +1867,7 @@ SplitPointObject(ShardInterval **shardIntervalArray, uint32 shardIntervalCount)
|
||||||
return splitPointObject;
|
return splitPointObject;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------
|
/* ------------------------------------------------------------
|
||||||
* Functions that relate to building and assigning tasks follow
|
* Functions that relate to building and assigning tasks follow
|
||||||
* ------------------------------------------------------------
|
* ------------------------------------------------------------
|
||||||
|
@ -2900,7 +2904,7 @@ HashableClauseMutator(Node *originalNode, Var *partitionColumn)
|
||||||
* If this node is not hashable, continue walking down the expression tree
|
* If this node is not hashable, continue walking down the expression tree
|
||||||
* to find and hash clauses which are eligible.
|
* to find and hash clauses which are eligible.
|
||||||
*/
|
*/
|
||||||
if(newNode == NULL)
|
if (newNode == NULL)
|
||||||
{
|
{
|
||||||
newNode = expression_tree_mutator(originalNode, HashableClauseMutator,
|
newNode = expression_tree_mutator(originalNode, HashableClauseMutator,
|
||||||
(void *) partitionColumn);
|
(void *) partitionColumn);
|
||||||
|
@ -3273,7 +3277,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended
|
||||||
joinSequenceArray[joinedTableCount].joiningRangeTableId = NON_PRUNABLE_JOIN;
|
joinSequenceArray[joinedTableCount].joiningRangeTableId = NON_PRUNABLE_JOIN;
|
||||||
joinedTableCount++;
|
joinedTableCount++;
|
||||||
|
|
||||||
foreach (joinExprCell, joinExprList)
|
foreach(joinExprCell, joinExprList)
|
||||||
{
|
{
|
||||||
JoinExpr *joinExpr = (JoinExpr *) lfirst(joinExprCell);
|
JoinExpr *joinExpr = (JoinExpr *) lfirst(joinExprCell);
|
||||||
JoinType joinType = joinExpr->jointype;
|
JoinType joinType = joinExpr->jointype;
|
||||||
|
@ -3347,7 +3351,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended
|
||||||
if (IS_OUTER_JOIN(joinType))
|
if (IS_OUTER_JOIN(joinType))
|
||||||
{
|
{
|
||||||
int innerRangeTableId = 0;
|
int innerRangeTableId = 0;
|
||||||
List * tableFragments = NIL;
|
List *tableFragments = NIL;
|
||||||
int fragmentCount = 0;
|
int fragmentCount = 0;
|
||||||
|
|
||||||
if (joinType == JOIN_RIGHT)
|
if (joinType == JOIN_RIGHT)
|
||||||
|
@ -3500,7 +3504,7 @@ FindRangeTableFragmentsList(List *rangeTableFragmentsList, int tableId)
|
||||||
if (tableFragments != NIL)
|
if (tableFragments != NIL)
|
||||||
{
|
{
|
||||||
RangeTableFragment *tableFragment =
|
RangeTableFragment *tableFragment =
|
||||||
(RangeTableFragment*) linitial(tableFragments);
|
(RangeTableFragment *) linitial(tableFragments);
|
||||||
if (tableFragment->rangeTableId == tableId)
|
if (tableFragment->rangeTableId == tableId)
|
||||||
{
|
{
|
||||||
foundTableFragments = tableFragments;
|
foundTableFragments = tableFragments;
|
||||||
|
@ -4046,6 +4050,7 @@ FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment)
|
||||||
return alias;
|
return alias;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* AnchorShardId walks over each fragment in the given fragment list, finds the
|
* AnchorShardId walks over each fragment in the given fragment list, finds the
|
||||||
* fragment that corresponds to the given anchor range tableId, and returns this
|
* fragment that corresponds to the given anchor range tableId, and returns this
|
||||||
|
@ -4360,7 +4365,7 @@ MergeTaskList(MapMergeJob *mapMergeJob, List *mapTaskList, uint32 taskIdIndex)
|
||||||
StringInfo intermediateTableQueryString =
|
StringInfo intermediateTableQueryString =
|
||||||
IntermediateTableQueryString(jobId, taskIdIndex, reduceQuery);
|
IntermediateTableQueryString(jobId, taskIdIndex, reduceQuery);
|
||||||
|
|
||||||
StringInfo mergeAndRunQueryString= makeStringInfo();
|
StringInfo mergeAndRunQueryString = makeStringInfo();
|
||||||
appendStringInfo(mergeAndRunQueryString, MERGE_FILES_AND_RUN_QUERY_COMMAND,
|
appendStringInfo(mergeAndRunQueryString, MERGE_FILES_AND_RUN_QUERY_COMMAND,
|
||||||
jobId, taskIdIndex, mergeTableQueryString->data,
|
jobId, taskIdIndex, mergeTableQueryString->data,
|
||||||
intermediateTableQueryString->data);
|
intermediateTableQueryString->data);
|
||||||
|
@ -4960,7 +4965,7 @@ List *
|
||||||
FirstReplicaAssignTaskList(List *taskList)
|
FirstReplicaAssignTaskList(List *taskList)
|
||||||
{
|
{
|
||||||
/* No additional reordering need take place for this algorithm */
|
/* No additional reordering need take place for this algorithm */
|
||||||
List * (*reorderFunction)(Task *, List *) = NULL;
|
List *(*reorderFunction)(Task *, List *) = NULL;
|
||||||
|
|
||||||
taskList = ReorderAndAssignTaskList(taskList, reorderFunction);
|
taskList = ReorderAndAssignTaskList(taskList, reorderFunction);
|
||||||
|
|
||||||
|
@ -4984,6 +4989,7 @@ RoundRobinAssignTaskList(List *taskList)
|
||||||
return taskList;
|
return taskList;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* RoundRobinReorder implements the core of the round-robin assignment policy.
|
* RoundRobinReorder implements the core of the round-robin assignment policy.
|
||||||
* It takes a task and placement list and rotates a copy of the placement list
|
* It takes a task and placement list and rotates a copy of the placement list
|
||||||
|
@ -5116,7 +5122,8 @@ ActiveShardPlacementLists(List *taskList)
|
||||||
List *activeShardPlacementList = ActivePlacementList(shardPlacementList);
|
List *activeShardPlacementList = ActivePlacementList(shardPlacementList);
|
||||||
|
|
||||||
/* sort shard placements by their insertion time */
|
/* sort shard placements by their insertion time */
|
||||||
activeShardPlacementList = SortList(activeShardPlacementList, CompareShardPlacements);
|
activeShardPlacementList = SortList(activeShardPlacementList,
|
||||||
|
CompareShardPlacements);
|
||||||
shardPlacementLists = lappend(shardPlacementLists, activeShardPlacementList);
|
shardPlacementLists = lappend(shardPlacementLists, activeShardPlacementList);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5257,7 +5264,8 @@ AssignDualHashTaskList(List *taskList)
|
||||||
uint32 replicaIndex = 0;
|
uint32 replicaIndex = 0;
|
||||||
for (replicaIndex = 0; replicaIndex < ShardReplicationFactor; replicaIndex++)
|
for (replicaIndex = 0; replicaIndex < ShardReplicationFactor; replicaIndex++)
|
||||||
{
|
{
|
||||||
uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + replicaIndex;
|
uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex +
|
||||||
|
replicaIndex;
|
||||||
uint32 assignmentIndex = assignmentOffset % workerNodeCount;
|
uint32 assignmentIndex = assignmentOffset % workerNodeCount;
|
||||||
WorkerNode *workerNode = list_nth(workerNodeList, assignmentIndex);
|
WorkerNode *workerNode = list_nth(workerNodeList, assignmentIndex);
|
||||||
|
|
||||||
|
|
|
@ -205,20 +205,31 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId)
|
||||||
switch (relationNameListLength)
|
switch (relationNameListLength)
|
||||||
{
|
{
|
||||||
case 1:
|
case 1:
|
||||||
|
{
|
||||||
relationNameValue = linitial(relationNameList);
|
relationNameValue = linitial(relationNameList);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 2:
|
case 2:
|
||||||
|
{
|
||||||
relationNameValue = lsecond(relationNameList);
|
relationNameValue = lsecond(relationNameList);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 3:
|
case 3:
|
||||||
|
{
|
||||||
relationNameValue = lthird(relationNameList);
|
relationNameValue = lthird(relationNameList);
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
{
|
||||||
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR),
|
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR),
|
||||||
errmsg("improper relation name: \"%s\"",
|
errmsg("improper relation name: \"%s\"",
|
||||||
NameListToString(relationNameList))));
|
NameListToString(relationNameList))));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
relationName = &(relationNameValue->val.str);
|
relationName = &(relationNameValue->val.str);
|
||||||
AppendShardIdToName(relationName, shardId);
|
AppendShardIdToName(relationName, shardId);
|
||||||
|
|
|
@ -48,23 +48,23 @@ static void NormalizeWorkerListPath(void);
|
||||||
|
|
||||||
/* GUC enum definitions */
|
/* GUC enum definitions */
|
||||||
static const struct config_enum_entry task_assignment_policy_options[] = {
|
static const struct config_enum_entry task_assignment_policy_options[] = {
|
||||||
{"greedy", TASK_ASSIGNMENT_GREEDY, false},
|
{ "greedy", TASK_ASSIGNMENT_GREEDY, false },
|
||||||
{"first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false},
|
{ "first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false },
|
||||||
{"round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false},
|
{ "round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false },
|
||||||
{NULL, 0, false}
|
{ NULL, 0, false }
|
||||||
};
|
};
|
||||||
|
|
||||||
static const struct config_enum_entry task_executor_type_options[] = {
|
static const struct config_enum_entry task_executor_type_options[] = {
|
||||||
{"real-time", MULTI_EXECUTOR_REAL_TIME, false},
|
{ "real-time", MULTI_EXECUTOR_REAL_TIME, false },
|
||||||
{"task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false},
|
{ "task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false },
|
||||||
{"router", MULTI_EXECUTOR_ROUTER, false},
|
{ "router", MULTI_EXECUTOR_ROUTER, false },
|
||||||
{NULL, 0, false}
|
{ NULL, 0, false }
|
||||||
};
|
};
|
||||||
|
|
||||||
static const struct config_enum_entry shard_placement_policy_options[] = {
|
static const struct config_enum_entry shard_placement_policy_options[] = {
|
||||||
{"local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false},
|
{ "local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false },
|
||||||
{"round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false},
|
{ "round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false },
|
||||||
{NULL, 0, false}
|
{ NULL, 0, false }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -206,9 +206,10 @@ RegisterCitusConfigVariables(void)
|
||||||
|
|
||||||
DefineCustomBoolVariable(
|
DefineCustomBoolVariable(
|
||||||
"citusdb.expire_cached_shards",
|
"citusdb.expire_cached_shards",
|
||||||
gettext_noop("Enables shard cache expiration if a shard's size on disk has changed. "),
|
gettext_noop("Enables shard cache expiration if a shard's size on disk has "
|
||||||
gettext_noop("When appending to an existing shard, old data may still be cached on "
|
"changed."),
|
||||||
"other workers. This configuration entry activates automatic "
|
gettext_noop("When appending to an existing shard, old data may still be cached "
|
||||||
|
"on other workers. This configuration entry activates automatic "
|
||||||
"expiration, but should not be used with manual updates to shards."),
|
"expiration, but should not be used with manual updates to shards."),
|
||||||
&ExpireCachedShards,
|
&ExpireCachedShards,
|
||||||
false,
|
false,
|
||||||
|
@ -488,6 +489,7 @@ RegisterCitusConfigVariables(void)
|
||||||
|
|
||||||
/* warn about config items in the citusdb namespace that are not registered above */
|
/* warn about config items in the citusdb namespace that are not registered above */
|
||||||
EmitWarningsOnPlaceholders("citusdb");
|
EmitWarningsOnPlaceholders("citusdb");
|
||||||
|
|
||||||
/* Also warn about citus namespace, as that's a very likely misspelling */
|
/* Also warn about citus namespace, as that's a very likely misspelling */
|
||||||
EmitWarningsOnPlaceholders("citus");
|
EmitWarningsOnPlaceholders("citus");
|
||||||
}
|
}
|
||||||
|
@ -515,8 +517,10 @@ NormalizeWorkerListPath(void)
|
||||||
{
|
{
|
||||||
absoluteFileName = malloc(strlen(DataDir) + strlen(WORKER_LIST_FILENAME) + 2);
|
absoluteFileName = malloc(strlen(DataDir) + strlen(WORKER_LIST_FILENAME) + 2);
|
||||||
if (absoluteFileName == NULL)
|
if (absoluteFileName == NULL)
|
||||||
|
{
|
||||||
ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY),
|
ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY),
|
||||||
errmsg("out of memory")));
|
errmsg("out of memory")));
|
||||||
|
}
|
||||||
|
|
||||||
sprintf(absoluteFileName, "%s/%s", DataDir, WORKER_LIST_FILENAME);
|
sprintf(absoluteFileName, "%s/%s", DataDir, WORKER_LIST_FILENAME);
|
||||||
}
|
}
|
||||||
|
@ -530,6 +534,7 @@ NormalizeWorkerListPath(void)
|
||||||
"environment variable.\n", progname, ConfigFileName)));
|
"environment variable.\n", progname, ConfigFileName)));
|
||||||
}
|
}
|
||||||
|
|
||||||
SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER, PGC_S_OVERRIDE);
|
SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER,
|
||||||
|
PGC_S_OVERRIDE);
|
||||||
free(absoluteFileName);
|
free(absoluteFileName);
|
||||||
}
|
}
|
||||||
|
|
|
@ -116,9 +116,9 @@ FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid,
|
||||||
ForeignPath *best_path, List *tlist, List *scan_clauses)
|
ForeignPath *best_path, List *tlist, List *scan_clauses)
|
||||||
#else
|
#else
|
||||||
static ForeignScan *
|
static ForeignScan *
|
||||||
FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid,
|
FakeGetForeignPlan(PlannerInfo * root, RelOptInfo * baserel, Oid foreigntableid,
|
||||||
ForeignPath *best_path, List *tlist, List *scan_clauses,
|
ForeignPath * best_path, List * tlist, List * scan_clauses,
|
||||||
Plan *outer_plan)
|
Plan * outer_plan)
|
||||||
#endif
|
#endif
|
||||||
{
|
{
|
||||||
Index scan_relid = baserel->relid;
|
Index scan_relid = baserel->relid;
|
||||||
|
|
|
@ -265,7 +265,7 @@ GetRangeTblKind(RangeTblEntry *rte)
|
||||||
{
|
{
|
||||||
CitusRTEKind rteKind = CITUS_RTE_RELATION /* invalid */;
|
CitusRTEKind rteKind = CITUS_RTE_RELATION /* invalid */;
|
||||||
|
|
||||||
switch(rte->rtekind)
|
switch (rte->rtekind)
|
||||||
{
|
{
|
||||||
/* directly rtekind if it's not possibly an extended RTE */
|
/* directly rtekind if it's not possibly an extended RTE */
|
||||||
case RTE_RELATION:
|
case RTE_RELATION:
|
||||||
|
@ -273,9 +273,13 @@ GetRangeTblKind(RangeTblEntry *rte)
|
||||||
case RTE_JOIN:
|
case RTE_JOIN:
|
||||||
case RTE_VALUES:
|
case RTE_VALUES:
|
||||||
case RTE_CTE:
|
case RTE_CTE:
|
||||||
|
{
|
||||||
rteKind = (CitusRTEKind) rte->rtekind;
|
rteKind = (CitusRTEKind) rte->rtekind;
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case RTE_FUNCTION:
|
case RTE_FUNCTION:
|
||||||
|
{
|
||||||
/*
|
/*
|
||||||
* Extract extra data - correct even if a plain RTE_FUNCTION, not
|
* Extract extra data - correct even if a plain RTE_FUNCTION, not
|
||||||
* an extended one, ExtractRangeTblExtraData handles that case
|
* an extended one, ExtractRangeTblExtraData handles that case
|
||||||
|
@ -284,6 +288,7 @@ GetRangeTblKind(RangeTblEntry *rte)
|
||||||
ExtractRangeTblExtraData(rte, &rteKind, NULL, NULL, NULL);
|
ExtractRangeTblExtraData(rte, &rteKind, NULL, NULL, NULL);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return rteKind;
|
return rteKind;
|
||||||
}
|
}
|
||||||
|
|
|
@ -102,6 +102,7 @@ pg_get_extensiondef_string(Oid tableRelationId)
|
||||||
static Oid
|
static Oid
|
||||||
get_extension_schema(Oid ext_oid)
|
get_extension_schema(Oid ext_oid)
|
||||||
{
|
{
|
||||||
|
/* *INDENT-OFF* */
|
||||||
Oid result;
|
Oid result;
|
||||||
Relation rel;
|
Relation rel;
|
||||||
SysScanDesc scandesc;
|
SysScanDesc scandesc;
|
||||||
|
@ -131,6 +132,7 @@ get_extension_schema(Oid ext_oid)
|
||||||
heap_close(rel, AccessShareLock);
|
heap_close(rel, AccessShareLock);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
|
/* *INDENT-ON* */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -186,7 +188,7 @@ AppendOptionListToString(StringInfo stringBuffer, List *optionList)
|
||||||
|
|
||||||
foreach(optionCell, optionList)
|
foreach(optionCell, optionList)
|
||||||
{
|
{
|
||||||
DefElem *option = (DefElem*) lfirst(optionCell);
|
DefElem *option = (DefElem *) lfirst(optionCell);
|
||||||
char *optionName = option->defname;
|
char *optionName = option->defname;
|
||||||
char *optionValue = defGetString(option);
|
char *optionValue = defGetString(option);
|
||||||
|
|
||||||
|
@ -447,22 +449,36 @@ pg_get_tablecolumnoptionsdef_string(Oid tableRelationId)
|
||||||
switch (attributeForm->attstorage)
|
switch (attributeForm->attstorage)
|
||||||
{
|
{
|
||||||
case 'p':
|
case 'p':
|
||||||
|
{
|
||||||
storageName = "PLAIN";
|
storageName = "PLAIN";
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'e':
|
case 'e':
|
||||||
|
{
|
||||||
storageName = "EXTERNAL";
|
storageName = "EXTERNAL";
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'm':
|
case 'm':
|
||||||
|
{
|
||||||
storageName = "MAIN";
|
storageName = "MAIN";
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
case 'x':
|
case 'x':
|
||||||
|
{
|
||||||
storageName = "EXTENDED";
|
storageName = "EXTENDED";
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
{
|
||||||
ereport(ERROR, (errmsg("unrecognized storage type: %c",
|
ereport(ERROR, (errmsg("unrecognized storage type: %c",
|
||||||
attributeForm->attstorage)));
|
attributeForm->attstorage)));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
appendStringInfo(&statement, "ALTER COLUMN %s ",
|
appendStringInfo(&statement, "ALTER COLUMN %s ",
|
||||||
quote_identifier(attributeName));
|
quote_identifier(attributeName));
|
||||||
|
|
|
@ -87,6 +87,7 @@ IsDistributedTable(Oid relationId)
|
||||||
return cacheEntry->isDistributedTable;
|
return cacheEntry->isDistributedTable;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* LoadShardInterval reads shard metadata for given shardId from pg_dist_shard,
|
* LoadShardInterval reads shard metadata for given shardId from pg_dist_shard,
|
||||||
* and converts min/max values in these metadata to their properly typed datum
|
* and converts min/max values in these metadata to their properly typed datum
|
||||||
|
@ -139,6 +140,7 @@ LoadShardInterval(uint64 shardId)
|
||||||
return shardInterval;
|
return shardInterval;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* DistributedTableCacheEntry looks up a pg_dist_partition entry for a
|
* DistributedTableCacheEntry looks up a pg_dist_partition entry for a
|
||||||
* relation.
|
* relation.
|
||||||
|
|
|
@ -22,7 +22,8 @@
|
||||||
#include "distributed/multi_resowner.h"
|
#include "distributed/multi_resowner.h"
|
||||||
|
|
||||||
|
|
||||||
typedef struct JobDirectoryEntry {
|
typedef struct JobDirectoryEntry
|
||||||
|
{
|
||||||
ResourceOwner owner;
|
ResourceOwner owner;
|
||||||
uint64 jobId;
|
uint64 jobId;
|
||||||
} JobDirectoryEntry;
|
} JobDirectoryEntry;
|
||||||
|
@ -91,15 +92,17 @@ ResourceOwnerEnlargeJobDirectories(ResourceOwner owner)
|
||||||
if (RegisteredJobDirectories == NULL)
|
if (RegisteredJobDirectories == NULL)
|
||||||
{
|
{
|
||||||
newMax = 16;
|
newMax = 16;
|
||||||
RegisteredJobDirectories = (JobDirectoryEntry *)
|
RegisteredJobDirectories =
|
||||||
MemoryContextAlloc(TopMemoryContext, newMax * sizeof(JobDirectoryEntry));
|
(JobDirectoryEntry *) MemoryContextAlloc(TopMemoryContext,
|
||||||
|
newMax * sizeof(JobDirectoryEntry));
|
||||||
NumAllocatedJobDirectories = newMax;
|
NumAllocatedJobDirectories = newMax;
|
||||||
}
|
}
|
||||||
else if (NumRegisteredJobDirectories + 1 > NumAllocatedJobDirectories)
|
else if (NumRegisteredJobDirectories + 1 > NumAllocatedJobDirectories)
|
||||||
{
|
{
|
||||||
newMax = NumAllocatedJobDirectories * 2;
|
newMax = NumAllocatedJobDirectories * 2;
|
||||||
RegisteredJobDirectories = (JobDirectoryEntry *)
|
RegisteredJobDirectories =
|
||||||
repalloc(RegisteredJobDirectories, newMax * sizeof(JobDirectoryEntry));
|
(JobDirectoryEntry *) repalloc(RegisteredJobDirectories,
|
||||||
|
newMax * sizeof(JobDirectoryEntry));
|
||||||
NumAllocatedJobDirectories = newMax;
|
NumAllocatedJobDirectories = newMax;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -135,7 +138,8 @@ ResourceOwnerForgetJobDirectory(ResourceOwner owner, uint64 jobId)
|
||||||
/* move all later entries one up */
|
/* move all later entries one up */
|
||||||
while (jobIndex < lastJobIndex)
|
while (jobIndex < lastJobIndex)
|
||||||
{
|
{
|
||||||
RegisteredJobDirectories[jobIndex] = RegisteredJobDirectories[jobIndex + 1];
|
RegisteredJobDirectories[jobIndex] =
|
||||||
|
RegisteredJobDirectories[jobIndex + 1];
|
||||||
jobIndex++;
|
jobIndex++;
|
||||||
}
|
}
|
||||||
NumRegisteredJobDirectories = lastJobIndex;
|
NumRegisteredJobDirectories = lastJobIndex;
|
||||||
|
|
|
@ -76,10 +76,10 @@ static void TrackerCleanupJobSchemas(void);
|
||||||
static void TrackerCleanupConnections(HTAB *WorkerTasksHash);
|
static void TrackerCleanupConnections(HTAB *WorkerTasksHash);
|
||||||
static void TrackerRegisterShutDown(HTAB *WorkerTasksHash);
|
static void TrackerRegisterShutDown(HTAB *WorkerTasksHash);
|
||||||
static void TrackerDelayLoop(void);
|
static void TrackerDelayLoop(void);
|
||||||
static List *SchedulableTaskList(HTAB *WorkerTasksHash);
|
static List * SchedulableTaskList(HTAB *WorkerTasksHash);
|
||||||
static WorkerTask * SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash);
|
static WorkerTask * SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash);
|
||||||
static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash,
|
static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash,
|
||||||
bool (*CriteriaFunction) (WorkerTask *));
|
bool (*CriteriaFunction)(WorkerTask *));
|
||||||
static bool RunningTask(WorkerTask *workerTask);
|
static bool RunningTask(WorkerTask *workerTask);
|
||||||
static bool SchedulableTask(WorkerTask *workerTask);
|
static bool SchedulableTask(WorkerTask *workerTask);
|
||||||
static int CompareTasksByTime(const void *first, const void *second);
|
static int CompareTasksByTime(const void *first, const void *second);
|
||||||
|
@ -494,6 +494,7 @@ TrackerDelayLoop(void)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------
|
/* ------------------------------------------------------------
|
||||||
* Signal handling and shared hash initialization functions follow
|
* Signal handling and shared hash initialization functions follow
|
||||||
* ------------------------------------------------------------
|
* ------------------------------------------------------------
|
||||||
|
@ -579,8 +580,8 @@ TaskTrackerShmemInit(void)
|
||||||
LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
|
LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
|
||||||
|
|
||||||
/* allocate struct containing task tracker related shared state */
|
/* allocate struct containing task tracker related shared state */
|
||||||
WorkerTasksSharedState = (WorkerTasksSharedStateData *)
|
WorkerTasksSharedState =
|
||||||
ShmemInitStruct("Worker Task Control",
|
(WorkerTasksSharedStateData *) ShmemInitStruct("Worker Task Control",
|
||||||
sizeof(WorkerTasksSharedStateData),
|
sizeof(WorkerTasksSharedStateData),
|
||||||
&alreadyInitialized);
|
&alreadyInitialized);
|
||||||
|
|
||||||
|
@ -607,6 +608,7 @@ TaskTrackerShmemInit(void)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------
|
/* ------------------------------------------------------------
|
||||||
* Task scheduling and management functions follow
|
* Task scheduling and management functions follow
|
||||||
* ------------------------------------------------------------
|
* ------------------------------------------------------------
|
||||||
|
@ -719,7 +721,7 @@ SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash)
|
||||||
/* Counts the number of tasks that match the given criteria function. */
|
/* Counts the number of tasks that match the given criteria function. */
|
||||||
static uint32
|
static uint32
|
||||||
CountTasksMatchingCriteria(HTAB *WorkerTasksHash,
|
CountTasksMatchingCriteria(HTAB *WorkerTasksHash,
|
||||||
bool (*CriteriaFunction) (WorkerTask *))
|
bool (*CriteriaFunction)(WorkerTask *))
|
||||||
{
|
{
|
||||||
HASH_SEQ_STATUS status;
|
HASH_SEQ_STATUS status;
|
||||||
WorkerTask *currentTask = NULL;
|
WorkerTask *currentTask = NULL;
|
||||||
|
@ -730,7 +732,7 @@ CountTasksMatchingCriteria(HTAB *WorkerTasksHash,
|
||||||
currentTask = (WorkerTask *) hash_seq_search(&status);
|
currentTask = (WorkerTask *) hash_seq_search(&status);
|
||||||
while (currentTask != NULL)
|
while (currentTask != NULL)
|
||||||
{
|
{
|
||||||
bool matchesCriteria = (*CriteriaFunction) (currentTask);
|
bool matchesCriteria = (*CriteriaFunction)(currentTask);
|
||||||
if (matchesCriteria)
|
if (matchesCriteria)
|
||||||
{
|
{
|
||||||
taskCount++;
|
taskCount++;
|
||||||
|
|
|
@ -331,7 +331,7 @@ UpdateTask(WorkerTask *workerTask, char *taskCallString)
|
||||||
if (taskStatus == TASK_SUCCEEDED || taskStatus == TASK_CANCEL_REQUESTED ||
|
if (taskStatus == TASK_SUCCEEDED || taskStatus == TASK_CANCEL_REQUESTED ||
|
||||||
taskStatus == TASK_CANCELED)
|
taskStatus == TASK_CANCELED)
|
||||||
{
|
{
|
||||||
; /* nothing to do */
|
/* nothing to do */
|
||||||
}
|
}
|
||||||
else if (taskStatus == TASK_PERMANENTLY_FAILED)
|
else if (taskStatus == TASK_PERMANENTLY_FAILED)
|
||||||
{
|
{
|
||||||
|
|
|
@ -53,11 +53,14 @@ static void ReceiveResourceCleanup(int32 connectionId, const char *filename,
|
||||||
static void DeleteFile(const char *filename);
|
static void DeleteFile(const char *filename);
|
||||||
static void FetchTableCommon(text *tableName, uint64 remoteTableSize,
|
static void FetchTableCommon(text *tableName, uint64 remoteTableSize,
|
||||||
ArrayType *nodeNameObject, ArrayType *nodePortObject,
|
ArrayType *nodeNameObject, ArrayType *nodePortObject,
|
||||||
bool (*FetchTableFunction) (const char *, uint32, StringInfo));
|
bool (*FetchTableFunction)(const char *, uint32,
|
||||||
|
StringInfo));
|
||||||
static uint64 LocalTableSize(Oid relationId);
|
static uint64 LocalTableSize(Oid relationId);
|
||||||
static uint64 ExtractShardId(StringInfo tableName);
|
static uint64 ExtractShardId(StringInfo tableName);
|
||||||
static bool FetchRegularTable(const char *nodeName, uint32 nodePort, StringInfo tableName);
|
static bool FetchRegularTable(const char *nodeName, uint32 nodePort,
|
||||||
static bool FetchForeignTable(const char *nodeName, uint32 nodePort, StringInfo tableName);
|
StringInfo tableName);
|
||||||
|
static bool FetchForeignTable(const char *nodeName, uint32 nodePort,
|
||||||
|
StringInfo tableName);
|
||||||
static List * TableDDLCommandList(const char *nodeName, uint32 nodePort,
|
static List * TableDDLCommandList(const char *nodeName, uint32 nodePort,
|
||||||
StringInfo tableName);
|
StringInfo tableName);
|
||||||
static StringInfo ForeignFilePath(const char *nodeName, uint32 nodePort,
|
static StringInfo ForeignFilePath(const char *nodeName, uint32 nodePort,
|
||||||
|
@ -309,7 +312,7 @@ ReceiveRegularFile(const char *nodeName, uint32 nodePort,
|
||||||
}
|
}
|
||||||
else if (copyStatus == CLIENT_COPY_MORE)
|
else if (copyStatus == CLIENT_COPY_MORE)
|
||||||
{
|
{
|
||||||
; /* remote node will continue to send more data */
|
/* remote node will continue to send more data */
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -468,7 +471,7 @@ worker_fetch_foreign_file(PG_FUNCTION_ARGS)
|
||||||
static void
|
static void
|
||||||
FetchTableCommon(text *tableNameText, uint64 remoteTableSize,
|
FetchTableCommon(text *tableNameText, uint64 remoteTableSize,
|
||||||
ArrayType *nodeNameObject, ArrayType *nodePortObject,
|
ArrayType *nodeNameObject, ArrayType *nodePortObject,
|
||||||
bool (*FetchTableFunction) (const char *, uint32, StringInfo))
|
bool (*FetchTableFunction)(const char *, uint32, StringInfo))
|
||||||
{
|
{
|
||||||
StringInfo tableName = NULL;
|
StringInfo tableName = NULL;
|
||||||
char *tableNameCString = NULL;
|
char *tableNameCString = NULL;
|
||||||
|
@ -531,7 +534,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize,
|
||||||
if (remoteTableSize > localTableSize)
|
if (remoteTableSize > localTableSize)
|
||||||
{
|
{
|
||||||
/* table is not up to date, drop the table */
|
/* table is not up to date, drop the table */
|
||||||
ObjectAddress tableObject = {InvalidOid, InvalidOid, 0};
|
ObjectAddress tableObject = { InvalidOid, InvalidOid, 0 };
|
||||||
|
|
||||||
tableObject.classId = RelationRelationId;
|
tableObject.classId = RelationRelationId;
|
||||||
tableObject.objectId = relationId;
|
tableObject.objectId = relationId;
|
||||||
|
@ -554,7 +557,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize,
|
||||||
char *nodeName = TextDatumGetCString(nodeNameDatum);
|
char *nodeName = TextDatumGetCString(nodeNameDatum);
|
||||||
uint32 nodePort = DatumGetUInt32(nodePortDatum);
|
uint32 nodePort = DatumGetUInt32(nodePortDatum);
|
||||||
|
|
||||||
tableFetched = (*FetchTableFunction) (nodeName, nodePort, tableName);
|
tableFetched = (*FetchTableFunction)(nodeName, nodePort, tableName);
|
||||||
|
|
||||||
nodeIndex++;
|
nodeIndex++;
|
||||||
}
|
}
|
||||||
|
|
|
@ -256,8 +256,8 @@ JobSchemaName(uint64 jobId)
|
||||||
*/
|
*/
|
||||||
#ifdef HAVE_INTTYPES_H
|
#ifdef HAVE_INTTYPES_H
|
||||||
StringInfo jobSchemaName = makeStringInfo();
|
StringInfo jobSchemaName = makeStringInfo();
|
||||||
appendStringInfo(jobSchemaName, "%s%0*"PRIu64,
|
appendStringInfo(jobSchemaName, "%s%0*" PRIu64, JOB_SCHEMA_PREFIX,
|
||||||
JOB_SCHEMA_PREFIX, MIN_JOB_DIRNAME_WIDTH, jobId);
|
MIN_JOB_DIRNAME_WIDTH, jobId);
|
||||||
#else
|
#else
|
||||||
StringInfo jobSchemaName = makeStringInfo();
|
StringInfo jobSchemaName = makeStringInfo();
|
||||||
appendStringInfo(jobSchemaName, "%s%0*llu",
|
appendStringInfo(jobSchemaName, "%s%0*llu",
|
||||||
|
|
|
@ -59,7 +59,7 @@ static void FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite)
|
||||||
static void FileOutputStreamFlush(FileOutputStream file);
|
static void FileOutputStreamFlush(FileOutputStream file);
|
||||||
static void FilterAndPartitionTable(const char *filterQuery,
|
static void FilterAndPartitionTable(const char *filterQuery,
|
||||||
const char *columnName, Oid columnType,
|
const char *columnName, Oid columnType,
|
||||||
uint32 (*PartitionIdFunction) (Datum, const void *),
|
uint32 (*PartitionIdFunction)(Datum, const void *),
|
||||||
const void *partitionIdContext,
|
const void *partitionIdContext,
|
||||||
FileOutputStream *partitionFileArray,
|
FileOutputStream *partitionFileArray,
|
||||||
uint32 fileCount);
|
uint32 fileCount);
|
||||||
|
@ -463,7 +463,7 @@ JobDirectoryName(uint64 jobId)
|
||||||
*/
|
*/
|
||||||
#ifdef HAVE_INTTYPES_H
|
#ifdef HAVE_INTTYPES_H
|
||||||
StringInfo jobDirectoryName = makeStringInfo();
|
StringInfo jobDirectoryName = makeStringInfo();
|
||||||
appendStringInfo(jobDirectoryName, "base/%s/%s%0*"PRIu64,
|
appendStringInfo(jobDirectoryName, "base/%s/%s%0*" PRIu64,
|
||||||
PG_JOB_CACHE_DIR, JOB_DIRECTORY_PREFIX,
|
PG_JOB_CACHE_DIR, JOB_DIRECTORY_PREFIX,
|
||||||
MIN_JOB_DIRNAME_WIDTH, jobId);
|
MIN_JOB_DIRNAME_WIDTH, jobId);
|
||||||
#else
|
#else
|
||||||
|
@ -726,7 +726,7 @@ FileOutputStreamFlush(FileOutputStream file)
|
||||||
static void
|
static void
|
||||||
FilterAndPartitionTable(const char *filterQuery,
|
FilterAndPartitionTable(const char *filterQuery,
|
||||||
const char *partitionColumnName, Oid partitionColumnType,
|
const char *partitionColumnName, Oid partitionColumnType,
|
||||||
uint32 (*PartitionIdFunction) (Datum, const void *),
|
uint32 (*PartitionIdFunction)(Datum, const void *),
|
||||||
const void *partitionIdContext,
|
const void *partitionIdContext,
|
||||||
FileOutputStream *partitionFileArray,
|
FileOutputStream *partitionFileArray,
|
||||||
uint32 fileCount)
|
uint32 fileCount)
|
||||||
|
@ -808,7 +808,7 @@ FilterAndPartitionTable(const char *filterQuery,
|
||||||
*/
|
*/
|
||||||
if (!partitionKeyNull)
|
if (!partitionKeyNull)
|
||||||
{
|
{
|
||||||
partitionId = (*PartitionIdFunction) (partitionKey, partitionIdContext);
|
partitionId = (*PartitionIdFunction)(partitionKey, partitionIdContext);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -946,7 +946,7 @@ InitRowOutputState(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* set up transcoding information and default text output characters */
|
/* set up transcoding information and default text output characters */
|
||||||
if ( (fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1) )
|
if ((fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1))
|
||||||
{
|
{
|
||||||
rowOutputState->need_transcoding = true;
|
rowOutputState->need_transcoding = true;
|
||||||
}
|
}
|
||||||
|
@ -1057,7 +1057,7 @@ OutputRow(HeapTuple row, TupleDesc rowDescriptor,
|
||||||
CopySendString(rowOutputState, rowOutputState->null_print_client);
|
CopySendString(rowOutputState, rowOutputState->null_print_client);
|
||||||
}
|
}
|
||||||
|
|
||||||
lastColumn = ((columnIndex+1) == columnCount);
|
lastColumn = ((columnIndex + 1) == columnCount);
|
||||||
if (!lastColumn)
|
if (!lastColumn)
|
||||||
{
|
{
|
||||||
CopySendChar(rowOutputState, rowOutputState->delim[0]);
|
CopySendChar(rowOutputState, rowOutputState->delim[0]);
|
||||||
|
@ -1094,9 +1094,9 @@ OutputBinaryHeaders(FileOutputStream *partitionFileArray, uint32 fileCount)
|
||||||
{
|
{
|
||||||
/* Generate header for a binary copy */
|
/* Generate header for a binary copy */
|
||||||
const int32 zero = 0;
|
const int32 zero = 0;
|
||||||
FileOutputStream partitionFile = {0, 0, 0};
|
FileOutputStream partitionFile = { 0, 0, 0 };
|
||||||
PartialCopyStateData headerOutputStateData;
|
PartialCopyStateData headerOutputStateData;
|
||||||
PartialCopyState headerOutputState = (PartialCopyState) &headerOutputStateData;
|
PartialCopyState headerOutputState = (PartialCopyState) & headerOutputStateData;
|
||||||
|
|
||||||
memset(headerOutputState, 0, sizeof(PartialCopyStateData));
|
memset(headerOutputState, 0, sizeof(PartialCopyStateData));
|
||||||
headerOutputState->fe_msgbuf = makeStringInfo();
|
headerOutputState->fe_msgbuf = makeStringInfo();
|
||||||
|
@ -1128,9 +1128,9 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount)
|
||||||
{
|
{
|
||||||
/* Generate footer for a binary copy */
|
/* Generate footer for a binary copy */
|
||||||
int16 negative = -1;
|
int16 negative = -1;
|
||||||
FileOutputStream partitionFile = {0, 0, 0};
|
FileOutputStream partitionFile = { 0, 0, 0 };
|
||||||
PartialCopyStateData footerOutputStateData;
|
PartialCopyStateData footerOutputStateData;
|
||||||
PartialCopyState footerOutputState = (PartialCopyState) &footerOutputStateData;
|
PartialCopyState footerOutputState = (PartialCopyState) & footerOutputStateData;
|
||||||
|
|
||||||
memset(footerOutputState, 0, sizeof(PartialCopyStateData));
|
memset(footerOutputState, 0, sizeof(PartialCopyStateData));
|
||||||
footerOutputState->fe_msgbuf = makeStringInfo();
|
footerOutputState->fe_msgbuf = makeStringInfo();
|
||||||
|
@ -1143,6 +1143,7 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* *INDENT-OFF* */
|
||||||
/* Append data to the copy buffer in outputState */
|
/* Append data to the copy buffer in outputState */
|
||||||
static void
|
static void
|
||||||
CopySendData(PartialCopyState outputState, const void *databuf, int datasize)
|
CopySendData(PartialCopyState outputState, const void *databuf, int datasize)
|
||||||
|
@ -1282,6 +1283,7 @@ CopyAttributeOutText(PartialCopyState cstate, char *string)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* *INDENT-ON* */
|
||||||
/* Helper function to send pending copy output */
|
/* Helper function to send pending copy output */
|
||||||
static inline void
|
static inline void
|
||||||
CopyFlushOutput(PartialCopyState cstate, char *start, char *pointer)
|
CopyFlushOutput(PartialCopyState cstate, char *start, char *pointer)
|
||||||
|
|
|
@ -16,7 +16,22 @@
|
||||||
#include "stringutils.h"
|
#include "stringutils.h"
|
||||||
|
|
||||||
|
|
||||||
/* Concatenates "more" onto "var", and frees the original value of *var. */
|
/* *INDENT-OFF* */
|
||||||
|
void
|
||||||
|
free_copy_options(copy_options * ptr)
|
||||||
|
{
|
||||||
|
if (!ptr)
|
||||||
|
return;
|
||||||
|
free(ptr->before_tofrom);
|
||||||
|
free(ptr->after_tofrom);
|
||||||
|
free(ptr->file);
|
||||||
|
free(ptr->tableName);
|
||||||
|
free(ptr->columnList);
|
||||||
|
free(ptr);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* concatenate "more" onto "var", freeing the original value of *var */
|
||||||
static void
|
static void
|
||||||
xstrcat(char **var, const char *more)
|
xstrcat(char **var, const char *more)
|
||||||
{
|
{
|
||||||
|
@ -210,21 +225,9 @@ error:
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* *INDENT-ON* */
|
||||||
|
|
||||||
/* Frees copy options. */
|
/* Frees copy options. */
|
||||||
void
|
|
||||||
free_copy_options(copy_options * ptr)
|
|
||||||
{
|
|
||||||
if (!ptr)
|
|
||||||
return;
|
|
||||||
free(ptr->before_tofrom);
|
|
||||||
free(ptr->after_tofrom);
|
|
||||||
free(ptr->file);
|
|
||||||
free(ptr->tableName);
|
|
||||||
free(ptr->columnList);
|
|
||||||
free(ptr);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* ParseStageOptions takes the given copy options, parses the additional options
|
* ParseStageOptions takes the given copy options, parses the additional options
|
||||||
|
|
|
@ -26,7 +26,8 @@
|
||||||
static bool FileSize(char *filename, uint64 *fileSize);
|
static bool FileSize(char *filename, uint64 *fileSize);
|
||||||
static PGconn * ConnectToWorkerNode(const char *nodeName, uint32 nodePort,
|
static PGconn * ConnectToWorkerNode(const char *nodeName, uint32 nodePort,
|
||||||
const char *nodeDatabase);
|
const char *nodeDatabase);
|
||||||
static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection, const char *remoteCommand,
|
static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection,
|
||||||
|
const char *remoteCommand,
|
||||||
const char **parameterValues, int parameterCount);
|
const char **parameterValues, int parameterCount);
|
||||||
static TableMetadata * InitTableMetadata(const char *tableName);
|
static TableMetadata * InitTableMetadata(const char *tableName);
|
||||||
static ShardMetadata * InitShardMetadata(int shardPlacementPolicy);
|
static ShardMetadata * InitShardMetadata(int shardPlacementPolicy);
|
||||||
|
@ -41,7 +42,8 @@ static uint64 GetValueUint64(const PGresult *result, int rowNumber, int columnNu
|
||||||
static bool MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata);
|
static bool MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata);
|
||||||
static bool MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata);
|
static bool MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata);
|
||||||
static bool MasterGetNewShardId(ShardMetadata *shardMetadata);
|
static bool MasterGetNewShardId(ShardMetadata *shardMetadata);
|
||||||
static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy);
|
static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata,
|
||||||
|
int shardPlacementPolicy);
|
||||||
static bool MasterInsertShardRow(uint32 logicalRelid, char storageType,
|
static bool MasterInsertShardRow(uint32 logicalRelid, char storageType,
|
||||||
const ShardMetadata *shardMetadata);
|
const ShardMetadata *shardMetadata);
|
||||||
static bool MasterInsertPlacementRows(const ShardMetadata *shardMetadata);
|
static bool MasterInsertPlacementRows(const ShardMetadata *shardMetadata);
|
||||||
|
@ -62,7 +64,8 @@ static bool ApplyShardDDLCommand(PGconn *workerNode, uint64 shardId, const char
|
||||||
static bool TransmitTableData(PGconn *workerNode, uint64 shardId,
|
static bool TransmitTableData(PGconn *workerNode, uint64 shardId,
|
||||||
uint64 shardMaxSize, copy_options *stageOptions,
|
uint64 shardMaxSize, copy_options *stageOptions,
|
||||||
uint64 currentFileOffset, uint64 *nextFileOffset);
|
uint64 currentFileOffset, uint64 *nextFileOffset);
|
||||||
static bool TransmitFile(PGconn *workerNode, const char *localPath, const char *remotePath);
|
static bool TransmitFile(PGconn *workerNode, const char *localPath,
|
||||||
|
const char *remotePath);
|
||||||
static bool FileStreamOK(const copy_options *stageOptions);
|
static bool FileStreamOK(const copy_options *stageOptions);
|
||||||
static PQExpBuffer CreateCopyQueryString(const char *tableName, const char *columnList,
|
static PQExpBuffer CreateCopyQueryString(const char *tableName, const char *columnList,
|
||||||
const char *afterToFrom);
|
const char *afterToFrom);
|
||||||
|
@ -341,7 +344,6 @@ DoStageData(const char *stageCommand)
|
||||||
|
|
||||||
/* update current file offset */
|
/* update current file offset */
|
||||||
currentFileOffset = nextFileOffset;
|
currentFileOffset = nextFileOffset;
|
||||||
|
|
||||||
} /* while more file data left for sharding */
|
} /* while more file data left for sharding */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -33,17 +33,19 @@
|
||||||
#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)"
|
#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)"
|
||||||
#define MASTER_GET_TABLE_DDL_EVENTS "SELECT * FROM master_get_table_ddl_events($1::text)"
|
#define MASTER_GET_TABLE_DDL_EVENTS "SELECT * FROM master_get_table_ddl_events($1::text)"
|
||||||
#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()"
|
#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()"
|
||||||
#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES "SELECT * FROM \
|
#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES \
|
||||||
master_get_local_first_candidate_nodes()"
|
"SELECT * FROM master_get_local_first_candidate_nodes()"
|
||||||
#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES "SELECT * FROM \
|
#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES \
|
||||||
master_get_round_robin_candidate_nodes($1::int8)"
|
"SELECT * FROM master_get_round_robin_candidate_nodes($1::int8)"
|
||||||
|
|
||||||
#define MASTER_INSERT_SHARD_ROW "INSERT INTO pg_dist_shard \
|
#define MASTER_INSERT_SHARD_ROW \
|
||||||
(logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES \
|
"INSERT INTO pg_dist_shard " \
|
||||||
($1::oid, $2::int8, $3::char, $4::text, $5::text)"
|
"(logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES " \
|
||||||
#define MASTER_INSERT_PLACEMENT_ROW "INSERT INTO pg_dist_shard_placement \
|
"($1::oid, $2::int8, $3::char, $4::text, $5::text)"
|
||||||
(shardid, shardstate, shardlength, nodename, nodeport) VALUES \
|
#define MASTER_INSERT_PLACEMENT_ROW \
|
||||||
($1::int8, $2::int4, $3::int8, $4::text, $5::int4)"
|
"INSERT INTO pg_dist_shard_placement " \
|
||||||
|
"(shardid, shardstate, shardlength, nodename, nodeport) VALUES " \
|
||||||
|
"($1::int8, $2::int4, $3::int8, $4::text, $5::int4)"
|
||||||
|
|
||||||
/* Column names used to identify response fields as returned from the master. */
|
/* Column names used to identify response fields as returned from the master. */
|
||||||
#define LOGICAL_RELID_FIELD "logical_relid"
|
#define LOGICAL_RELID_FIELD "logical_relid"
|
||||||
|
@ -61,11 +63,11 @@
|
||||||
#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s"
|
#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s"
|
||||||
#define SHARD_TABLE_SIZE_COMMAND "SELECT pg_table_size('%s')"
|
#define SHARD_TABLE_SIZE_COMMAND "SELECT pg_table_size('%s')"
|
||||||
#define SET_FOREIGN_TABLE_FILENAME "ALTER FOREIGN TABLE %s OPTIONS (SET filename '%s')"
|
#define SET_FOREIGN_TABLE_FILENAME "ALTER FOREIGN TABLE %s OPTIONS (SET filename '%s')"
|
||||||
#define GET_COLUMNAR_TABLE_FILENAME_OPTION "SELECT * FROM \
|
#define GET_COLUMNAR_TABLE_FILENAME_OPTION \
|
||||||
(SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table \
|
"SELECT * FROM (SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table " \
|
||||||
WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';"
|
"WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';"
|
||||||
#define APPLY_SHARD_DDL_COMMAND "SELECT * FROM worker_apply_shard_ddl_command \
|
#define APPLY_SHARD_DDL_COMMAND \
|
||||||
($1::int8, $2::text)"
|
"SELECT * FROM worker_apply_shard_ddl_command ($1::int8, $2::text)"
|
||||||
#define REMOTE_FILE_SIZE_COMMAND "SELECT size FROM pg_stat_file('%s')"
|
#define REMOTE_FILE_SIZE_COMMAND "SELECT size FROM pg_stat_file('%s')"
|
||||||
#define SHARD_COLUMNAR_TABLE_SIZE_COMMAND "SELECT cstore_table_size('%s')"
|
#define SHARD_COLUMNAR_TABLE_SIZE_COMMAND "SELECT cstore_table_size('%s')"
|
||||||
|
|
||||||
|
@ -100,7 +102,6 @@ typedef struct TableMetadata
|
||||||
|
|
||||||
char **ddlEventList; /* DDL statements used for creating new shard */
|
char **ddlEventList; /* DDL statements used for creating new shard */
|
||||||
uint32 ddlEventCount; /* DDL statement count; statement list size */
|
uint32 ddlEventCount; /* DDL statement count; statement list size */
|
||||||
|
|
||||||
} TableMetadata;
|
} TableMetadata;
|
||||||
|
|
||||||
|
|
||||||
|
@ -122,7 +123,6 @@ typedef struct ShardMetadata
|
||||||
char *shardMinValue; /* partition key's minimum value in shard */
|
char *shardMinValue; /* partition key's minimum value in shard */
|
||||||
char *shardMaxValue; /* partition key's maximum value in shard */
|
char *shardMaxValue; /* partition key's maximum value in shard */
|
||||||
uint64 shardSize; /* shard size; updated during staging */
|
uint64 shardSize; /* shard size; updated during staging */
|
||||||
|
|
||||||
} ShardMetadata;
|
} ShardMetadata;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -16,16 +16,17 @@
|
||||||
|
|
||||||
|
|
||||||
/* Function declarations for version independent CitusDB ruleutils wrapper functions */
|
/* Function declarations for version independent CitusDB ruleutils wrapper functions */
|
||||||
extern char *pg_get_extensiondef_string(Oid tableRelationId);
|
extern char * pg_get_extensiondef_string(Oid tableRelationId);
|
||||||
extern char *pg_get_serverdef_string(Oid tableRelationId);
|
extern char * pg_get_serverdef_string(Oid tableRelationId);
|
||||||
extern char *pg_get_tableschemadef_string(Oid tableRelationId);
|
extern char * pg_get_tableschemadef_string(Oid tableRelationId);
|
||||||
extern char *pg_get_tablecolumnoptionsdef_string(Oid tableRelationId);
|
extern char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId);
|
||||||
extern char *pg_get_indexclusterdef_string(Oid indexRelationId);
|
extern char * pg_get_indexclusterdef_string(Oid indexRelationId);
|
||||||
|
|
||||||
/* Function declarations for version dependent PostgreSQL ruleutils functions */
|
/* Function declarations for version dependent PostgreSQL ruleutils functions */
|
||||||
extern void pg_get_query_def(Query *query, StringInfo buffer);
|
extern void pg_get_query_def(Query *query, StringInfo buffer);
|
||||||
extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer);
|
extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo
|
||||||
extern char *generate_relation_name(Oid relid, List *namespaces);
|
buffer);
|
||||||
|
extern char * generate_relation_name(Oid relid, List *namespaces);
|
||||||
|
|
||||||
|
|
||||||
#endif /* CITUS_RULEUTILS_H */
|
#endif /* CITUS_RULEUTILS_H */
|
||||||
|
|
|
@ -38,7 +38,6 @@ typedef struct ShardInterval
|
||||||
Datum minValue; /* a shard's typed min value datum */
|
Datum minValue; /* a shard's typed min value datum */
|
||||||
Datum maxValue; /* a shard's typed max value datum */
|
Datum maxValue; /* a shard's typed max value datum */
|
||||||
uint64 shardId;
|
uint64 shardId;
|
||||||
|
|
||||||
} ShardInterval;
|
} ShardInterval;
|
||||||
|
|
||||||
|
|
||||||
|
@ -52,7 +51,6 @@ typedef struct ShardPlacement
|
||||||
RelayFileState shardState;
|
RelayFileState shardState;
|
||||||
char *nodeName;
|
char *nodeName;
|
||||||
uint32 nodePort;
|
uint32 nodePort;
|
||||||
|
|
||||||
} ShardPlacement;
|
} ShardPlacement;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -49,10 +49,10 @@
|
||||||
#define SHARDID_SEQUENCE_NAME "pg_dist_shardid_seq"
|
#define SHARDID_SEQUENCE_NAME "pg_dist_shardid_seq"
|
||||||
|
|
||||||
/* Remote call definitions to help with data staging and deletion */
|
/* Remote call definitions to help with data staging and deletion */
|
||||||
#define WORKER_APPLY_SHARD_DDL_COMMAND "SELECT worker_apply_shard_ddl_command \
|
#define WORKER_APPLY_SHARD_DDL_COMMAND \
|
||||||
("UINT64_FORMAT", %s)"
|
"SELECT worker_apply_shard_ddl_command (" UINT64_FORMAT ", %s)"
|
||||||
#define WORKER_APPEND_TABLE_TO_SHARD "SELECT worker_append_table_to_shard \
|
#define WORKER_APPEND_TABLE_TO_SHARD \
|
||||||
(%s, %s, %s, %u)"
|
"SELECT worker_append_table_to_shard (%s, %s, %s, %u)"
|
||||||
#define SHARD_MIN_VALUE_QUERY "SELECT min(%s) FROM %s"
|
#define SHARD_MIN_VALUE_QUERY "SELECT min(%s) FROM %s"
|
||||||
#define SHARD_MAX_VALUE_QUERY "SELECT max(%s) FROM %s"
|
#define SHARD_MAX_VALUE_QUERY "SELECT max(%s) FROM %s"
|
||||||
#define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size('%s')"
|
#define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size('%s')"
|
||||||
|
@ -67,7 +67,6 @@ typedef enum
|
||||||
SHARD_PLACEMENT_INVALID_FIRST = 0,
|
SHARD_PLACEMENT_INVALID_FIRST = 0,
|
||||||
SHARD_PLACEMENT_LOCAL_NODE_FIRST = 1,
|
SHARD_PLACEMENT_LOCAL_NODE_FIRST = 1,
|
||||||
SHARD_PLACEMENT_ROUND_ROBIN = 2
|
SHARD_PLACEMENT_ROUND_ROBIN = 2
|
||||||
|
|
||||||
} ShardPlacementPolicyType;
|
} ShardPlacementPolicyType;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@
|
||||||
#define INVALID_TASK_ID 0
|
#define INVALID_TASK_ID 0
|
||||||
|
|
||||||
#if (PG_VERSION_NUM >= 90500)
|
#if (PG_VERSION_NUM >= 90500)
|
||||||
|
|
||||||
/* reserved alias name for UPSERTs */
|
/* reserved alias name for UPSERTs */
|
||||||
#define UPSERT_ALIAS "citus_table_alias"
|
#define UPSERT_ALIAS "citus_table_alias"
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -29,7 +29,6 @@ typedef enum
|
||||||
CLIENT_CONNECTION_BAD = 1,
|
CLIENT_CONNECTION_BAD = 1,
|
||||||
CLIENT_CONNECTION_BUSY = 2,
|
CLIENT_CONNECTION_BUSY = 2,
|
||||||
CLIENT_CONNECTION_READY = 3
|
CLIENT_CONNECTION_READY = 3
|
||||||
|
|
||||||
} ConnectStatus;
|
} ConnectStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -40,7 +39,6 @@ typedef enum
|
||||||
CLIENT_RESULT_UNAVAILABLE = 1,
|
CLIENT_RESULT_UNAVAILABLE = 1,
|
||||||
CLIENT_RESULT_BUSY = 2,
|
CLIENT_RESULT_BUSY = 2,
|
||||||
CLIENT_RESULT_READY = 3
|
CLIENT_RESULT_READY = 3
|
||||||
|
|
||||||
} ResultStatus;
|
} ResultStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -51,7 +49,6 @@ typedef enum
|
||||||
CLIENT_QUERY_FAILED = 1,
|
CLIENT_QUERY_FAILED = 1,
|
||||||
CLIENT_QUERY_DONE = 2,
|
CLIENT_QUERY_DONE = 2,
|
||||||
CLIENT_QUERY_COPY = 3
|
CLIENT_QUERY_COPY = 3
|
||||||
|
|
||||||
} QueryStatus;
|
} QueryStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -62,7 +59,6 @@ typedef enum
|
||||||
CLIENT_COPY_MORE = 1,
|
CLIENT_COPY_MORE = 1,
|
||||||
CLIENT_COPY_FAILED = 2,
|
CLIENT_COPY_FAILED = 2,
|
||||||
CLIENT_COPY_DONE = 3
|
CLIENT_COPY_DONE = 3
|
||||||
|
|
||||||
} CopyStatus;
|
} CopyStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -73,7 +69,6 @@ typedef enum
|
||||||
CLIENT_BATCH_QUERY_FAILED = 1,
|
CLIENT_BATCH_QUERY_FAILED = 1,
|
||||||
CLIENT_BATCH_QUERY_CONTINUE = 2,
|
CLIENT_BATCH_QUERY_CONTINUE = 2,
|
||||||
CLIENT_BATCH_QUERY_DONE = 3
|
CLIENT_BATCH_QUERY_DONE = 3
|
||||||
|
|
||||||
} BatchQueryStatus;
|
} BatchQueryStatus;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,6 @@ typedef enum JoinRuleType
|
||||||
* RuleNameArray.
|
* RuleNameArray.
|
||||||
*/
|
*/
|
||||||
JOIN_RULE_LAST
|
JOIN_RULE_LAST
|
||||||
|
|
||||||
} JoinRuleType;
|
} JoinRuleType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -53,7 +52,6 @@ typedef struct TableEntry
|
||||||
{
|
{
|
||||||
Oid relationId;
|
Oid relationId;
|
||||||
uint32 rangeTableId;
|
uint32 rangeTableId;
|
||||||
|
|
||||||
} TableEntry;
|
} TableEntry;
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,7 +70,6 @@ typedef struct JoinOrderNode
|
||||||
char partitionMethod;
|
char partitionMethod;
|
||||||
List *joinClauseList; /* not relevant for the first table */
|
List *joinClauseList; /* not relevant for the first table */
|
||||||
List *shardIntervalList;
|
List *shardIntervalList;
|
||||||
|
|
||||||
} JoinOrderNode;
|
} JoinOrderNode;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -55,7 +55,6 @@ typedef enum
|
||||||
AGGREGATE_SUM = 4,
|
AGGREGATE_SUM = 4,
|
||||||
AGGREGATE_COUNT = 5,
|
AGGREGATE_COUNT = 5,
|
||||||
AGGREGATE_ARRAY_AGG = 6
|
AGGREGATE_ARRAY_AGG = 6
|
||||||
|
|
||||||
} AggregateType;
|
} AggregateType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -69,7 +68,6 @@ typedef enum
|
||||||
PUSH_DOWN_VALID = 1,
|
PUSH_DOWN_VALID = 1,
|
||||||
PUSH_DOWN_NOT_VALID = 2,
|
PUSH_DOWN_NOT_VALID = 2,
|
||||||
PUSH_DOWN_SPECIAL_CONDITIONS = 3
|
PUSH_DOWN_SPECIAL_CONDITIONS = 3
|
||||||
|
|
||||||
} PushDownStatus;
|
} PushDownStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -82,7 +80,6 @@ typedef enum
|
||||||
PULL_UP_INVALID_FIRST = 0,
|
PULL_UP_INVALID_FIRST = 0,
|
||||||
PULL_UP_VALID = 1,
|
PULL_UP_VALID = 1,
|
||||||
PULL_UP_NOT_VALID = 2
|
PULL_UP_NOT_VALID = 2
|
||||||
|
|
||||||
} PullUpStatus;
|
} PullUpStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -97,8 +94,10 @@ typedef enum
|
||||||
* Please note that the order of elements in this array is tied to the order of
|
* Please note that the order of elements in this array is tied to the order of
|
||||||
* values in the preceding AggregateType enum. This order needs to be preserved.
|
* values in the preceding AggregateType enum. This order needs to be preserved.
|
||||||
*/
|
*/
|
||||||
static const char * const AggregateNames[] = { "invalid", "avg", "min", "max",
|
static const char *const AggregateNames[] = {
|
||||||
"sum", "count", "array_agg" };
|
"invalid", "avg", "min", "max", "sum",
|
||||||
|
"count", "array_agg"
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
/* Config variable managed via guc.c */
|
/* Config variable managed via guc.c */
|
||||||
|
|
|
@ -40,8 +40,8 @@ typedef struct MultiNode
|
||||||
CitusNodeTag type;
|
CitusNodeTag type;
|
||||||
|
|
||||||
struct MultiNode *parentNode;
|
struct MultiNode *parentNode;
|
||||||
/* child node(s) are defined in unary and binary nodes */
|
|
||||||
|
|
||||||
|
/* child node(s) are defined in unary and binary nodes */
|
||||||
} MultiNode;
|
} MultiNode;
|
||||||
|
|
||||||
|
|
||||||
|
@ -51,7 +51,6 @@ typedef struct MultiUnaryNode
|
||||||
MultiNode node;
|
MultiNode node;
|
||||||
|
|
||||||
struct MultiNode *childNode;
|
struct MultiNode *childNode;
|
||||||
|
|
||||||
} MultiUnaryNode;
|
} MultiUnaryNode;
|
||||||
|
|
||||||
|
|
||||||
|
@ -62,7 +61,6 @@ typedef struct MultiBinaryNode
|
||||||
|
|
||||||
struct MultiNode *leftChildNode;
|
struct MultiNode *leftChildNode;
|
||||||
struct MultiNode *rightChildNode;
|
struct MultiNode *rightChildNode;
|
||||||
|
|
||||||
} MultiBinaryNode;
|
} MultiBinaryNode;
|
||||||
|
|
||||||
|
|
||||||
|
@ -73,7 +71,6 @@ typedef struct MultiBinaryNode
|
||||||
typedef struct MultiTreeRoot
|
typedef struct MultiTreeRoot
|
||||||
{
|
{
|
||||||
MultiUnaryNode unaryNode;
|
MultiUnaryNode unaryNode;
|
||||||
|
|
||||||
} MultiTreeRoot;
|
} MultiTreeRoot;
|
||||||
|
|
||||||
|
|
||||||
|
@ -91,7 +88,6 @@ typedef struct MultiTable
|
||||||
Alias *alias;
|
Alias *alias;
|
||||||
Alias *referenceNames;
|
Alias *referenceNames;
|
||||||
Query *subquery; /* this field is only valid for non-relation subquery types */
|
Query *subquery; /* this field is only valid for non-relation subquery types */
|
||||||
|
|
||||||
} MultiTable;
|
} MultiTable;
|
||||||
|
|
||||||
|
|
||||||
|
@ -100,7 +96,6 @@ typedef struct MultiProject
|
||||||
{
|
{
|
||||||
MultiUnaryNode unaryNode;
|
MultiUnaryNode unaryNode;
|
||||||
List *columnList;
|
List *columnList;
|
||||||
|
|
||||||
} MultiProject;
|
} MultiProject;
|
||||||
|
|
||||||
|
|
||||||
|
@ -112,7 +107,6 @@ typedef struct MultiProject
|
||||||
typedef struct MultiCollect
|
typedef struct MultiCollect
|
||||||
{
|
{
|
||||||
MultiUnaryNode unaryNode;
|
MultiUnaryNode unaryNode;
|
||||||
|
|
||||||
} MultiCollect;
|
} MultiCollect;
|
||||||
|
|
||||||
|
|
||||||
|
@ -125,7 +119,6 @@ typedef struct MultiSelect
|
||||||
{
|
{
|
||||||
MultiUnaryNode unaryNode;
|
MultiUnaryNode unaryNode;
|
||||||
List *selectClauseList;
|
List *selectClauseList;
|
||||||
|
|
||||||
} MultiSelect;
|
} MultiSelect;
|
||||||
|
|
||||||
|
|
||||||
|
@ -140,7 +133,6 @@ typedef struct MultiJoin
|
||||||
List *joinClauseList;
|
List *joinClauseList;
|
||||||
JoinRuleType joinRuleType;
|
JoinRuleType joinRuleType;
|
||||||
JoinType joinType;
|
JoinType joinType;
|
||||||
|
|
||||||
} MultiJoin;
|
} MultiJoin;
|
||||||
|
|
||||||
|
|
||||||
|
@ -150,7 +142,6 @@ typedef struct MultiPartition
|
||||||
MultiUnaryNode unaryNode;
|
MultiUnaryNode unaryNode;
|
||||||
Var *partitionColumn;
|
Var *partitionColumn;
|
||||||
uint32 splitPointTableId;
|
uint32 splitPointTableId;
|
||||||
|
|
||||||
} MultiPartition;
|
} MultiPartition;
|
||||||
|
|
||||||
|
|
||||||
|
@ -158,7 +149,6 @@ typedef struct MultiPartition
|
||||||
typedef struct MultiCartesianProduct
|
typedef struct MultiCartesianProduct
|
||||||
{
|
{
|
||||||
MultiBinaryNode binaryNode;
|
MultiBinaryNode binaryNode;
|
||||||
|
|
||||||
} MultiCartesianProduct;
|
} MultiCartesianProduct;
|
||||||
|
|
||||||
|
|
||||||
|
@ -183,7 +173,6 @@ typedef struct MultiExtendedOp
|
||||||
List *sortClauseList;
|
List *sortClauseList;
|
||||||
Node *limitCount;
|
Node *limitCount;
|
||||||
Node *limitOffset;
|
Node *limitOffset;
|
||||||
|
|
||||||
} MultiExtendedOp;
|
} MultiExtendedOp;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,8 @@
|
||||||
(" UINT64_FORMAT ", %d, %s, '%s', %d, %d)"
|
(" UINT64_FORMAT ", %d, %s, '%s', %d, %d)"
|
||||||
#define MERGE_FILES_INTO_TABLE_COMMAND "SELECT worker_merge_files_into_table \
|
#define MERGE_FILES_INTO_TABLE_COMMAND "SELECT worker_merge_files_into_table \
|
||||||
(" UINT64_FORMAT ", %d, '%s', '%s')"
|
(" UINT64_FORMAT ", %d, '%s', '%s')"
|
||||||
#define MERGE_FILES_AND_RUN_QUERY_COMMAND "SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')"
|
#define MERGE_FILES_AND_RUN_QUERY_COMMAND \
|
||||||
|
"SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')"
|
||||||
|
|
||||||
|
|
||||||
typedef enum CitusRTEKind
|
typedef enum CitusRTEKind
|
||||||
|
@ -62,7 +63,6 @@ typedef enum
|
||||||
PARTITION_INVALID_FIRST = 0,
|
PARTITION_INVALID_FIRST = 0,
|
||||||
RANGE_PARTITION_TYPE = 1,
|
RANGE_PARTITION_TYPE = 1,
|
||||||
HASH_PARTITION_TYPE = 2
|
HASH_PARTITION_TYPE = 2
|
||||||
|
|
||||||
} PartitionType;
|
} PartitionType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -77,7 +77,6 @@ typedef enum
|
||||||
MAP_OUTPUT_FETCH_TASK = 5,
|
MAP_OUTPUT_FETCH_TASK = 5,
|
||||||
MERGE_FETCH_TASK = 6,
|
MERGE_FETCH_TASK = 6,
|
||||||
MODIFY_TASK = 7
|
MODIFY_TASK = 7
|
||||||
|
|
||||||
} TaskType;
|
} TaskType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -88,7 +87,6 @@ typedef enum
|
||||||
TASK_ASSIGNMENT_GREEDY = 1,
|
TASK_ASSIGNMENT_GREEDY = 1,
|
||||||
TASK_ASSIGNMENT_ROUND_ROBIN = 2,
|
TASK_ASSIGNMENT_ROUND_ROBIN = 2,
|
||||||
TASK_ASSIGNMENT_FIRST_REPLICA = 3
|
TASK_ASSIGNMENT_FIRST_REPLICA = 3
|
||||||
|
|
||||||
} TaskAssignmentPolicyType;
|
} TaskAssignmentPolicyType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -99,7 +97,6 @@ typedef enum
|
||||||
JOIN_MAP_MERGE_JOB = 1,
|
JOIN_MAP_MERGE_JOB = 1,
|
||||||
SUBQUERY_MAP_MERGE_JOB = 2,
|
SUBQUERY_MAP_MERGE_JOB = 2,
|
||||||
TOP_LEVEL_WORKER_JOB = 3
|
TOP_LEVEL_WORKER_JOB = 3
|
||||||
|
|
||||||
} BoundaryNodeJobType;
|
} BoundaryNodeJobType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -133,7 +130,6 @@ typedef struct MapMergeJob
|
||||||
ShardInterval **sortedShardIntervalArray; /* only applies to range partitioning */
|
ShardInterval **sortedShardIntervalArray; /* only applies to range partitioning */
|
||||||
List *mapTaskList;
|
List *mapTaskList;
|
||||||
List *mergeTaskList;
|
List *mergeTaskList;
|
||||||
|
|
||||||
} MapMergeJob;
|
} MapMergeJob;
|
||||||
|
|
||||||
|
|
||||||
|
@ -164,7 +160,6 @@ typedef struct Task
|
||||||
uint64 shardId; /* only applies to shard fetch tasks */
|
uint64 shardId; /* only applies to shard fetch tasks */
|
||||||
TaskExecution *taskExecution; /* used by task tracker executor */
|
TaskExecution *taskExecution; /* used by task tracker executor */
|
||||||
bool upsertQuery; /* only applies to modify tasks */
|
bool upsertQuery; /* only applies to modify tasks */
|
||||||
|
|
||||||
} Task;
|
} Task;
|
||||||
|
|
||||||
|
|
||||||
|
@ -177,7 +172,6 @@ typedef struct RangeTableFragment
|
||||||
CitusRTEKind fragmentType;
|
CitusRTEKind fragmentType;
|
||||||
void *fragmentReference;
|
void *fragmentReference;
|
||||||
uint32 rangeTableId;
|
uint32 rangeTableId;
|
||||||
|
|
||||||
} RangeTableFragment;
|
} RangeTableFragment;
|
||||||
|
|
||||||
|
|
||||||
|
@ -190,7 +184,6 @@ typedef struct JoinSequenceNode
|
||||||
{
|
{
|
||||||
uint32 rangeTableId;
|
uint32 rangeTableId;
|
||||||
int32 joiningRangeTableId;
|
int32 joiningRangeTableId;
|
||||||
|
|
||||||
} JoinSequenceNode;
|
} JoinSequenceNode;
|
||||||
|
|
||||||
|
|
||||||
|
@ -203,7 +196,6 @@ typedef struct MultiPlan
|
||||||
Job *workerJob;
|
Job *workerJob;
|
||||||
Query *masterQuery;
|
Query *masterQuery;
|
||||||
char *masterTableName;
|
char *masterTableName;
|
||||||
|
|
||||||
} MultiPlan;
|
} MultiPlan;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
#include "nodes/plannodes.h"
|
#include "nodes/plannodes.h"
|
||||||
#include "nodes/relation.h"
|
#include "nodes/relation.h"
|
||||||
|
|
||||||
extern PlannedStmt *multi_planner(Query *parse, int cursorOptions,
|
extern PlannedStmt * multi_planner(Query *parse, int cursorOptions,
|
||||||
ParamListInfo boundParams);
|
ParamListInfo boundParams);
|
||||||
|
|
||||||
extern bool HasCitusToplevelNode(PlannedStmt *planStatement);
|
extern bool HasCitusToplevelNode(PlannedStmt *planStatement);
|
||||||
|
|
|
@ -32,9 +32,9 @@
|
||||||
|
|
||||||
/* Task tracker executor related defines */
|
/* Task tracker executor related defines */
|
||||||
#define TASK_ASSIGNMENT_QUERY "SELECT task_tracker_assign_task \
|
#define TASK_ASSIGNMENT_QUERY "SELECT task_tracker_assign_task \
|
||||||
("UINT64_FORMAT", %u, %s)"
|
("UINT64_FORMAT ", %u, %s)"
|
||||||
#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT", %u)"
|
#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT ", %u)"
|
||||||
#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT")"
|
#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT ")"
|
||||||
#define JOB_CLEANUP_TASK_ID INT_MAX
|
#define JOB_CLEANUP_TASK_ID INT_MAX
|
||||||
|
|
||||||
|
|
||||||
|
@ -60,7 +60,6 @@ typedef enum
|
||||||
EXEC_TASK_TRACKER_FAILED = 14,
|
EXEC_TASK_TRACKER_FAILED = 14,
|
||||||
EXEC_SOURCE_TASK_TRACKER_RETRY = 15,
|
EXEC_SOURCE_TASK_TRACKER_RETRY = 15,
|
||||||
EXEC_SOURCE_TASK_TRACKER_FAILED = 16
|
EXEC_SOURCE_TASK_TRACKER_FAILED = 16
|
||||||
|
|
||||||
} TaskExecStatus;
|
} TaskExecStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -74,7 +73,6 @@ typedef enum
|
||||||
EXEC_TRANSMIT_TRACKER_RETRY = 4,
|
EXEC_TRANSMIT_TRACKER_RETRY = 4,
|
||||||
EXEC_TRANSMIT_TRACKER_FAILED = 5,
|
EXEC_TRANSMIT_TRACKER_FAILED = 5,
|
||||||
EXEC_TRANSMIT_DONE = 6
|
EXEC_TRANSMIT_DONE = 6
|
||||||
|
|
||||||
} TransmitExecStatus;
|
} TransmitExecStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -86,7 +84,6 @@ typedef enum
|
||||||
TRACKER_CONNECT_POLL = 2,
|
TRACKER_CONNECT_POLL = 2,
|
||||||
TRACKER_CONNECTED = 3,
|
TRACKER_CONNECTED = 3,
|
||||||
TRACKER_CONNECTION_FAILED = 4
|
TRACKER_CONNECTION_FAILED = 4
|
||||||
|
|
||||||
} TrackerStatus;
|
} TrackerStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -97,7 +94,6 @@ typedef enum
|
||||||
MULTI_EXECUTOR_REAL_TIME = 1,
|
MULTI_EXECUTOR_REAL_TIME = 1,
|
||||||
MULTI_EXECUTOR_TASK_TRACKER = 2,
|
MULTI_EXECUTOR_TASK_TRACKER = 2,
|
||||||
MULTI_EXECUTOR_ROUTER = 3
|
MULTI_EXECUTOR_ROUTER = 3
|
||||||
|
|
||||||
} MultiExecutorType;
|
} MultiExecutorType;
|
||||||
|
|
||||||
|
|
||||||
|
@ -107,7 +103,6 @@ typedef enum
|
||||||
CONNECT_ACTION_NONE = 0,
|
CONNECT_ACTION_NONE = 0,
|
||||||
CONNECT_ACTION_OPENED = 1,
|
CONNECT_ACTION_OPENED = 1,
|
||||||
CONNECT_ACTION_CLOSED = 2
|
CONNECT_ACTION_CLOSED = 2
|
||||||
|
|
||||||
} ConnectAction;
|
} ConnectAction;
|
||||||
|
|
||||||
|
|
||||||
|
@ -132,7 +127,6 @@ struct TaskExecution
|
||||||
uint32 querySourceNodeIndex; /* only applies to map fetch tasks */
|
uint32 querySourceNodeIndex; /* only applies to map fetch tasks */
|
||||||
int32 dataFetchTaskIndex;
|
int32 dataFetchTaskIndex;
|
||||||
uint32 failureCount;
|
uint32 failureCount;
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -147,7 +141,6 @@ typedef struct TrackerTaskState
|
||||||
uint32 taskId;
|
uint32 taskId;
|
||||||
TaskStatus status;
|
TaskStatus status;
|
||||||
StringInfo taskAssignmentQuery;
|
StringInfo taskAssignmentQuery;
|
||||||
|
|
||||||
} TrackerTaskState;
|
} TrackerTaskState;
|
||||||
|
|
||||||
|
|
||||||
|
@ -171,7 +164,6 @@ typedef struct TaskTracker
|
||||||
int32 currentTaskIndex;
|
int32 currentTaskIndex;
|
||||||
bool connectionBusy;
|
bool connectionBusy;
|
||||||
TrackerTaskState *connectionBusyOnTask;
|
TrackerTaskState *connectionBusyOnTask;
|
||||||
|
|
||||||
} TaskTracker;
|
} TaskTracker;
|
||||||
|
|
||||||
|
|
||||||
|
@ -184,7 +176,6 @@ typedef struct WorkerNodeState
|
||||||
uint32 workerPort;
|
uint32 workerPort;
|
||||||
char workerName[WORKER_LENGTH];
|
char workerName[WORKER_LENGTH];
|
||||||
uint32 openConnectionCount;
|
uint32 openConnectionCount;
|
||||||
|
|
||||||
} WorkerNodeState;
|
} WorkerNodeState;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,6 @@ typedef enum
|
||||||
FILE_CACHED = 2,
|
FILE_CACHED = 2,
|
||||||
FILE_INACTIVE = 3,
|
FILE_INACTIVE = 3,
|
||||||
FILE_TO_DELETE = 4
|
FILE_TO_DELETE = 4
|
||||||
|
|
||||||
} RelayFileState;
|
} RelayFileState;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -29,6 +29,7 @@ typedef enum AdvisoryLocktagClass
|
||||||
/* values defined in postgres' lockfuncs.c */
|
/* values defined in postgres' lockfuncs.c */
|
||||||
ADV_LOCKTAG_CLASS_INT64 = 1,
|
ADV_LOCKTAG_CLASS_INT64 = 1,
|
||||||
ADV_LOCKTAG_CLASS_INT32 = 2,
|
ADV_LOCKTAG_CLASS_INT32 = 2,
|
||||||
|
|
||||||
/* CitusDB lock types */
|
/* CitusDB lock types */
|
||||||
ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4,
|
ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4,
|
||||||
ADV_LOCKTAG_CLASS_CITUS_SHARD = 5,
|
ADV_LOCKTAG_CLASS_CITUS_SHARD = 5,
|
||||||
|
|
|
@ -63,7 +63,6 @@ typedef enum
|
||||||
* TASK_STATUS_LAST, should never have their numbers changed.
|
* TASK_STATUS_LAST, should never have their numbers changed.
|
||||||
*/
|
*/
|
||||||
TASK_STATUS_LAST
|
TASK_STATUS_LAST
|
||||||
|
|
||||||
} TaskStatus;
|
} TaskStatus;
|
||||||
|
|
||||||
|
|
||||||
|
@ -85,7 +84,6 @@ typedef struct WorkerTask
|
||||||
char databaseName[NAMEDATALEN]; /* name to use for local backend connection */
|
char databaseName[NAMEDATALEN]; /* name to use for local backend connection */
|
||||||
int32 connectionId; /* connection id to local backend */
|
int32 connectionId; /* connection id to local backend */
|
||||||
uint32 failureCount; /* number of task failures */
|
uint32 failureCount; /* number of task failures */
|
||||||
|
|
||||||
} WorkerTask;
|
} WorkerTask;
|
||||||
|
|
||||||
|
|
||||||
|
@ -97,6 +95,7 @@ typedef struct WorkerTasksSharedStateData
|
||||||
{
|
{
|
||||||
/* Hash table shared by the task tracker and task tracker protocol functions */
|
/* Hash table shared by the task tracker and task tracker protocol functions */
|
||||||
HTAB *taskHash;
|
HTAB *taskHash;
|
||||||
|
|
||||||
/* Lock protecting workerNodesHash */
|
/* Lock protecting workerNodesHash */
|
||||||
LWLock *taskHashLock;
|
LWLock *taskHashLock;
|
||||||
} WorkerTasksSharedStateData;
|
} WorkerTasksSharedStateData;
|
||||||
|
|
|
@ -48,7 +48,6 @@ typedef struct WorkerNode
|
||||||
char workerRack[WORKER_LENGTH]; /* node's network location */
|
char workerRack[WORKER_LENGTH]; /* node's network location */
|
||||||
|
|
||||||
bool inWorkerFile; /* is node in current membership file? */
|
bool inWorkerFile; /* is node in current membership file? */
|
||||||
|
|
||||||
} WorkerNode;
|
} WorkerNode;
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -65,7 +65,6 @@ typedef struct RangePartitionContext
|
||||||
FmgrInfo *comparisonFunction;
|
FmgrInfo *comparisonFunction;
|
||||||
Datum *splitPointArray;
|
Datum *splitPointArray;
|
||||||
int32 splitPointCount;
|
int32 splitPointCount;
|
||||||
|
|
||||||
} RangePartitionContext;
|
} RangePartitionContext;
|
||||||
|
|
||||||
|
|
||||||
|
@ -77,7 +76,6 @@ typedef struct HashPartitionContext
|
||||||
{
|
{
|
||||||
FmgrInfo *hashFunction;
|
FmgrInfo *hashFunction;
|
||||||
uint32 partitionCount;
|
uint32 partitionCount;
|
||||||
|
|
||||||
} HashPartitionContext;
|
} HashPartitionContext;
|
||||||
|
|
||||||
|
|
||||||
|
@ -114,7 +112,6 @@ typedef struct FileOutputStream
|
||||||
File fileDescriptor;
|
File fileDescriptor;
|
||||||
StringInfo fileBuffer;
|
StringInfo fileBuffer;
|
||||||
StringInfo filePath;
|
StringInfo filePath;
|
||||||
|
|
||||||
} FileOutputStream;
|
} FileOutputStream;
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue