From 698d585fb5eac7a097e28bab2d4d4b668f0c8d45 Mon Sep 17 00:00:00 2001 From: velioglu Date: Tue, 26 Dec 2017 14:43:32 +0300 Subject: [PATCH] Remove broadcast join logic After this change all the logic related to shard data fetch logic will be removed. Planner won't plan any ShardFetchTask anymore. Shard fetch related steps in real time executor and task-tracker executor have been removed. --- Makefile | 1 - src/backend/distributed/Makefile | 5 +- .../distributed/citus--7.3-3--7.4-1.sql | 5 + src/backend/distributed/citus.control | 2 +- .../executor/multi_real_time_executor.c | 108 +--- .../executor/multi_server_executor.c | 2 - .../executor/multi_task_tracker_executor.c | 61 +- .../master/master_expire_table_cache.c | 204 ------- .../planner/multi_physical_planner.c | 200 +------ .../planner/multi_router_planner.c | 1 - src/backend/distributed/shared_library_init.c | 10 +- .../distributed/utils/citus_copyfuncs.c | 2 - .../distributed/utils/citus_outfuncs.c | 1 - .../worker/worker_data_fetch_protocol.c | 538 ++---------------- .../distributed/multi_physical_planner.h | 20 +- .../distributed/multi_server_executor.h | 12 +- src/include/distributed/worker_protocol.h | 9 +- .../expected/multi_expire_table_cache.out | 108 ---- src/test/regress/expected/multi_extension.out | 1 + .../multi_large_table_join_planning.out | 150 ++--- .../multi_large_table_join_planning_0.out | 150 ++--- .../expected/multi_large_table_pruning.out | 52 +- .../multi_large_table_task_assignment.out | 170 +++--- .../multi_large_table_task_assignment_0.out | 170 +++--- .../expected/multi_modifying_xacts.out | 2 +- src/test/regress/expected/multi_multiuser.out | 5 +- .../regress/expected/multi_mx_explain.out | 4 + .../regress/expected/multi_mx_explain_0.out | 4 + .../expected/multi_mx_modifying_xacts.out | 4 +- .../multi_mx_repartition_udt_prepare.out | 6 +- .../expected/multi_mx_repartition_udt_w1.out | 7 +- .../expected/multi_mx_repartition_udt_w2.out | 7 +- .../expected/multi_repartition_udt.out | 6 +- .../regress/expected/multi_router_planner.out | 110 +++- .../regress/expected/multi_simple_queries.out | 18 +- .../expected/multi_simple_queries_0.out | 18 +- src/test/regress/expected/multi_subquery.out | 61 +- .../multi_subquery_behavioral_analytics.out | 21 + .../expected/multi_task_assignment_policy.out | 54 +- .../multi_task_assignment_policy_0.out | 60 +- .../set_operation_and_local_tables.out | 16 +- src/test/regress/expected/set_operations.out | 32 +- .../input/multi_append_table_to_shard.source | 30 +- .../regress/input/multi_mx_copy_data.source | 7 +- src/test/regress/multi_schedule | 8 +- .../output/multi_append_table_to_shard.source | 47 +- .../regress/output/multi_mx_copy_data.source | 7 +- .../regress/sql/multi_expire_table_cache.sql | 53 -- src/test/regress/sql/multi_extension.sql | 1 + .../regress/sql/multi_modifying_xacts.sql | 2 +- src/test/regress/sql/multi_multiuser.sql | 7 +- src/test/regress/sql/multi_mx_explain.sql | 5 + .../regress/sql/multi_mx_modifying_xacts.sql | 4 +- .../sql/multi_mx_repartition_udt_prepare.sql | 6 +- .../sql/multi_mx_repartition_udt_w1.sql | 6 +- .../sql/multi_mx_repartition_udt_w2.sql | 6 +- .../regress/sql/multi_repartition_udt.sql | 6 +- src/test/regress/sql/multi_router_planner.sql | 13 +- src/test/regress/sql/multi_simple_queries.sql | 5 + src/test/regress/sql/multi_subquery.sql | 32 -- .../multi_subquery_behavioral_analytics.sql | 18 + 61 files changed, 759 insertions(+), 1921 deletions(-) create mode 100644 src/backend/distributed/citus--7.3-3--7.4-1.sql delete mode 100644 src/backend/distributed/master/master_expire_table_cache.c delete mode 100644 src/test/regress/expected/multi_expire_table_cache.out delete mode 100644 src/test/regress/sql/multi_expire_table_cache.sql diff --git a/Makefile b/Makefile index 6eb203207..b855e481f 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,6 @@ OBJS = src/backend/distributed/shared_library_init.o \ src/backend/distributed/master/master_citus_tools.o \ src/backend/distributed/master/master_create_shards.o \ src/backend/distributed/master/master_delete_protocol.o \ - src/backend/distributed/master/master_expire_table_cache.o \ src/backend/distributed/master/master_metadata_utility.o \ src/backend/distributed/master/master_modify_multiple_shards.o \ src/backend/distributed/master/master_node_protocol.o \ diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index 470737813..4adb22b30 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -14,7 +14,8 @@ EXTVERSIONS = 5.0 5.0-1 5.0-2 \ 7.0-1 7.0-2 7.0-3 7.0-4 7.0-5 7.0-6 7.0-7 7.0-8 7.0-9 7.0-10 7.0-11 7.0-12 7.0-13 7.0-14 7.0-15 \ 7.1-1 7.1-2 7.1-3 7.1-4 \ 7.2-1 7.2-2 7.2-3 \ - 7.3-1 7.3-2 7.3-3 + 7.3-1 7.3-2 7.3-3 \ + 7.4-1 # All citus--*.sql files in the source directory DATA = $(patsubst $(citus_abs_srcdir)/%.sql,%.sql,$(wildcard $(citus_abs_srcdir)/$(EXTENSION)--*--*.sql)) @@ -192,6 +193,8 @@ $(EXTENSION)--7.3-2.sql: $(EXTENSION)--7.3-1.sql $(EXTENSION)--7.3-1--7.3-2.sql cat $^ > $@ $(EXTENSION)--7.3-3.sql: $(EXTENSION)--7.3-2.sql $(EXTENSION)--7.3-2--7.3-3.sql cat $^ > $@ +$(EXTENSION)--7.4-1.sql: $(EXTENSION)--7.3-3.sql $(EXTENSION)--7.3-3--7.4-1.sql + cat $^ > $@ NO_PGXS = 1 diff --git a/src/backend/distributed/citus--7.3-3--7.4-1.sql b/src/backend/distributed/citus--7.3-3--7.4-1.sql new file mode 100644 index 000000000..037f6e282 --- /dev/null +++ b/src/backend/distributed/citus--7.3-3--7.4-1.sql @@ -0,0 +1,5 @@ +/* citus--7.3-3--7.4-1 */ + +DROP FUNCTION IF EXISTS master_expire_table_cache(regclass); +DROP FUNCTION IF EXISTS pg_catalog.worker_fetch_regular_table(text, bigint, text[], integer[]); +DROP FUNCTION IF EXISTS pg_catalog.worker_fetch_foreign_file(text, bigint, text[], integer[]); diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control index 5287d7826..5493b8234 100644 --- a/src/backend/distributed/citus.control +++ b/src/backend/distributed/citus.control @@ -1,6 +1,6 @@ # Citus extension comment = 'Citus distributed database' -default_version = '7.3-3' +default_version = '7.4-1' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog diff --git a/src/backend/distributed/executor/multi_real_time_executor.c b/src/backend/distributed/executor/multi_real_time_executor.c index 3f3741499..067d101d3 100644 --- a/src/backend/distributed/executor/multi_real_time_executor.c +++ b/src/backend/distributed/executor/multi_real_time_executor.c @@ -331,20 +331,18 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution, ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); /* - * If the connection is established, we reset the data fetch counter and - * change our status to data fetching. + * If the connection is established, we change our state based on + * whether a coordinated transaction has been started. */ if (pollStatus == CLIENT_CONNECTION_READY) { - taskExecution->dataFetchTaskIndex = -1; - if (InCoordinatedTransaction()) { taskStatusArray[currentIndex] = EXEC_BEGIN_START; } else { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; } } else if (pollStatus == CLIENT_CONNECTION_BUSY) @@ -393,8 +391,8 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution, /* * On task failure, we close the connection. We also reset our execution * status assuming that we might fail on all other worker nodes and come - * back to this failed node. In that case, we will retry the same fetch - * and compute task(s) on this node again. + * back to this failed node. In that case, we will retry compute task(s) + * on this node again. */ int32 connectionId = connectionIdArray[currentIndex]; MultiConnection *connection = MultiClientGetConnection(connectionId); @@ -453,11 +451,6 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution, } else { - /* - * We skip data fetches when in a distributed transaction since - * they cannot be performed in a transactional way (e.g. would - * trigger deadlock detection). - */ taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; break; } @@ -493,101 +486,11 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution, } else { - /* - * We skip data fetches when in a distributed transaction since - * they cannot be performed in a transactional way (e.g. would - * trigger deadlock detection). - */ taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; break; } } - case EXEC_FETCH_TASK_LOOP: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskCount = list_length(dataFetchTaskList); - - /* move to the next data fetch task */ - taskExecution->dataFetchTaskIndex++; - - if (taskExecution->dataFetchTaskIndex < dataFetchTaskCount) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_START; - } - else - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; - } - - break; - } - - case EXEC_FETCH_TASK_START: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex; - Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, - dataFetchTaskIndex); - - char *dataFetchQuery = dataFetchTask->queryString; - int32 connectionId = connectionIdArray[currentIndex]; - - bool querySent = MultiClientSendQuery(connectionId, dataFetchQuery); - if (querySent) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - } - else - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - break; - } - - case EXEC_FETCH_TASK_RUNNING: - { - int32 connectionId = connectionIdArray[currentIndex]; - ResultStatus resultStatus = MultiClientResultStatus(connectionId); - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - - /* check if query results are in progress or unavailable */ - if (resultStatus == CLIENT_RESULT_BUSY) - { - *executionStatus = TASK_STATUS_SOCKET_READ; - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - break; - } - else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - break; - } - - Assert(resultStatus == CLIENT_RESULT_READY); - - /* - * If the query executed successfully, loop onto the next data fetch - * task. Else if the query failed, try data fetching on another node. - */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_DONE) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; - } - else if (queryStatus == CLIENT_QUERY_FAILED) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - else - { - ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); - } - - break; - } - case EXEC_COMPUTE_TASK_START: { int32 connectionId = connectionIdArray[currentIndex]; @@ -827,7 +730,6 @@ CancelRequestIfActive(TaskExecStatus taskStatus, int connectionId) /* * We use the task status to determine if we have an active request being * processed by the worker node. If we do, we send a cancellation request. - * Note that we don't cancel data fetch tasks, and allow them to complete. */ if (taskStatus == EXEC_COMPUTE_TASK_RUNNING) { diff --git a/src/backend/distributed/executor/multi_server_executor.c b/src/backend/distributed/executor/multi_server_executor.c index 46d408fe8..6ae88d54e 100644 --- a/src/backend/distributed/executor/multi_server_executor.c +++ b/src/backend/distributed/executor/multi_server_executor.c @@ -182,7 +182,6 @@ InitTaskExecution(Task *task, TaskExecStatus initialTaskExecStatus) taskExecution->nodeCount = nodeCount; taskExecution->connectStartTime = 0; taskExecution->currentNodeIndex = 0; - taskExecution->dataFetchTaskIndex = -1; taskExecution->failureCount = 0; taskExecution->taskStatusArray = palloc0(nodeCount * sizeof(TaskExecStatus)); @@ -283,7 +282,6 @@ AdjustStateForFailure(TaskExecution *taskExecution) taskExecution->currentNodeIndex = 0; /* go back to the first worker node */ } - taskExecution->dataFetchTaskIndex = -1; /* reset data fetch counter */ taskExecution->failureCount++; /* record failure */ } diff --git a/src/backend/distributed/executor/multi_task_tracker_executor.c b/src/backend/distributed/executor/multi_task_tracker_executor.c index 47c05e9e3..2f57906ab 100644 --- a/src/backend/distributed/executor/multi_task_tracker_executor.c +++ b/src/backend/distributed/executor/multi_task_tracker_executor.c @@ -121,7 +121,6 @@ static List * ConstrainedMergeTaskList(List *taskAndExecutionList, Task *task); static List * MergeTaskList(List *taskList); static void ReassignTaskList(List *taskList); static void ReassignMapFetchTaskList(List *mapFetchTaskList); -static List * ShardFetchTaskList(List *taskList); /* Local functions forward declarations to manage task trackers */ static void ManageTaskTracker(TaskTracker *taskTracker); @@ -1839,9 +1838,9 @@ TransmitTrackerConnectionId(TaskTracker *transmitTracker, Task *task) * ConstrainedTaskList finds the given task's constraint group within the given * task and execution list. We define a constraint group as all tasks that need * to be assigned (or reassigned) to the same task tracker for query execution - * to complete. At a high level, compute tasks and their data fetch dependencies - * are part of the same constraint group. Also, the transitive closure of tasks - * that have the same merge task dependency are part of one constraint group. + * to complete. At a high level, compute tasks are part of the same constraint + * group. Also, the transitive closure of tasks that have the same merge task + * dependency are part of one constraint group. */ static List * ConstrainedTaskList(List *taskAndExecutionList, Task *task) @@ -1907,8 +1906,7 @@ ConstrainedTaskList(List *taskAndExecutionList, Task *task) /* * ConstrainedNonMergeTaskList finds the constraint group for the given task, * assuming that the given task doesn't have any merge task dependencies. This - * constraint group includes a compute task and its downstream data fetch task - * dependencies. + * constraint group includes compute task. */ static List * ConstrainedNonMergeTaskList(List *taskAndExecutionList, Task *task) @@ -1923,14 +1921,6 @@ ConstrainedNonMergeTaskList(List *taskAndExecutionList, Task *task) upstreamTask = task; dependedTaskList = upstreamTask->dependedTaskList; } - else if (taskType == SHARD_FETCH_TASK) - { - List *upstreamTaskList = UpstreamDependencyList(taskAndExecutionList, task); - Assert(list_length(upstreamTaskList) == 1); - - upstreamTask = (Task *) linitial(upstreamTaskList); - dependedTaskList = upstreamTask->dependedTaskList; - } Assert(upstreamTask != NULL); constrainedTaskList = list_make1(upstreamTask); @@ -2008,20 +1998,6 @@ ConstrainedMergeTaskList(List *taskAndExecutionList, Task *task) { constrainedMergeTaskList = MergeTaskList(task->dependedTaskList); } - else if (taskType == SHARD_FETCH_TASK) - { - Task *upstreamTask = NULL; - List *upstreamTaskList = UpstreamDependencyList(taskAndExecutionList, task); - - /* - * A shard fetch task can only have one SQL/map task parent. We now get - * that parent. From the parent, we find any merge task dependencies. - */ - Assert(list_length(upstreamTaskList) == 1); - upstreamTask = (Task *) linitial(upstreamTaskList); - - constrainedMergeTaskList = MergeTaskList(upstreamTask->dependedTaskList); - } else if (taskType == MAP_OUTPUT_FETCH_TASK) { List *taskList = UpstreamDependencyList(taskAndExecutionList, task); @@ -2093,8 +2069,7 @@ ReassignTaskList(List *taskList) /* * As an optimization, we first find the SQL tasks whose results we already - * fetched to the master node. We don't need to re-execute these SQL tasks - * or their shard fetch dependencies. + * fetched to the master node. We don't need to re-execute these SQL tasks. */ foreach(taskCell, taskList) { @@ -2104,10 +2079,7 @@ ReassignTaskList(List *taskList) bool transmitCompleted = TransmitExecutionCompleted(taskExecution); if ((task->taskType == SQL_TASK) && transmitCompleted) { - List *shardFetchTaskList = ShardFetchTaskList(task->dependedTaskList); - completedTaskList = lappend(completedTaskList, task); - completedTaskList = TaskListUnion(completedTaskList, shardFetchTaskList); } } @@ -2162,29 +2134,6 @@ ReassignMapFetchTaskList(List *mapFetchTaskList) } -/* - * ShardFetchTaskList walks over the given task list, finds the shard fetch tasks - * in the list, and returns the found tasks in a new list. - */ -static List * -ShardFetchTaskList(List *taskList) -{ - List *shardFetchTaskList = NIL; - ListCell *taskCell = NULL; - - foreach(taskCell, taskList) - { - Task *task = (Task *) lfirst(taskCell); - if (task->taskType == SHARD_FETCH_TASK) - { - shardFetchTaskList = lappend(shardFetchTaskList, task); - } - } - - return shardFetchTaskList; -} - - /* * ManageTaskTracker manages tasks assigned to the given task tracker. For this, * the function coordinates access to the underlying connection. The function diff --git a/src/backend/distributed/master/master_expire_table_cache.c b/src/backend/distributed/master/master_expire_table_cache.c deleted file mode 100644 index 4ddc31656..000000000 --- a/src/backend/distributed/master/master_expire_table_cache.c +++ /dev/null @@ -1,204 +0,0 @@ -/*------------------------------------------------------------------------- - * - * master_expire_table_cache.c - * UDF to refresh shard cache at workers - * - * This file contains master_expire_table_cache function. The function - * accepts a table name and drops tables cached shards from all workers. - * It does not change existing shard placement. Only drops cached copies - * of shards. - * - * Copyright (c) 2012-2016, Citus Data, Inc. - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" -#include "funcapi.h" -#include "libpq-fe.h" - -#include "catalog/pg_class.h" -#include "distributed/connection_management.h" -#include "distributed/master_protocol.h" -#include "distributed/metadata_cache.h" -#include "distributed/multi_join_order.h" -#include "distributed/pg_dist_shard.h" -#include "distributed/remote_commands.h" -#include "distributed/worker_manager.h" -#include "distributed/worker_protocol.h" -#include "utils/builtins.h" -#include "utils/lsyscache.h" - - -static List * FindAbsentShardPlacementsOnWorker(WorkerNode *workerNode, - ShardInterval **shardIntervalArray, - List **placementListArray, - int shardCount); -static void DropShardsFromWorker(WorkerNode *workerNode, Oid relationId, - List *shardIntervalList); - -PG_FUNCTION_INFO_V1(master_expire_table_cache); - - -/* - * master_expire_table_cache drops table's caches shards in all workers. The function - * expects a passed table to be a small distributed table meaning it has less than - * large_table_shard_count. - */ -Datum -master_expire_table_cache(PG_FUNCTION_ARGS) -{ - Oid relationId = PG_GETARG_OID(0); - DistTableCacheEntry *cacheEntry = NULL; - List *workerNodeList = NIL; - ListCell *workerNodeCell = NULL; - int shardCount = 0; - ShardInterval **shardIntervalArray = NULL; - List **placementListArray = NULL; - int shardIndex = 0; - - CheckCitusVersion(ERROR); - - cacheEntry = DistributedTableCacheEntry(relationId); - workerNodeList = ActivePrimaryNodeList(); - shardCount = cacheEntry->shardIntervalArrayLength; - shardIntervalArray = cacheEntry->sortedShardIntervalArray; - - if (shardCount == 0) - { - ereport(WARNING, (errmsg("Table has no shards, no action is taken"))); - PG_RETURN_VOID(); - } - - if (shardCount >= LargeTableShardCount) - { - ereport(ERROR, (errmsg("Must be called on tables smaller than %d shards", - LargeTableShardCount))); - } - - placementListArray = palloc(shardCount * sizeof(List *)); - - for (shardIndex = 0; shardIndex < shardCount; shardIndex++) - { - ShardInterval *shardInterval = shardIntervalArray[shardIndex]; - placementListArray[shardIndex] = - FinalizedShardPlacementList(shardInterval->shardId); - } - - foreach(workerNodeCell, workerNodeList) - { - WorkerNode *workerNode = (WorkerNode *) lfirst(workerNodeCell); - List *shardDropList = FindAbsentShardPlacementsOnWorker(workerNode, - shardIntervalArray, - placementListArray, - shardCount); - DropShardsFromWorker(workerNode, relationId, shardDropList); - } - - pfree(placementListArray); - - PG_RETURN_VOID(); -} - - -/* - * FindAbsentShardPlacementsOnWorker compiles shard interval list of shards - * that do not have registered placement at given worker node. - */ -List * -FindAbsentShardPlacementsOnWorker(WorkerNode *workerNode, - ShardInterval **shardIntervalArray, - List **placementListArray, int shardCount) -{ - List *absentShardIntervalList = NIL; - - int shardIndex = 0; - for (shardIndex = 0; shardIndex < shardCount; shardIndex++) - { - ShardInterval *shardInterval = shardIntervalArray[shardIndex]; - List *placementList = placementListArray[shardIndex]; - - ListCell *placementCell = NULL; - foreach(placementCell, placementList) - { - ShardPlacement *placement = (ShardPlacement *) lfirst(placementCell); - - /* - * Append shard interval to absent list if none of its placements is on - * the worker. - */ - if (placement->nodePort == workerNode->workerPort && - strncmp(placement->nodeName, workerNode->workerName, WORKER_LENGTH) == 0) - { - break; - } - else if (lnext(placementCell) == NULL) - { - absentShardIntervalList = lappend(absentShardIntervalList, shardInterval); - } - } - } - - return absentShardIntervalList; -} - - -/* - * DropShardsFromWorker drops provided shards belonging to a relation from - * given worker. It does not change any metadata at the master. - */ -static void -DropShardsFromWorker(WorkerNode *workerNode, Oid relationId, List *shardIntervalList) -{ - Oid schemaId = get_rel_namespace(relationId); - char *schemaName = get_namespace_name(schemaId); - char *relationName = get_rel_name(relationId); - char relationKind = get_rel_relkind(relationId); - StringInfo workerCommand = makeStringInfo(); - StringInfo shardNames = makeStringInfo(); - ListCell *shardIntervalCell = NULL; - MultiConnection *connection = NULL; - int connectionFlag = FORCE_NEW_CONNECTION; - PGresult *result = NULL; - - if (shardIntervalList == NIL) - { - return; - } - - foreach(shardIntervalCell, shardIntervalList) - { - ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell); - char *shardName = pstrdup(relationName); - char *quotedShardName = NULL; - - AppendShardIdToName(&shardName, shardInterval->shardId); - quotedShardName = quote_qualified_identifier(schemaName, shardName); - appendStringInfo(shardNames, "%s", quotedShardName); - - /* append a comma after the shard name if there are more shards */ - if (lnext(shardIntervalCell) != NULL) - { - appendStringInfo(shardNames, ", "); - } - } - - if (RegularTable(relationId)) - { - appendStringInfo(workerCommand, DROP_REGULAR_TABLE_COMMAND, shardNames->data); - } - else if (relationKind == RELKIND_FOREIGN_TABLE) - { - appendStringInfo(workerCommand, DROP_FOREIGN_TABLE_COMMAND, shardNames->data); - } - else - { - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("expire target is not a regular, foreign or partitioned " - "table"))); - } - - connection = GetNodeConnection(connectionFlag, workerNode->workerName, - workerNode->workerPort); - ExecuteOptionalRemoteCommand(connection, workerCommand->data, &result); -} diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index c5ae6ce0f..f54f943a7 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -157,8 +157,6 @@ static bool JoinPrunable(RangeTableFragment *leftFragment, static ShardInterval * FragmentInterval(RangeTableFragment *fragment); static StringInfo FragmentIntervalString(ShardInterval *fragmentInterval); static List * DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList); -static StringInfo NodeNameArrayString(List *workerNodeList); -static StringInfo NodePortArrayString(List *workerNodeList); static StringInfo DatumArrayString(Datum *datumArray, uint32 datumCount, Oid datumTypeId); static List * BuildRelationShardList(List *rangeTableList, List *fragmentList); static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList); @@ -3759,8 +3757,9 @@ FragmentIntervalString(ShardInterval *fragmentInterval) /* - * DataFetchTaskList builds a data fetch task for every shard in the given shard - * list, appends these data fetch tasks into a list, and returns this list. + * DataFetchTaskList builds a merge fetch task for every remote query result + * in the given fragment list, appends these merge fetch tasks into a list, + * and returns this list. */ static List * DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList) @@ -3771,20 +3770,7 @@ DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList) foreach(fragmentCell, fragmentList) { RangeTableFragment *fragment = (RangeTableFragment *) lfirst(fragmentCell); - if (fragment->fragmentType == CITUS_RTE_RELATION) - { - ShardInterval *shardInterval = fragment->fragmentReference; - uint64 shardId = shardInterval->shardId; - StringInfo shardFetchQueryString = ShardFetchQueryString(shardId); - - Task *shardFetchTask = CreateBasicTask(jobId, taskIdIndex, SHARD_FETCH_TASK, - shardFetchQueryString->data); - shardFetchTask->shardId = shardId; - - dataFetchTaskList = lappend(dataFetchTaskList, shardFetchTask); - taskIdIndex++; - } - else if (fragment->fragmentType == CITUS_RTE_REMOTE_QUERY) + if (fragment->fragmentType == CITUS_RTE_REMOTE_QUERY) { Task *mergeTask = (Task *) fragment->fragmentReference; char *undefinedQueryString = NULL; @@ -3803,136 +3789,6 @@ DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList) } -/* - * ShardFetchQueryString constructs a query string to fetch the given shard from - * the shards' placements. - */ -StringInfo -ShardFetchQueryString(uint64 shardId) -{ - StringInfo shardFetchQuery = NULL; - uint64 shardLength = ShardLength(shardId); - - /* construct two array strings for node names and port numbers */ - List *shardPlacements = FinalizedShardPlacementList(shardId); - StringInfo nodeNameArrayString = NodeNameArrayString(shardPlacements); - StringInfo nodePortArrayString = NodePortArrayString(shardPlacements); - - /* check storage type to create the correct query string */ - ShardInterval *shardInterval = LoadShardInterval(shardId); - char storageType = shardInterval->storageType; - char *shardSchemaName = NULL; - char *shardTableName = NULL; - - /* construct the shard name */ - Oid shardSchemaId = get_rel_namespace(shardInterval->relationId); - char *tableName = get_rel_name(shardInterval->relationId); - - shardSchemaName = get_namespace_name(shardSchemaId); - shardTableName = pstrdup(tableName); - AppendShardIdToName(&shardTableName, shardId); - - shardFetchQuery = makeStringInfo(); - if (storageType == SHARD_STORAGE_TABLE || storageType == SHARD_STORAGE_RELAY || - storageType == SHARD_STORAGE_COLUMNAR) - { - if (strcmp(shardSchemaName, "public") != 0) - { - char *qualifiedTableName = quote_qualified_identifier(shardSchemaName, - shardTableName); - - appendStringInfo(shardFetchQuery, TABLE_FETCH_COMMAND, qualifiedTableName, - shardLength, nodeNameArrayString->data, - nodePortArrayString->data); - } - else - { - appendStringInfo(shardFetchQuery, TABLE_FETCH_COMMAND, shardTableName, - shardLength, nodeNameArrayString->data, - nodePortArrayString->data); - } - } - else if (storageType == SHARD_STORAGE_FOREIGN) - { - if (strcmp(shardSchemaName, "public") != 0) - { - char *qualifiedTableName = quote_qualified_identifier(shardSchemaName, - shardTableName); - - appendStringInfo(shardFetchQuery, FOREIGN_FETCH_COMMAND, qualifiedTableName, - shardLength, nodeNameArrayString->data, - nodePortArrayString->data); - } - else - { - appendStringInfo(shardFetchQuery, FOREIGN_FETCH_COMMAND, shardTableName, - shardLength, nodeNameArrayString->data, - nodePortArrayString->data); - } - } - - return shardFetchQuery; -} - - -/* - * NodeNameArrayString extracts the node names from the given node list, stores - * these node names in an array, and returns the array's string representation. - */ -static StringInfo -NodeNameArrayString(List *shardPlacementList) -{ - StringInfo nodeNameArrayString = NULL; - ListCell *shardPlacementCell = NULL; - - uint32 nodeNameCount = (uint32) list_length(shardPlacementList); - Datum *nodeNameArray = palloc0(nodeNameCount * sizeof(Datum)); - uint32 nodeNameIndex = 0; - - foreach(shardPlacementCell, shardPlacementList) - { - ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell); - Datum nodeName = CStringGetDatum(shardPlacement->nodeName); - - nodeNameArray[nodeNameIndex] = nodeName; - nodeNameIndex++; - } - - nodeNameArrayString = DatumArrayString(nodeNameArray, nodeNameCount, CSTRINGOID); - - return nodeNameArrayString; -} - - -/* - * NodePortArrayString extracts the node ports from the given node list, stores - * these node ports in an array, and returns the array's string representation. - */ -static StringInfo -NodePortArrayString(List *shardPlacementList) -{ - StringInfo nodePortArrayString = NULL; - ListCell *shardPlacementCell = NULL; - - uint32 nodePortCount = (uint32) list_length(shardPlacementList); - Datum *nodePortArray = palloc0(nodePortCount * sizeof(Datum)); - uint32 nodePortIndex = 0; - - foreach(shardPlacementCell, shardPlacementList) - { - ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell); - Datum nodePort = UInt32GetDatum(shardPlacement->nodePort); - - nodePortArray[nodePortIndex] = nodePort; - nodePortIndex++; - } - - nodePortArrayString = DatumArrayString(nodePortArray, nodePortCount, INT4OID); - - return nodePortArrayString; -} - - /* Helper function to return a datum array's external string representation. */ static StringInfo DatumArrayString(Datum *datumArray, uint32 datumCount, Oid datumTypeId) @@ -4153,10 +4009,8 @@ AnchorShardId(List *fragmentList, uint32 anchorRangeTableId) /* * PruneSqlTaskDependencies iterates over each sql task from the given sql task - * list, and prunes away any data fetch tasks which are redundant or not needed - * for the completion of that task. Specifically the function prunes away data - * fetch tasks for the anchor shard and any merge-fetch tasks, as the task - * assignment algorithm ensures co-location of these tasks. + * list, and prunes away merge-fetch tasks, as the task assignment algorithm + * ensures co-location of these tasks. */ static List * PruneSqlTaskDependencies(List *sqlTaskList) @@ -4174,17 +4028,11 @@ PruneSqlTaskDependencies(List *sqlTaskList) Task *dataFetchTask = (Task *) lfirst(dependedTaskCell); /* - * If we have a shard fetch task for the anchor shard, or if we have - * a merge fetch task, our task assignment algorithm makes sure that - * the sql task is colocated with the anchor shard / merge task. We - * can therefore prune out this data fetch task. + * If we have a merge fetch task, our task assignment algorithm makes + * sure that the sql task is colocated with the anchor shard / merge + * task. We can therefore prune out this data fetch task. */ - if (dataFetchTask->taskType == SHARD_FETCH_TASK && - dataFetchTask->shardId != sqlTask->anchorShardId) - { - prunedDependedTaskList = lappend(prunedDependedTaskList, dataFetchTask); - } - else if (dataFetchTask->taskType == MERGE_FETCH_TASK) + if (dataFetchTask->taskType == MERGE_FETCH_TASK) { Task *mergeTaskReference = NULL; List *mergeFetchDependencyList = dataFetchTask->dependedTaskList; @@ -4827,31 +4675,6 @@ TaskListDifference(const List *list1, const List *list2) } -/* - * TaskListUnion generate the union of two tasks lists. This is calculated by - * copying list1 via list_copy(), then adding to it all the members of list2 - * that aren't already in list1. - */ -List * -TaskListUnion(const List *list1, const List *list2) -{ - const ListCell *taskCell = NULL; - List *resultList = NIL; - - resultList = list_copy(list1); - - foreach(taskCell, list2) - { - if (!TaskListMember(resultList, lfirst(taskCell))) - { - resultList = lappend(resultList, lfirst(taskCell)); - } - } - - return resultList; -} - - /* * AssignAnchorShardTaskList assigns locations to the given tasks based on the * configured task assignment policy. The distributed executor later sends these @@ -5409,8 +5232,7 @@ AssignDataFetchDependencies(List *taskList) foreach(dependedTaskCell, dependedTaskList) { Task *dependedTask = (Task *) lfirst(dependedTaskCell); - if (dependedTask->taskType == SHARD_FETCH_TASK || - dependedTask->taskType == MAP_OUTPUT_FETCH_TASK) + if (dependedTask->taskType == MAP_OUTPUT_FETCH_TASK) { dependedTask->taskPlacementList = task->taskPlacementList; } diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 4eef09f7d..c8d92a902 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -1351,7 +1351,6 @@ CreateTask(TaskType taskType) task->upstreamTaskId = INVALID_TASK_ID; task->shardInterval = NULL; task->assignmentConstrained = false; - task->shardId = INVALID_SHARD_ID; task->taskExecution = NULL; task->upsertQuery = false; task->replicationModel = REPLICATION_MODEL_INVALID; diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index fe852ff1b..e46e20354 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -69,6 +69,9 @@ static void NormalizeWorkerListPath(void); static bool StatisticsCollectionGucCheckHook(bool *newval, void **extra, GucSource source); +/* static variable to hold value of deprecated GUC variable */ +static bool ExpireCachedShards = false; + /* *INDENT-OFF* */ /* GUC enum definitions */ @@ -360,11 +363,8 @@ RegisterCitusConfigVariables(void) DefineCustomBoolVariable( "citus.expire_cached_shards", - gettext_noop("Enables shard cache expiration if a shard's size on disk has " - "changed."), - gettext_noop("When appending to an existing shard, old data may still be cached " - "on other workers. This configuration entry activates automatic " - "expiration, but should not be used with manual updates to shards."), + gettext_noop("This GUC variable has been deprecated."), + NULL, &ExpireCachedShards, false, PGC_SIGHUP, diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index 5ee49a8b7..c4ab25d54 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -241,7 +241,6 @@ CopyNodeTask(COPYFUNC_ARGS) COPY_SCALAR_FIELD(upstreamTaskId); COPY_NODE_FIELD(shardInterval); COPY_SCALAR_FIELD(assignmentConstrained); - COPY_SCALAR_FIELD(shardId); COPY_NODE_FIELD(taskExecution); COPY_SCALAR_FIELD(upsertQuery); COPY_SCALAR_FIELD(replicationModel); @@ -268,7 +267,6 @@ CopyNodeTaskExecution(COPYFUNC_ARGS) COPY_SCALAR_FIELD(connectStartTime); COPY_SCALAR_FIELD(currentNodeIndex); COPY_SCALAR_FIELD(querySourceNodeIndex); - COPY_SCALAR_FIELD(dataFetchTaskIndex); COPY_SCALAR_FIELD(failureCount); } diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index 433007770..45dad90f1 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -477,7 +477,6 @@ OutTaskExecution(OUTFUNC_ARGS) WRITE_INT64_FIELD(connectStartTime); WRITE_UINT_FIELD(currentNodeIndex); WRITE_UINT_FIELD(querySourceNodeIndex); - WRITE_INT_FIELD(dataFetchTaskIndex); WRITE_UINT_FIELD(failureCount); } diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 41b1c8635..cb4939086 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -52,10 +52,6 @@ #endif -/* Config variable managed via guc.c */ -bool ExpireCachedShards = false; - - /* Local functions forward declarations */ static void FetchRegularFileAsSuperUser(const char *nodeName, uint32 nodePort, StringInfo remoteFilename, @@ -66,20 +62,7 @@ static bool ReceiveRegularFile(const char *nodeName, uint32 nodePort, static void ReceiveResourceCleanup(int32 connectionId, const char *filename, int32 fileDescriptor); static void CitusDeleteFile(const char *filename); -static void FetchTableCommon(text *tableName, uint64 remoteTableSize, - ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction)(const char *, uint32, - const char *)); -static uint64 LocalTableSize(Oid relationId); static uint64 ExtractShardId(const char *tableName); -static bool FetchRegularTable(const char *nodeName, uint32 nodePort, - const char *tableName); -static bool FetchForeignTable(const char *nodeName, uint32 nodePort, - const char *tableName); -static const char * RemoteTableOwner(const char *nodeName, uint32 nodePort, - const char *tableName); -static StringInfo ForeignFilePath(const char *nodeName, uint32 nodePort, - const char *tableName); static bool check_log_statement(List *stmt_list); static void AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName); static void SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg); @@ -91,9 +74,15 @@ PG_FUNCTION_INFO_V1(worker_fetch_query_results_file); PG_FUNCTION_INFO_V1(worker_apply_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_inter_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_sequence_command); +PG_FUNCTION_INFO_V1(worker_append_table_to_shard); + +/* + * Following UDFs are stub functions, you can check their comments for more + * detail. + */ PG_FUNCTION_INFO_V1(worker_fetch_regular_table); PG_FUNCTION_INFO_V1(worker_fetch_foreign_file); -PG_FUNCTION_INFO_V1(worker_append_table_to_shard); +PG_FUNCTION_INFO_V1(master_expire_table_cache); /* @@ -537,236 +526,6 @@ worker_apply_sequence_command(PG_FUNCTION_ARGS) } -/* - * worker_fetch_regular_table caches the given PostgreSQL table on the local - * node. The function caches this table by trying the given list of node names - * and node ports in sequential order. On success, the function simply returns. - */ -Datum -worker_fetch_regular_table(PG_FUNCTION_ARGS) -{ - text *regularTableName = PG_GETARG_TEXT_P(0); - uint64 generationStamp = PG_GETARG_INT64(1); - ArrayType *nodeNameObject = PG_GETARG_ARRAYTYPE_P(2); - ArrayType *nodePortObject = PG_GETARG_ARRAYTYPE_P(3); - - CheckCitusVersion(ERROR); - - /* - * Run common logic to fetch the remote table, and use the provided function - * pointer to perform the actual table fetching. - */ - FetchTableCommon(regularTableName, generationStamp, nodeNameObject, nodePortObject, - &FetchRegularTable); - - PG_RETURN_VOID(); -} - - -/* - * worker_fetch_foreign_file caches the given file-backed foreign table on the - * local node. The function caches this table by trying the given list of node - * names and node ports in sequential order. On success, the function returns. - */ -Datum -worker_fetch_foreign_file(PG_FUNCTION_ARGS) -{ - text *foreignTableName = PG_GETARG_TEXT_P(0); - uint64 foreignFileSize = PG_GETARG_INT64(1); - ArrayType *nodeNameObject = PG_GETARG_ARRAYTYPE_P(2); - ArrayType *nodePortObject = PG_GETARG_ARRAYTYPE_P(3); - - CheckCitusVersion(ERROR); - - /* - * Run common logic to fetch the remote table, and use the provided function - * pointer to perform the actual table fetching. - */ - FetchTableCommon(foreignTableName, foreignFileSize, nodeNameObject, nodePortObject, - &FetchForeignTable); - - PG_RETURN_VOID(); -} - - -/* - * FetchTableCommon executes common logic that wraps around the actual data - * fetching function. This common logic includes ensuring that only one process - * tries to fetch this table at any given time, and that data fetch operations - * are retried in case of node failures. - */ -static void -FetchTableCommon(text *tableNameText, uint64 remoteTableSize, - ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction)(const char *, uint32, const char *)) -{ - uint64 shardId = INVALID_SHARD_ID; - Oid relationId = InvalidOid; - List *relationNameList = NIL; - RangeVar *relation = NULL; - uint32 nodeIndex = 0; - bool tableFetched = false; - char *tableName = text_to_cstring(tableNameText); - - Datum *nodeNameArray = DeconstructArrayObject(nodeNameObject); - Datum *nodePortArray = DeconstructArrayObject(nodePortObject); - int32 nodeNameCount = ArrayObjectCount(nodeNameObject); - int32 nodePortCount = ArrayObjectCount(nodePortObject); - - /* we should have the same number of node names and port numbers */ - if (nodeNameCount != nodePortCount) - { - ereport(ERROR, (errmsg("node name array size: %d and node port array size: %d" - " do not match", nodeNameCount, nodePortCount))); - } - - /* - * We lock on the shardId, but do not unlock. When the function returns, and - * the transaction for this function commits, this lock will automatically - * be released. This ensures that concurrent caching commands will see the - * newly created table when they acquire the lock (in read committed mode). - */ - shardId = ExtractShardId(tableName); - LockShardResource(shardId, AccessExclusiveLock); - - relationNameList = textToQualifiedNameList(tableNameText); - relation = makeRangeVarFromNameList(relationNameList); - relationId = RangeVarGetRelid(relation, NoLock, true); - - /* check if we already fetched the table */ - if (relationId != InvalidOid) - { - uint64 localTableSize = 0; - - if (!ExpireCachedShards) - { - return; - } - - /* - * Check if the cached shard has the same size on disk as it has as on - * the placement (is up to date). - * - * Note 1: performing updates or deletes on the original shard leads to - * inconsistent sizes between different databases in which case the data - * would be fetched every time, or worse, the placement would get into - * a deadlock when it tries to fetch from itself while holding the lock. - * Therefore, this option is disabled by default. - * - * Note 2: when appending data to a shard, the size on disk only - * increases when a new page is added (the next 8kB block). - */ - localTableSize = LocalTableSize(relationId); - - if (remoteTableSize > localTableSize) - { - /* table is not up to date, drop the table */ - ObjectAddress tableObject = { InvalidOid, InvalidOid, 0 }; - - tableObject.classId = RelationRelationId; - tableObject.objectId = relationId; - tableObject.objectSubId = 0; - - performDeletion(&tableObject, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); - } - else - { - /* table is up to date */ - return; - } - } - - /* loop until we fetch the table or try all nodes */ - while (!tableFetched && (nodeIndex < nodeNameCount)) - { - Datum nodeNameDatum = nodeNameArray[nodeIndex]; - Datum nodePortDatum = nodePortArray[nodeIndex]; - char *nodeName = TextDatumGetCString(nodeNameDatum); - uint32 nodePort = DatumGetUInt32(nodePortDatum); - - tableFetched = (*FetchTableFunction)(nodeName, nodePort, tableName); - - nodeIndex++; - } - - /* error out if we tried all nodes and could not fetch the table */ - if (!tableFetched) - { - ereport(ERROR, (errmsg("could not fetch relation: \"%s\"", tableName))); - } -} - - -/* LocalTableSize returns the size on disk of the given table. */ -static uint64 -LocalTableSize(Oid relationId) -{ - uint64 tableSize = 0; - char relationType = 0; - Datum relationIdDatum = ObjectIdGetDatum(relationId); - - relationType = get_rel_relkind(relationId); - if (RegularTable(relationId)) - { - Datum tableSizeDatum = DirectFunctionCall1(pg_table_size, relationIdDatum); - - tableSize = DatumGetInt64(tableSizeDatum); - } - else if (relationType == RELKIND_FOREIGN_TABLE) - { - bool cstoreTable = CStoreTable(relationId); - if (cstoreTable) - { - /* extract schema name of cstore */ - Oid cstoreId = get_extension_oid(CSTORE_FDW_NAME, false); - Oid cstoreSchemaOid = get_extension_schema(cstoreId); - const char *cstoreSchemaName = get_namespace_name(cstoreSchemaOid); - - const int tableSizeArgumentCount = 1; - - Oid tableSizeFunctionOid = FunctionOid(cstoreSchemaName, - CSTORE_TABLE_SIZE_FUNCTION_NAME, - tableSizeArgumentCount); - Datum tableSizeDatum = OidFunctionCall1(tableSizeFunctionOid, - relationIdDatum); - - tableSize = DatumGetInt64(tableSizeDatum); - } - else - { - char *relationName = get_rel_name(relationId); - struct stat fileStat; - - int statOK = 0; - - StringInfo localFilePath = makeStringInfo(); - appendStringInfo(localFilePath, FOREIGN_CACHED_FILE_PATH, relationName); - - /* extract the file size using stat, analogous to pg_stat_file */ - statOK = stat(localFilePath->data, &fileStat); - if (statOK < 0) - { - ereport(ERROR, (errcode_for_file_access(), - errmsg("could not stat file \"%s\": %m", - localFilePath->data))); - } - - tableSize = (uint64) fileStat.st_size; - } - } - else - { - char *relationName = get_rel_name(relationId); - - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot get size for table \"%s\"", relationName), - errdetail("Only regular and foreign tables are supported."))); - } - - return tableSize; -} - - /* Extracts shard id from the given table name, and returns it. */ static uint64 ExtractShardId(const char *tableName) @@ -797,222 +556,6 @@ ExtractShardId(const char *tableName) } -/* - * FetchRegularTable fetches the given table's data using the copy out command. - * The function then fetches the DDL commands necessary to create this table's - * replica, and locally applies these DDL commands. Last, the function copies - * the fetched table data into the created table; and on success, returns true. - * On failure due to connectivity issues with remote node, the function returns - * false. On other types of failures, the function errors out. - */ -static bool -FetchRegularTable(const char *nodeName, uint32 nodePort, const char *tableName) -{ - StringInfo localFilePath = NULL; - StringInfo remoteCopyCommand = NULL; - List *ddlCommandList = NIL; - ListCell *ddlCommandCell = NULL; - CopyStmt *localCopyCommand = NULL; - RangeVar *localTable = NULL; - uint64 shardId = 0; - bool received = false; - StringInfo queryString = NULL; - const char *tableOwner = NULL; - Oid tableOwnerId = InvalidOid; - Oid savedUserId = InvalidOid; - int savedSecurityContext = 0; - List *tableNameList = NIL; - - /* copy remote table's data to this node in an idempotent manner */ - shardId = ExtractShardId(tableName); - localFilePath = makeStringInfo(); - appendStringInfo(localFilePath, "base/%s/%s" UINT64_FORMAT, - PG_JOB_CACHE_DIR, TABLE_FILE_PREFIX, shardId); - - remoteCopyCommand = makeStringInfo(); - appendStringInfo(remoteCopyCommand, COPY_OUT_COMMAND, tableName); - - received = ReceiveRegularFile(nodeName, nodePort, NULL, remoteCopyCommand, - localFilePath); - if (!received) - { - return false; - } - - /* fetch the ddl commands needed to create the table */ - tableOwner = RemoteTableOwner(nodeName, nodePort, tableName); - if (tableOwner == NULL) - { - return false; - } - tableOwnerId = get_role_oid(tableOwner, false); - - /* fetch the ddl commands needed to create the table */ - ddlCommandList = TableDDLCommandList(nodeName, nodePort, tableName); - if (ddlCommandList == NIL) - { - return false; - } - - /* - * Apply DDL commands against the database. Note that on failure from here - * on, we immediately error out instead of returning false. Have to do - * this as the table's owner to ensure the local table is created with - * compatible permissions. - */ - GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); - SetUserIdAndSecContext(tableOwnerId, SECURITY_LOCAL_USERID_CHANGE); - - foreach(ddlCommandCell, ddlCommandList) - { - StringInfo ddlCommand = (StringInfo) lfirst(ddlCommandCell); - Node *ddlCommandNode = ParseTreeNode(ddlCommand->data); - - CitusProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); - CommandCounterIncrement(); - } - - /* - * Copy local file into the relation. We call ProcessUtility() instead of - * directly calling DoCopy() because some extensions (e.g. cstore_fdw) hook - * into process utility to provide their custom COPY behavior. - */ - tableNameList = stringToQualifiedNameList(tableName); - localTable = makeRangeVarFromNameList(tableNameList); - localCopyCommand = CopyStatement(localTable, localFilePath->data); - - queryString = makeStringInfo(); - appendStringInfo(queryString, COPY_IN_COMMAND, tableName, localFilePath->data); - - CitusProcessUtility((Node *) localCopyCommand, queryString->data, - PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); - - /* finally delete the temporary file we created */ - CitusDeleteFile(localFilePath->data); - - SetUserIdAndSecContext(savedUserId, savedSecurityContext); - - return true; -} - - -/* - * FetchForeignTable fetches the foreign file for the given table name from the - * remote node. The function then fetches the DDL commands needed to create the - * table, and applies these DDL commands locally to create the foreign table. - * On success, the function returns true. On failure due to connectivity issues - * with remote node, the function returns false. On failure due to applying DDL - * commands against the local database, the function errors out. - */ -static bool -FetchForeignTable(const char *nodeName, uint32 nodePort, const char *tableName) -{ - const char *nodeUser = NULL; - StringInfo localFilePath = NULL; - StringInfo remoteFilePath = NULL; - StringInfo transmitCommand = NULL; - StringInfo alterTableCommand = NULL; - bool received = false; - List *ddlCommandList = NIL; - ListCell *ddlCommandCell = NULL; - - /* - * Fetch a foreign file to this node in an idempotent manner. It's OK that - * this file name lacks the schema, as the table name will have a shard id - * attached to it, which is unique (so conflicts are avoided even if two - * tables in different schemas have the same name). - */ - localFilePath = makeStringInfo(); - appendStringInfo(localFilePath, FOREIGN_CACHED_FILE_PATH, tableName); - - remoteFilePath = ForeignFilePath(nodeName, nodePort, tableName); - if (remoteFilePath == NULL) - { - return false; - } - - transmitCommand = makeStringInfo(); - appendStringInfo(transmitCommand, TRANSMIT_REGULAR_COMMAND, remoteFilePath->data); - - /* - * We allow some arbitrary input in the file name and connect to the remote - * node as superuser to transmit. Therefore, we only allow calling this - * function when already running as superuser. - */ - EnsureSuperUser(); - nodeUser = CitusExtensionOwnerName(); - - received = ReceiveRegularFile(nodeName, nodePort, nodeUser, transmitCommand, - localFilePath); - if (!received) - { - return false; - } - - /* fetch the ddl commands needed to create the table */ - ddlCommandList = TableDDLCommandList(nodeName, nodePort, tableName); - if (ddlCommandList == NIL) - { - return false; - } - - alterTableCommand = makeStringInfo(); - appendStringInfo(alterTableCommand, SET_FOREIGN_TABLE_FILENAME, tableName, - localFilePath->data); - - ddlCommandList = lappend(ddlCommandList, alterTableCommand); - - /* - * Apply DDL commands against the database. Note that on failure here, we - * immediately error out instead of returning false. - */ - foreach(ddlCommandCell, ddlCommandList) - { - StringInfo ddlCommand = (StringInfo) lfirst(ddlCommandCell); - Node *ddlCommandNode = ParseTreeNode(ddlCommand->data); - - CitusProcessUtility(ddlCommandNode, ddlCommand->data, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); - CommandCounterIncrement(); - } - - return true; -} - - -/* - * RemoteTableOwner takes in the given table name, and fetches the owner of - * the table. If an error occurs during fetching, return NULL. - */ -static const char * -RemoteTableOwner(const char *nodeName, uint32 nodePort, const char *tableName) -{ - List *ownerList = NIL; - StringInfo queryString = NULL; - StringInfo relationOwner; - MultiConnection *connection = NULL; - uint32 connectionFlag = FORCE_NEW_CONNECTION; - PGresult *result = NULL; - - queryString = makeStringInfo(); - appendStringInfo(queryString, GET_TABLE_OWNER, tableName); - connection = GetNodeConnection(connectionFlag, nodeName, nodePort); - - ExecuteOptionalRemoteCommand(connection, queryString->data, &result); - - ownerList = ReadFirstColumnAsText(result); - if (list_length(ownerList) != 1) - { - return NULL; - } - - relationOwner = (StringInfo) linitial(ownerList); - - return relationOwner->data; -} - - /* * TableDDLCommandList takes in the given table name, and fetches the list of * DDL commands used in creating the table. If an error occurs during fetching, @@ -1041,37 +584,6 @@ TableDDLCommandList(const char *nodeName, uint32 nodePort, const char *tableName } -/* - * ForeignFilePath takes in the foreign table name, and fetches this table's - * remote file path. If an error occurs during fetching, the function returns - * null. - */ -static StringInfo -ForeignFilePath(const char *nodeName, uint32 nodePort, const char *tableName) -{ - List *foreignPathList = NIL; - StringInfo foreignPathCommand = NULL; - StringInfo foreignPath = NULL; - MultiConnection *connection = NULL; - PGresult *result = NULL; - int connectionFlag = FORCE_NEW_CONNECTION; - - foreignPathCommand = makeStringInfo(); - appendStringInfo(foreignPathCommand, FOREIGN_FILE_PATH_COMMAND, tableName); - connection = GetNodeConnection(connectionFlag, nodeName, nodePort); - - ExecuteOptionalRemoteCommand(connection, foreignPathCommand->data, &result); - - foreignPathList = ReadFirstColumnAsText(result); - if (foreignPathList != NIL) - { - foreignPath = (StringInfo) linitial(foreignPathList); - } - - return foreignPath; -} - - /* * ExecuteRemoteQuery executes the given query, copies the query's results to a * sorted list, and returns this list. The function assumes that query results @@ -1427,3 +939,39 @@ SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg) statement->options = lappend(statement->options, defElem); } + + +/* + * worker_fetch_regular_table UDF is a stub UDF to install Citus flawlessly. + * Otherwise we need to delete them from our sql files, which is confusing + */ +Datum +worker_fetch_regular_table(PG_FUNCTION_ARGS) +{ + ereport(DEBUG2, (errmsg("this function is deprecated and no longer is used"))); + PG_RETURN_VOID(); +} + + +/* + * worker_fetch_foreign_file UDF is a stub UDF to install Citus flawlessly. + * Otherwise we need to delete them from our sql files, which is confusing + */ +Datum +worker_fetch_foreign_file(PG_FUNCTION_ARGS) +{ + ereport(DEBUG2, (errmsg("this function is deprecated and no longer is used"))); + PG_RETURN_VOID(); +} + + +/* + * master_expire_table_cache UDF is a stub UDF to install Citus flawlessly. + * Otherwise we need to delete them from our sql files, which is confusing + */ +Datum +master_expire_table_cache(PG_FUNCTION_ARGS) +{ + ereport(DEBUG2, (errmsg("this function is deprecated and no longer is used"))); + PG_RETURN_VOID(); +} diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index cacd80d05..8b27823d2 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -33,10 +33,6 @@ #define NON_PRUNABLE_JOIN -1 #define RESERVED_HASHED_COLUMN_ID MaxAttrNumber #define MERGE_COLUMN_FORMAT "merge_column_%u" -#define TABLE_FETCH_COMMAND "SELECT worker_fetch_regular_table \ - ('%s', " UINT64_FORMAT ", '%s', '%s')" -#define FOREIGN_FETCH_COMMAND "SELECT worker_fetch_foreign_file \ - ('%s', " UINT64_FORMAT ", '%s', '%s')" #define MAP_OUTPUT_FETCH_COMMAND "SELECT worker_fetch_partition_file \ (" UINT64_FORMAT ", %u, %u, %u, '%s', %u)" #define RANGE_PARTITION_COMMAND "SELECT worker_range_partition_table \ @@ -84,12 +80,11 @@ typedef enum SQL_TASK = 1, MAP_TASK = 2, MERGE_TASK = 3, - SHARD_FETCH_TASK = 4, - MAP_OUTPUT_FETCH_TASK = 5, - MERGE_FETCH_TASK = 6, - MODIFY_TASK = 7, - ROUTER_TASK = 8, - DDL_TASK = 9 + MAP_OUTPUT_FETCH_TASK = 4, + MERGE_FETCH_TASK = 5, + MODIFY_TASK = 6, + ROUTER_TASK = 7, + DDL_TASK = 8 } TaskType; @@ -151,7 +146,7 @@ typedef struct MapMergeJob /* * Task represents an executable unit of work. We conceptualize our tasks into * compute and data fetch task types. SQL, map, and merge tasks are considered - * as compute tasks; and shard fetch, map fetch, and merge fetch tasks are data + * as compute tasks; and map fetch, and merge fetch tasks are data * fetch tasks. We also forward declare the task execution struct here to avoid * including the executor header files. * @@ -180,7 +175,6 @@ typedef struct Task uint32 upstreamTaskId; /* only applies to data fetch tasks */ ShardInterval *shardInterval; /* only applies to merge tasks */ bool assignmentConstrained; /* only applies to merge tasks */ - uint64 shardId; /* only applies to shard fetch tasks */ TaskExecution *taskExecution; /* used by task tracker executor */ bool upsertQuery; /* only applies to modify tasks */ char replicationModel; /* only applies to modify tasks */ @@ -303,7 +297,6 @@ extern bool EnableUniqueJobIds; extern DistributedPlan * CreatePhysicalDistributedPlan(MultiTreeRoot *multiTree, PlannerRestrictionContext * plannerRestrictionContext); -extern StringInfo ShardFetchQueryString(uint64 shardId); extern Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType, char *queryString); @@ -331,7 +324,6 @@ extern List * TaskListAppendUnique(List *list, Task *task); extern List * TaskListConcatUnique(List *list1, List *list2); extern bool TaskListMember(const List *taskList, const Task *task); extern List * TaskListDifference(const List *list1, const List *list2); -extern List * TaskListUnion(const List *list1, const List *list2); extern List * AssignAnchorShardTaskList(List *taskList); extern List * FirstReplicaAssignTaskList(List *taskList); diff --git a/src/include/distributed/multi_server_executor.h b/src/include/distributed/multi_server_executor.h index 66e5dbec6..496149f4f 100644 --- a/src/include/distributed/multi_server_executor.h +++ b/src/include/distributed/multi_server_executor.h @@ -44,13 +44,10 @@ typedef enum EXEC_TASK_CONNECT_START = 1, EXEC_TASK_CONNECT_POLL = 2, EXEC_TASK_FAILED = 3, - EXEC_FETCH_TASK_LOOP = 4, - EXEC_FETCH_TASK_START = 5, - EXEC_FETCH_TASK_RUNNING = 6, - EXEC_COMPUTE_TASK_START = 7, - EXEC_COMPUTE_TASK_RUNNING = 8, - EXEC_COMPUTE_TASK_COPYING = 9, - EXEC_TASK_DONE = 10, + EXEC_COMPUTE_TASK_START = 4, + EXEC_COMPUTE_TASK_RUNNING = 5, + EXEC_COMPUTE_TASK_COPYING = 6, + EXEC_TASK_DONE = 7, /* used for task tracker executor */ EXEC_TASK_UNASSIGNED = 11, @@ -143,7 +140,6 @@ struct TaskExecution uint32 nodeCount; uint32 currentNodeIndex; uint32 querySourceNodeIndex; /* only applies to map fetch tasks */ - int32 dataFetchTaskIndex; uint32 failureCount; bool criticalErrorOccurred; }; diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index 5d91486db..d78c47c30 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -47,14 +47,8 @@ #define COPY_OUT_COMMAND "COPY %s TO STDOUT" #define COPY_IN_COMMAND "COPY %s FROM '%s'" -/* Defines that relate to fetching foreign tables */ -#define FOREIGN_CACHED_FILE_PATH "pg_foreign_file/cached/%s" -#define GET_TABLE_OWNER \ - "SELECT rolname FROM pg_class JOIN pg_roles ON (pg_roles.oid = pg_class.relowner) " \ - "WHERE pg_class.oid = '%s'::regclass" +/* Defines that relate to creating tables */ #define GET_TABLE_DDL_EVENTS "SELECT master_get_table_ddl_events('%s')" -#define SET_FOREIGN_TABLE_FILENAME "ALTER FOREIGN TABLE %s OPTIONS (SET filename '%s')" -#define FOREIGN_FILE_PATH_COMMAND "SELECT worker_foreign_file_path('%s')" #define SET_SEARCH_PATH_COMMAND "SET search_path TO %s" #define CREATE_TABLE_COMMAND "CREATE TABLE %s (%s)" #define CREATE_TABLE_AS_COMMAND "CREATE TABLE %s (%s) AS (%s)" @@ -99,7 +93,6 @@ typedef struct FileOutputStream /* Config variables managed via guc.c */ extern int PartitionBufferSize; -extern bool ExpireCachedShards; extern bool BinaryWorkerCopyFormat; diff --git a/src/test/regress/expected/multi_expire_table_cache.out b/src/test/regress/expected/multi_expire_table_cache.out deleted file mode 100644 index fe7b4030d..000000000 --- a/src/test/regress/expected/multi_expire_table_cache.out +++ /dev/null @@ -1,108 +0,0 @@ ---- ---- MULTI_EXPIRE_TABLE_CACHE ---- -SET citus.next_shard_id TO 1220000; --- create test table -CREATE TABLE large_table(a int, b int); -SELECT master_create_distributed_table('large_table', 'a', 'hash'); - master_create_distributed_table ---------------------------------- - -(1 row) - -SELECT master_create_worker_shards('large_table', 8, 1); - master_create_worker_shards ------------------------------ - -(1 row) - -CREATE TABLE broadcast_table(a int, b int); -SELECT master_create_distributed_table('broadcast_table', 'a', 'hash'); - master_create_distributed_table ---------------------------------- - -(1 row) - -SELECT master_create_worker_shards('broadcast_table', 2, 1); - master_create_worker_shards ------------------------------ - -(1 row) - --- verify only small tables are supported -SELECT master_expire_table_cache('large_table'); -ERROR: Must be called on tables smaller than 4 shards -SELECT master_expire_table_cache('broadcast_table'); - master_expire_table_cache ---------------------------- - -(1 row) - --- run a join so that broadcast tables are cached on other workers -SELECT * from large_table l, broadcast_table b where l.a = b.b; - a | b | a | b ----+---+---+--- -(0 rows) - --- insert some data -INSERT INTO large_table VALUES(1, 1); -INSERT INTO large_table VALUES(1, 2); -INSERT INTO large_table VALUES(2, 1); -INSERT INTO large_table VALUES(2, 2); -INSERT INTO large_table VALUES(3, 1); -INSERT INTO large_table VALUES(3, 2); -INSERT INTO broadcast_table VALUES(1, 1); --- verify returned results are wrong -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - a | b | a | b ----+---+---+--- - 1 | 1 | 1 | 1 - 2 | 1 | 1 | 1 -(2 rows) - --- expire cache and re-run, results should be correct this time -SELECT master_expire_table_cache('broadcast_table'); - master_expire_table_cache ---------------------------- - -(1 row) - -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - a | b | a | b ----+---+---+--- - 1 | 1 | 1 | 1 - 2 | 1 | 1 | 1 - 3 | 1 | 1 | 1 -(3 rows) - --- insert some more data into broadcast table -INSERT INTO broadcast_table VALUES(2, 2); --- run the same query, get wrong results -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - a | b | a | b ----+---+---+--- - 1 | 1 | 1 | 1 - 2 | 1 | 1 | 1 - 3 | 1 | 1 | 1 - 3 | 2 | 2 | 2 -(4 rows) - --- expire cache and re-run, results should be correct this time -SELECT master_expire_table_cache('broadcast_table'); - master_expire_table_cache ---------------------------- - -(1 row) - -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - a | b | a | b ----+---+---+--- - 1 | 1 | 1 | 1 - 1 | 2 | 2 | 2 - 2 | 1 | 1 | 1 - 2 | 2 | 2 | 2 - 3 | 1 | 1 | 1 - 3 | 2 | 2 | 2 -(6 rows) - -DROP TABLE large_table, broadcast_table; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 19f9c9e63..f3bc4edda 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -133,6 +133,7 @@ ALTER EXTENSION citus UPDATE TO '7.2-1'; ALTER EXTENSION citus UPDATE TO '7.2-2'; ALTER EXTENSION citus UPDATE TO '7.2-3'; ALTER EXTENSION citus UPDATE TO '7.3-3'; +ALTER EXTENSION citus UPDATE TO '7.4-1'; -- show running version SHOW citus.version; citus.version diff --git a/src/test/regress/expected/multi_large_table_join_planning.out b/src/test/regress/expected/multi_large_table_join_planning.out index 922435f17..75451fae1 100644 --- a/src/test/regress/expected/multi_large_table_join_planning.out +++ b/src/test/regress/expected/multi_large_table_join_planning.out @@ -49,63 +49,63 @@ DEBUG: join prunable for intervals [8997,10560] and [1,5986] DEBUG: join prunable for intervals [10560,12036] and [1,5986] DEBUG: join prunable for intervals [12036,13473] and [1,5986] DEBUG: join prunable for intervals [13473,14947] and [1,5986] -DEBUG: generated sql query for task 3 +DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290000 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 6 +DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290001 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 9 +DEBUG: generated sql query for task 3 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290002 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 12 +DEBUG: generated sql query for task 4 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290003 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 15 +DEBUG: generated sql query for task 5 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290004 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 18 +DEBUG: generated sql query for task 6 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290005 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 21 +DEBUG: generated sql query for task 7 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290006 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 24 +DEBUG: generated sql query for task 8 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290007 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 3 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 24 to node localhost:57637 -DEBUG: assigned task 21 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] -DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0001.task_000025".intermediate_column_1_0, "pg_merge_job_0001.task_000025".intermediate_column_1_1, "pg_merge_job_0001.task_000025".intermediate_column_1_2, "pg_merge_job_0001.task_000025".intermediate_column_1_3, "pg_merge_job_0001.task_000025".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000025 "pg_merge_job_0001.task_000025" JOIN part_290011 part ON (("pg_merge_job_0001.task_000025".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" -DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0001.task_000034".intermediate_column_1_0, "pg_merge_job_0001.task_000034".intermediate_column_1_1, "pg_merge_job_0001.task_000034".intermediate_column_1_2, "pg_merge_job_0001.task_000034".intermediate_column_1_3, "pg_merge_job_0001.task_000034".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000034 "pg_merge_job_0001.task_000034" JOIN part_280002 part ON (("pg_merge_job_0001.task_000034".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000009".intermediate_column_1_0, "pg_merge_job_0001.task_000009".intermediate_column_1_1, "pg_merge_job_0001.task_000009".intermediate_column_1_2, "pg_merge_job_0001.task_000009".intermediate_column_1_3, "pg_merge_job_0001.task_000009".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000009 "pg_merge_job_0001.task_000009" JOIN part_290011 part ON (("pg_merge_job_0001.task_000009".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000018".intermediate_column_1_0, "pg_merge_job_0001.task_000018".intermediate_column_1_1, "pg_merge_job_0001.task_000018".intermediate_column_1_2, "pg_merge_job_0001.task_000018".intermediate_column_1_3, "pg_merge_job_0001.task_000018".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000018 "pg_merge_job_0001.task_000018" JOIN part_280002 part ON (("pg_merge_job_0001.task_000018".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 25 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 34 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 +DETAIL: Creating dependency on merge taskId 9 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 18 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] -DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000007".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000007".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000007 "pg_merge_job_0002.task_000007" JOIN customer_290010 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000007".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000007".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000007".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000007".intermediate_column_2_0, "pg_merge_job_0002.task_000007".intermediate_column_2_1" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000005".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000005".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000005 "pg_merge_job_0002.task_000005" JOIN customer_290010 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000005".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000005".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000005".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000005".intermediate_column_2_0, "pg_merge_job_0002.task_000005".intermediate_column_2_1" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000008".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000008".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000008 "pg_merge_job_0002.task_000008" JOIN customer_280001 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000008".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000008".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000008".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000008".intermediate_column_2_0, "pg_merge_job_0002.task_000008".intermediate_column_2_1" DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000010".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000010".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000010 "pg_merge_job_0002.task_000010" JOIN customer_280001 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000010".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000010".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000010".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000010".intermediate_column_2_0, "pg_merge_job_0002.task_000010".intermediate_column_2_1" -DEBUG: generated sql query for task 9 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000013".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000013".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000013 "pg_merge_job_0002.task_000013" JOIN customer_280000 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000013".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000013".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000013".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000013".intermediate_column_2_0, "pg_merge_job_0002.task_000013".intermediate_column_2_1" +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000011".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000011".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000011 "pg_merge_job_0002.task_000011" JOIN customer_280000 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000011".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000011".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000011".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000011".intermediate_column_2_0, "pg_merge_job_0002.task_000011".intermediate_column_2_1" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 7 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 10 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 8 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 11 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: completed cleanup query for job 3 DEBUG: completed cleanup query for job 3 DEBUG: completed cleanup query for job 2 @@ -156,36 +156,36 @@ GROUP BY l_partkey, o_orderkey ORDER BY l_partkey, o_orderkey; -DEBUG: generated sql query for task 2 +DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 4 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290002 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 8 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290003 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 10 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290004 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 12 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290005 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 14 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290006 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 16 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290007 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 16 to node localhost:57637 -DEBUG: assigned task 14 to node localhost:57638 DEBUG: generated sql query for task 2 -DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290008 orders WHERE (o_totalprice <> 4.0)" +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290002 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for task 4 -DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290009 orders WHERE (o_totalprice <> 4.0)" +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290003 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 5 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290004 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290005 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 7 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290006 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 8 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290007 lineitem WHERE (l_quantity < 5.0)" +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 +DEBUG: generated sql query for task 1 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290008 orders WHERE (o_totalprice <> 4.0)" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290009 orders WHERE (o_totalprice <> 4.0)" +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 @@ -199,29 +199,29 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000017".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000005".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000017 "pg_merge_job_0004.task_000017" JOIN pg_merge_job_0005.task_000005 "pg_merge_job_0005.task_000005" ON (("pg_merge_job_0004.task_000017".intermediate_column_4_1 = "pg_merge_job_0005.task_000005".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000017".intermediate_column_4_0, "pg_merge_job_0005.task_000005".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000009".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000003".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000009 "pg_merge_job_0004.task_000009" JOIN pg_merge_job_0005.task_000003 "pg_merge_job_0005.task_000003" ON (("pg_merge_job_0004.task_000009".intermediate_column_4_1 = "pg_merge_job_0005.task_000003".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000009".intermediate_column_4_0, "pg_merge_job_0005.task_000003".intermediate_column_5_0" DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000026".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000008".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000026 "pg_merge_job_0004.task_000026" JOIN pg_merge_job_0005.task_000008 "pg_merge_job_0005.task_000008" ON (("pg_merge_job_0004.task_000026".intermediate_column_4_1 = "pg_merge_job_0005.task_000008".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000026".intermediate_column_4_0, "pg_merge_job_0005.task_000008".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000018".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000006".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000018 "pg_merge_job_0004.task_000018" JOIN pg_merge_job_0005.task_000006 "pg_merge_job_0005.task_000006" ON (("pg_merge_job_0004.task_000018".intermediate_column_4_1 = "pg_merge_job_0005.task_000006".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000018".intermediate_column_4_0, "pg_merge_job_0005.task_000006".intermediate_column_5_0" DEBUG: generated sql query for task 9 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000035".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000011".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000035 "pg_merge_job_0004.task_000035" JOIN pg_merge_job_0005.task_000011 "pg_merge_job_0005.task_000011" ON (("pg_merge_job_0004.task_000035".intermediate_column_4_1 = "pg_merge_job_0005.task_000011".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000035".intermediate_column_4_0, "pg_merge_job_0005.task_000011".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000027".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000009".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000027 "pg_merge_job_0004.task_000027" JOIN pg_merge_job_0005.task_000009 "pg_merge_job_0005.task_000009" ON (("pg_merge_job_0004.task_000027".intermediate_column_4_1 = "pg_merge_job_0005.task_000009".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000027".intermediate_column_4_0, "pg_merge_job_0005.task_000009".intermediate_column_5_0" DEBUG: generated sql query for task 12 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000044".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000014".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000044 "pg_merge_job_0004.task_000044" JOIN pg_merge_job_0005.task_000014 "pg_merge_job_0005.task_000014" ON (("pg_merge_job_0004.task_000044".intermediate_column_4_1 = "pg_merge_job_0005.task_000014".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000044".intermediate_column_4_0, "pg_merge_job_0005.task_000014".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000036".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000012".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000036 "pg_merge_job_0004.task_000036" JOIN pg_merge_job_0005.task_000012 "pg_merge_job_0005.task_000012" ON (("pg_merge_job_0004.task_000036".intermediate_column_4_1 = "pg_merge_job_0005.task_000012".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000036".intermediate_column_4_0, "pg_merge_job_0005.task_000012".intermediate_column_5_0" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 3 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 8 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 12 DEBUG: assigned task 3 to node localhost:57637 DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 9 to node localhost:57637 diff --git a/src/test/regress/expected/multi_large_table_join_planning_0.out b/src/test/regress/expected/multi_large_table_join_planning_0.out index 5bae0e5cd..c62634f49 100644 --- a/src/test/regress/expected/multi_large_table_join_planning_0.out +++ b/src/test/regress/expected/multi_large_table_join_planning_0.out @@ -57,63 +57,63 @@ DEBUG: join prunable for intervals [8997,10560] and [1,5986] DEBUG: join prunable for intervals [10560,12036] and [1,5986] DEBUG: join prunable for intervals [12036,13473] and [1,5986] DEBUG: join prunable for intervals [13473,14947] and [1,5986] -DEBUG: generated sql query for task 3 +DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290000 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 6 +DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290001 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 9 +DEBUG: generated sql query for task 3 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290002 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 12 +DEBUG: generated sql query for task 4 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290003 lineitem JOIN orders_290008 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 15 +DEBUG: generated sql query for task 5 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290004 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 18 +DEBUG: generated sql query for task 6 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290005 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 21 +DEBUG: generated sql query for task 7 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290006 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: generated sql query for task 24 +DEBUG: generated sql query for task 8 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290007 lineitem JOIN orders_290009 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE ((lineitem.l_partkey < 1000) AND (orders.o_totalprice > 10.0))" -DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 3 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 24 to node localhost:57637 -DEBUG: assigned task 21 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] -DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0001.task_000025".intermediate_column_1_0, "pg_merge_job_0001.task_000025".intermediate_column_1_1, "pg_merge_job_0001.task_000025".intermediate_column_1_2, "pg_merge_job_0001.task_000025".intermediate_column_1_3, "pg_merge_job_0001.task_000025".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000025 "pg_merge_job_0001.task_000025" JOIN part_290011 part ON (("pg_merge_job_0001.task_000025".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" -DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0001.task_000034".intermediate_column_1_0, "pg_merge_job_0001.task_000034".intermediate_column_1_1, "pg_merge_job_0001.task_000034".intermediate_column_1_2, "pg_merge_job_0001.task_000034".intermediate_column_1_3, "pg_merge_job_0001.task_000034".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000034 "pg_merge_job_0001.task_000034" JOIN part_280002 part ON (("pg_merge_job_0001.task_000034".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000009".intermediate_column_1_0, "pg_merge_job_0001.task_000009".intermediate_column_1_1, "pg_merge_job_0001.task_000009".intermediate_column_1_2, "pg_merge_job_0001.task_000009".intermediate_column_1_3, "pg_merge_job_0001.task_000009".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000009 "pg_merge_job_0001.task_000009" JOIN part_290011 part ON (("pg_merge_job_0001.task_000009".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT "pg_merge_job_0001.task_000018".intermediate_column_1_0, "pg_merge_job_0001.task_000018".intermediate_column_1_1, "pg_merge_job_0001.task_000018".intermediate_column_1_2, "pg_merge_job_0001.task_000018".intermediate_column_1_3, "pg_merge_job_0001.task_000018".intermediate_column_1_4 FROM (pg_merge_job_0001.task_000018 "pg_merge_job_0001.task_000018" JOIN part_280002 part ON (("pg_merge_job_0001.task_000018".intermediate_column_1_0 = part.p_partkey))) WHERE (part.p_size > 8)" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 25 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 34 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 +DETAIL: Creating dependency on merge taskId 9 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 18 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] -DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000007".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000007".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000007 "pg_merge_job_0002.task_000007" JOIN customer_290010 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000007".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000007".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000007".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000007".intermediate_column_2_0, "pg_merge_job_0002.task_000007".intermediate_column_2_1" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000005".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000005".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000005 "pg_merge_job_0002.task_000005" JOIN customer_290010 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000005".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000005".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000005".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000005".intermediate_column_2_0, "pg_merge_job_0002.task_000005".intermediate_column_2_1" +DEBUG: generated sql query for task 4 +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000008".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000008".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000008 "pg_merge_job_0002.task_000008" JOIN customer_280001 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000008".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000008".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000008".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000008".intermediate_column_2_0, "pg_merge_job_0002.task_000008".intermediate_column_2_1" DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000010".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000010".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000010 "pg_merge_job_0002.task_000010" JOIN customer_280001 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000010".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000010".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000010".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000010".intermediate_column_2_0, "pg_merge_job_0002.task_000010".intermediate_column_2_1" -DEBUG: generated sql query for task 9 -DETAIL: query string: "SELECT "pg_merge_job_0002.task_000013".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000013".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000013 "pg_merge_job_0002.task_000013" JOIN customer_280000 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000013".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000013".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000013".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000013".intermediate_column_2_0, "pg_merge_job_0002.task_000013".intermediate_column_2_1" +DETAIL: query string: "SELECT "pg_merge_job_0002.task_000011".intermediate_column_2_0 AS l_partkey, "pg_merge_job_0002.task_000011".intermediate_column_2_1 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0002.task_000011 "pg_merge_job_0002.task_000011" JOIN customer_280000 customer ON ((customer.c_custkey = "pg_merge_job_0002.task_000011".intermediate_column_2_4))) WHERE ((("pg_merge_job_0002.task_000011".intermediate_column_2_2 > 5.0) OR ("pg_merge_job_0002.task_000011".intermediate_column_2_3 > 1200.0)) AND (customer.c_acctbal < 5000.0)) GROUP BY "pg_merge_job_0002.task_000011".intermediate_column_2_0, "pg_merge_job_0002.task_000011".intermediate_column_2_1" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 7 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 10 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 8 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 11 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: completed cleanup query for job 3 DEBUG: completed cleanup query for job 3 DEBUG: completed cleanup query for job 2 @@ -166,36 +166,36 @@ GROUP BY ORDER BY l_partkey, o_orderkey; DEBUG: StartTransactionCommand -DEBUG: generated sql query for task 2 +DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 4 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290002 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 8 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290003 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 10 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290004 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 12 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290005 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 14 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290006 lineitem WHERE (l_quantity < 5.0)" -DEBUG: generated sql query for task 16 -DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290007 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 16 to node localhost:57637 -DEBUG: assigned task 14 to node localhost:57638 DEBUG: generated sql query for task 2 -DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290008 orders WHERE (o_totalprice <> 4.0)" +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 3 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290002 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for task 4 -DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290009 orders WHERE (o_totalprice <> 4.0)" +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290003 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 5 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290004 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 6 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290005 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 7 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290006 lineitem WHERE (l_quantity < 5.0)" +DEBUG: generated sql query for task 8 +DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290007 lineitem WHERE (l_quantity < 5.0)" +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 +DEBUG: generated sql query for task 1 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290008 orders WHERE (o_totalprice <> 4.0)" +DEBUG: generated sql query for task 2 +DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290009 orders WHERE (o_totalprice <> 4.0)" +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 @@ -209,29 +209,29 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: generated sql query for task 3 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000017".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000005".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000017 "pg_merge_job_0004.task_000017" JOIN pg_merge_job_0005.task_000005 "pg_merge_job_0005.task_000005" ON (("pg_merge_job_0004.task_000017".intermediate_column_4_1 = "pg_merge_job_0005.task_000005".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000017".intermediate_column_4_0, "pg_merge_job_0005.task_000005".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000009".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000003".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000009 "pg_merge_job_0004.task_000009" JOIN pg_merge_job_0005.task_000003 "pg_merge_job_0005.task_000003" ON (("pg_merge_job_0004.task_000009".intermediate_column_4_1 = "pg_merge_job_0005.task_000003".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000009".intermediate_column_4_0, "pg_merge_job_0005.task_000003".intermediate_column_5_0" DEBUG: generated sql query for task 6 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000026".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000008".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000026 "pg_merge_job_0004.task_000026" JOIN pg_merge_job_0005.task_000008 "pg_merge_job_0005.task_000008" ON (("pg_merge_job_0004.task_000026".intermediate_column_4_1 = "pg_merge_job_0005.task_000008".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000026".intermediate_column_4_0, "pg_merge_job_0005.task_000008".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000018".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000006".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000018 "pg_merge_job_0004.task_000018" JOIN pg_merge_job_0005.task_000006 "pg_merge_job_0005.task_000006" ON (("pg_merge_job_0004.task_000018".intermediate_column_4_1 = "pg_merge_job_0005.task_000006".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000018".intermediate_column_4_0, "pg_merge_job_0005.task_000006".intermediate_column_5_0" DEBUG: generated sql query for task 9 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000035".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000011".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000035 "pg_merge_job_0004.task_000035" JOIN pg_merge_job_0005.task_000011 "pg_merge_job_0005.task_000011" ON (("pg_merge_job_0004.task_000035".intermediate_column_4_1 = "pg_merge_job_0005.task_000011".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000035".intermediate_column_4_0, "pg_merge_job_0005.task_000011".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000027".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000009".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000027 "pg_merge_job_0004.task_000027" JOIN pg_merge_job_0005.task_000009 "pg_merge_job_0005.task_000009" ON (("pg_merge_job_0004.task_000027".intermediate_column_4_1 = "pg_merge_job_0005.task_000009".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000027".intermediate_column_4_0, "pg_merge_job_0005.task_000009".intermediate_column_5_0" DEBUG: generated sql query for task 12 -DETAIL: query string: "SELECT "pg_merge_job_0004.task_000044".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000014".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000044 "pg_merge_job_0004.task_000044" JOIN pg_merge_job_0005.task_000014 "pg_merge_job_0005.task_000014" ON (("pg_merge_job_0004.task_000044".intermediate_column_4_1 = "pg_merge_job_0005.task_000014".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000044".intermediate_column_4_0, "pg_merge_job_0005.task_000014".intermediate_column_5_0" +DETAIL: query string: "SELECT "pg_merge_job_0004.task_000036".intermediate_column_4_0 AS l_partkey, "pg_merge_job_0005.task_000012".intermediate_column_5_0 AS o_orderkey, count(*) AS count FROM (pg_merge_job_0004.task_000036 "pg_merge_job_0004.task_000036" JOIN pg_merge_job_0005.task_000012 "pg_merge_job_0005.task_000012" ON (("pg_merge_job_0004.task_000036".intermediate_column_4_1 = "pg_merge_job_0005.task_000012".intermediate_column_5_1))) WHERE true GROUP BY "pg_merge_job_0004.task_000036".intermediate_column_4_0, "pg_merge_job_0005.task_000012".intermediate_column_5_0" DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 3 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 8 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 12 DEBUG: assigned task 3 to node localhost:57637 DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 9 to node localhost:57637 diff --git a/src/test/regress/expected/multi_large_table_pruning.out b/src/test/regress/expected/multi_large_table_pruning.out index c44cc6408..de6fb1bbd 100644 --- a/src/test/regress/expected/multi_large_table_pruning.out +++ b/src/test/regress/expected/multi_large_table_pruning.out @@ -23,11 +23,11 @@ DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 5 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 8 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 3 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 9 QUERY PLAN ------------------------------------------------------------------- Aggregate @@ -52,11 +52,11 @@ DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 5 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 8 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 3 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 9 count ------- 2984 @@ -151,21 +151,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 15 +DETAIL: Creating dependency on merge taskId 12 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 16 QUERY PLAN ------------------------------------------------------------------- Aggregate @@ -199,21 +199,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 15 +DETAIL: Creating dependency on merge taskId 12 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 16 count ------- 125 diff --git a/src/test/regress/expected/multi_large_table_task_assignment.out b/src/test/regress/expected/multi_large_table_task_assignment.out index d310691e9..5c290b6cd 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment.out +++ b/src/test/regress/expected/multi_large_table_task_assignment.out @@ -26,8 +26,8 @@ FROM orders, customer WHERE o_custkey = c_custkey; -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] @@ -35,14 +35,14 @@ DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 5 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 8 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 11 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 +DETAIL: Creating dependency on merge taskId 3 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 9 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 count ------- 2984 @@ -60,12 +60,12 @@ FROM WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; -DEBUG: assigned task 9 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 18 to node localhost:57638 DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 DEBUG: join prunable for intervals [1,1509] and [2951,4455] DEBUG: join prunable for intervals [1,1509] and [4480,5986] DEBUG: join prunable for intervals [1,1509] and [8997,10560] @@ -111,65 +111,65 @@ DEBUG: join prunable for intervals [13473,14947] and [4480,5986] DEBUG: join prunable for intervals [13473,14947] and [8997,10560] DEBUG: join prunable for intervals [13473,14947] and [10560,12036] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 19 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 14 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 26 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 14 +DEBUG: pruning merge fetch taskId 9 +DETAIL: Creating dependency on merge taskId 14 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 14 DEBUG: pruning merge fetch taskId 13 -DETAIL: Creating dependency on merge taskId 26 -DEBUG: pruning merge fetch taskId 16 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 21 +DEBUG: pruning merge fetch taskId 15 +DETAIL: Creating dependency on merge taskId 21 +DEBUG: pruning merge fetch taskId 17 +DETAIL: Creating dependency on merge taskId 28 DEBUG: pruning merge fetch taskId 19 -DETAIL: Creating dependency on merge taskId 33 -DEBUG: pruning merge fetch taskId 22 -DETAIL: Creating dependency on merge taskId 33 +DETAIL: Creating dependency on merge taskId 28 +DEBUG: pruning merge fetch taskId 21 +DETAIL: Creating dependency on merge taskId 35 +DEBUG: pruning merge fetch taskId 23 +DETAIL: Creating dependency on merge taskId 35 DEBUG: pruning merge fetch taskId 25 -DETAIL: Creating dependency on merge taskId 40 -DEBUG: pruning merge fetch taskId 28 -DETAIL: Creating dependency on merge taskId 40 +DETAIL: Creating dependency on merge taskId 42 +DEBUG: pruning merge fetch taskId 27 +DETAIL: Creating dependency on merge taskId 42 +DEBUG: pruning merge fetch taskId 29 +DETAIL: Creating dependency on merge taskId 42 DEBUG: pruning merge fetch taskId 31 -DETAIL: Creating dependency on merge taskId 47 -DEBUG: pruning merge fetch taskId 34 -DETAIL: Creating dependency on merge taskId 47 +DETAIL: Creating dependency on merge taskId 49 +DEBUG: pruning merge fetch taskId 33 +DETAIL: Creating dependency on merge taskId 49 +DEBUG: pruning merge fetch taskId 35 +DETAIL: Creating dependency on merge taskId 49 DEBUG: pruning merge fetch taskId 37 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 40 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 43 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 46 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 49 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 52 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 55 -DETAIL: Creating dependency on merge taskId 68 -DEBUG: pruning merge fetch taskId 58 -DETAIL: Creating dependency on merge taskId 68 -DEBUG: assigned task 21 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 -DEBUG: assigned task 27 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 48 to node localhost:57637 -DEBUG: assigned task 33 to node localhost:57638 -DEBUG: assigned task 39 to node localhost:57637 -DEBUG: assigned task 57 to node localhost:57638 -DEBUG: propagating assignment from merge task 19 to constrained sql task 6 -DEBUG: propagating assignment from merge task 26 to constrained sql task 12 -DEBUG: propagating assignment from merge task 26 to constrained sql task 15 -DEBUG: propagating assignment from merge task 26 to constrained sql task 18 -DEBUG: propagating assignment from merge task 33 to constrained sql task 24 -DEBUG: propagating assignment from merge task 40 to constrained sql task 30 -DEBUG: propagating assignment from merge task 47 to constrained sql task 36 -DEBUG: propagating assignment from merge task 54 to constrained sql task 42 -DEBUG: propagating assignment from merge task 54 to constrained sql task 45 -DEBUG: propagating assignment from merge task 61 to constrained sql task 51 -DEBUG: propagating assignment from merge task 61 to constrained sql task 54 -DEBUG: propagating assignment from merge task 68 to constrained sql task 60 +DETAIL: Creating dependency on merge taskId 56 +DEBUG: pruning merge fetch taskId 39 +DETAIL: Creating dependency on merge taskId 56 +DEBUG: assigned task 14 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 18 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 32 to node localhost:57637 +DEBUG: assigned task 22 to node localhost:57638 +DEBUG: assigned task 26 to node localhost:57637 +DEBUG: assigned task 38 to node localhost:57638 +DEBUG: propagating assignment from merge task 7 to constrained sql task 4 +DEBUG: propagating assignment from merge task 14 to constrained sql task 8 +DEBUG: propagating assignment from merge task 14 to constrained sql task 10 +DEBUG: propagating assignment from merge task 14 to constrained sql task 12 +DEBUG: propagating assignment from merge task 21 to constrained sql task 16 +DEBUG: propagating assignment from merge task 28 to constrained sql task 20 +DEBUG: propagating assignment from merge task 35 to constrained sql task 24 +DEBUG: propagating assignment from merge task 42 to constrained sql task 28 +DEBUG: propagating assignment from merge task 42 to constrained sql task 30 +DEBUG: propagating assignment from merge task 49 to constrained sql task 34 +DEBUG: propagating assignment from merge task 49 to constrained sql task 36 +DEBUG: propagating assignment from merge task 56 to constrained sql task 40 count ------- 11998 @@ -184,17 +184,17 @@ FROM lineitem, customer WHERE l_partkey = c_nationkey; -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 16 to node localhost:57637 -DEBUG: assigned task 14 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 @@ -208,21 +208,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 15 +DETAIL: Creating dependency on merge taskId 12 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 16 DEBUG: assigned task 3 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 9 to node localhost:57638 diff --git a/src/test/regress/expected/multi_large_table_task_assignment_0.out b/src/test/regress/expected/multi_large_table_task_assignment_0.out index d853bda7d..39e8f7a26 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment_0.out +++ b/src/test/regress/expected/multi_large_table_task_assignment_0.out @@ -34,8 +34,8 @@ FROM WHERE o_custkey = c_custkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] @@ -43,14 +43,14 @@ DEBUG: join prunable for intervals [1001,2000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: join prunable for intervals [6001,7000] and [1001,2000] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 5 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 8 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 11 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 +DETAIL: Creating dependency on merge taskId 3 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 9 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: CommitTransactionCommand count ------- @@ -73,12 +73,12 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 9 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 18 to node localhost:57638 DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 DEBUG: join prunable for intervals [1,1509] and [2951,4455] DEBUG: join prunable for intervals [1,1509] and [4480,5986] DEBUG: join prunable for intervals [1,1509] and [8997,10560] @@ -124,65 +124,65 @@ DEBUG: join prunable for intervals [13473,14947] and [4480,5986] DEBUG: join prunable for intervals [13473,14947] and [8997,10560] DEBUG: join prunable for intervals [13473,14947] and [10560,12036] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 19 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 7 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 14 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 26 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 14 +DEBUG: pruning merge fetch taskId 9 +DETAIL: Creating dependency on merge taskId 14 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 14 DEBUG: pruning merge fetch taskId 13 -DETAIL: Creating dependency on merge taskId 26 -DEBUG: pruning merge fetch taskId 16 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 21 +DEBUG: pruning merge fetch taskId 15 +DETAIL: Creating dependency on merge taskId 21 +DEBUG: pruning merge fetch taskId 17 +DETAIL: Creating dependency on merge taskId 28 DEBUG: pruning merge fetch taskId 19 -DETAIL: Creating dependency on merge taskId 33 -DEBUG: pruning merge fetch taskId 22 -DETAIL: Creating dependency on merge taskId 33 +DETAIL: Creating dependency on merge taskId 28 +DEBUG: pruning merge fetch taskId 21 +DETAIL: Creating dependency on merge taskId 35 +DEBUG: pruning merge fetch taskId 23 +DETAIL: Creating dependency on merge taskId 35 DEBUG: pruning merge fetch taskId 25 -DETAIL: Creating dependency on merge taskId 40 -DEBUG: pruning merge fetch taskId 28 -DETAIL: Creating dependency on merge taskId 40 +DETAIL: Creating dependency on merge taskId 42 +DEBUG: pruning merge fetch taskId 27 +DETAIL: Creating dependency on merge taskId 42 +DEBUG: pruning merge fetch taskId 29 +DETAIL: Creating dependency on merge taskId 42 DEBUG: pruning merge fetch taskId 31 -DETAIL: Creating dependency on merge taskId 47 -DEBUG: pruning merge fetch taskId 34 -DETAIL: Creating dependency on merge taskId 47 +DETAIL: Creating dependency on merge taskId 49 +DEBUG: pruning merge fetch taskId 33 +DETAIL: Creating dependency on merge taskId 49 +DEBUG: pruning merge fetch taskId 35 +DETAIL: Creating dependency on merge taskId 49 DEBUG: pruning merge fetch taskId 37 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 40 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 43 -DETAIL: Creating dependency on merge taskId 54 -DEBUG: pruning merge fetch taskId 46 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 49 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 52 -DETAIL: Creating dependency on merge taskId 61 -DEBUG: pruning merge fetch taskId 55 -DETAIL: Creating dependency on merge taskId 68 -DEBUG: pruning merge fetch taskId 58 -DETAIL: Creating dependency on merge taskId 68 -DEBUG: assigned task 21 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 -DEBUG: assigned task 27 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 48 to node localhost:57637 -DEBUG: assigned task 33 to node localhost:57638 -DEBUG: assigned task 39 to node localhost:57637 -DEBUG: assigned task 57 to node localhost:57638 -DEBUG: propagating assignment from merge task 19 to constrained sql task 6 -DEBUG: propagating assignment from merge task 26 to constrained sql task 12 -DEBUG: propagating assignment from merge task 26 to constrained sql task 15 -DEBUG: propagating assignment from merge task 26 to constrained sql task 18 -DEBUG: propagating assignment from merge task 33 to constrained sql task 24 -DEBUG: propagating assignment from merge task 40 to constrained sql task 30 -DEBUG: propagating assignment from merge task 47 to constrained sql task 36 -DEBUG: propagating assignment from merge task 54 to constrained sql task 42 -DEBUG: propagating assignment from merge task 54 to constrained sql task 45 -DEBUG: propagating assignment from merge task 61 to constrained sql task 51 -DEBUG: propagating assignment from merge task 61 to constrained sql task 54 -DEBUG: propagating assignment from merge task 68 to constrained sql task 60 +DETAIL: Creating dependency on merge taskId 56 +DEBUG: pruning merge fetch taskId 39 +DETAIL: Creating dependency on merge taskId 56 +DEBUG: assigned task 14 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 18 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 32 to node localhost:57637 +DEBUG: assigned task 22 to node localhost:57638 +DEBUG: assigned task 26 to node localhost:57637 +DEBUG: assigned task 38 to node localhost:57638 +DEBUG: propagating assignment from merge task 7 to constrained sql task 4 +DEBUG: propagating assignment from merge task 14 to constrained sql task 8 +DEBUG: propagating assignment from merge task 14 to constrained sql task 10 +DEBUG: propagating assignment from merge task 14 to constrained sql task 12 +DEBUG: propagating assignment from merge task 21 to constrained sql task 16 +DEBUG: propagating assignment from merge task 28 to constrained sql task 20 +DEBUG: propagating assignment from merge task 35 to constrained sql task 24 +DEBUG: propagating assignment from merge task 42 to constrained sql task 28 +DEBUG: propagating assignment from merge task 42 to constrained sql task 30 +DEBUG: propagating assignment from merge task 49 to constrained sql task 34 +DEBUG: propagating assignment from merge task 49 to constrained sql task 36 +DEBUG: propagating assignment from merge task 56 to constrained sql task 40 DEBUG: CommitTransactionCommand count ------- @@ -202,17 +202,17 @@ FROM WHERE l_partkey = c_nationkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 16 to node localhost:57637 -DEBUG: assigned task 14 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 5 to node localhost:57638 +DEBUG: assigned task 8 to node localhost:57637 +DEBUG: assigned task 7 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 @@ -226,21 +226,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 17 +DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 26 +DETAIL: Creating dependency on merge taskId 18 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 11 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 35 +DETAIL: Creating dependency on merge taskId 27 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 15 +DETAIL: Creating dependency on merge taskId 12 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 44 +DETAIL: Creating dependency on merge taskId 36 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 16 DEBUG: assigned task 3 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 9 to node localhost:57638 diff --git a/src/test/regress/expected/multi_modifying_xacts.out b/src/test/regress/expected/multi_modifying_xacts.out index 386aaec57..a35ff4c8b 100644 --- a/src/test/regress/expected/multi_modifying_xacts.out +++ b/src/test/regress/expected/multi_modifying_xacts.out @@ -136,7 +136,7 @@ BEGIN; INSERT INTO researchers VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers, labs WHERE labs.id = researchers.lab_id; +SELECT * FROM researchers, labs WHERE labs.id = researchers.lab_id AND researchers.lab_id = 5; id | lab_id | name | id | name ----+--------+-------------------+----+------------ 8 | 5 | Douglas Engelbart | 5 | Los Alamos diff --git a/src/test/regress/expected/multi_multiuser.out b/src/test/regress/expected/multi_multiuser.out index 61b075d60..d6ac26418 100644 --- a/src/test/regress/expected/multi_multiuser.out +++ b/src/test/regress/expected/multi_multiuser.out @@ -6,7 +6,6 @@ SET citus.next_shard_id TO 1420000; ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 1420000; SET citus.shard_replication_factor TO 1; -SET citus.shard_count TO 2; CREATE TABLE test (id integer, val integer); SELECT create_distributed_table('test', 'id'); create_distributed_table @@ -40,6 +39,8 @@ NOTICE: not propagating CREATE ROLE/USER commands to worker nodes HINT: Connect to worker nodes directly to manually create all necessary users and roles. GRANT ALL ON TABLE test_1420000 TO full_access; GRANT SELECT ON TABLE test_1420000 TO read_access; +GRANT ALL ON TABLE test_1420002 TO full_access; +GRANT SELECT ON TABLE test_1420002 TO read_access; \c - - - :worker_2_port CREATE USER full_access; NOTICE: not propagating CREATE ROLE/USER commands to worker nodes @@ -52,6 +53,8 @@ NOTICE: not propagating CREATE ROLE/USER commands to worker nodes HINT: Connect to worker nodes directly to manually create all necessary users and roles. GRANT ALL ON TABLE test_1420001 TO full_access; GRANT SELECT ON TABLE test_1420001 TO read_access; +GRANT ALL ON TABLE test_1420003 TO full_access; +GRANT SELECT ON TABLE test_1420003 TO read_access; \c - - - :master_port -- create prepare tests PREPARE prepare_insert AS INSERT INTO test VALUES ($1); diff --git a/src/test/regress/expected/multi_mx_explain.out b/src/test/regress/expected/multi_mx_explain.out index 00739dd2a..28a284f28 100644 --- a/src/test/regress/expected/multi_mx_explain.out +++ b/src/test/regress/expected/multi_mx_explain.out @@ -8,6 +8,10 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1320000; \a\t SET citus.task_executor_type TO 'real-time'; SET citus.explain_distributed_queries TO on; +VACUUM ANALYZE lineitem_mx; +VACUUM ANALYZE orders_mx; +VACUUM ANALYZE customer_mx; +VACUUM ANALYZE supplier_mx; \c - - - :worker_1_port -- Function that parses explain output as JSON CREATE FUNCTION explain_json(query text) diff --git a/src/test/regress/expected/multi_mx_explain_0.out b/src/test/regress/expected/multi_mx_explain_0.out index a26186c87..ec753ea2f 100644 --- a/src/test/regress/expected/multi_mx_explain_0.out +++ b/src/test/regress/expected/multi_mx_explain_0.out @@ -8,6 +8,10 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1320000; \a\t SET citus.task_executor_type TO 'real-time'; SET citus.explain_distributed_queries TO on; +VACUUM ANALYZE lineitem_mx; +VACUUM ANALYZE orders_mx; +VACUUM ANALYZE customer_mx; +VACUUM ANALYZE supplier_mx; \c - - - :worker_1_port -- Function that parses explain output as JSON CREATE FUNCTION explain_json(query text) diff --git a/src/test/regress/expected/multi_mx_modifying_xacts.out b/src/test/regress/expected/multi_mx_modifying_xacts.out index 3bec55391..3c9c66538 100644 --- a/src/test/regress/expected/multi_mx_modifying_xacts.out +++ b/src/test/regress/expected/multi_mx_modifying_xacts.out @@ -129,7 +129,7 @@ BEGIN; INSERT INTO researchers_mx VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs_mx VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id; +SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id and researchers_mx.lab_id = 5;; id | lab_id | name | id | name ----+--------+-------------------+----+------------ 8 | 5 | Douglas Engelbart | 5 | Los Alamos @@ -147,7 +147,7 @@ BEGIN; INSERT INTO researchers_mx VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs_mx VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id; +SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id and researchers_mx.lab_id = 5; id | lab_id | name | id | name ----+--------+-------------------+----+------------ 8 | 5 | Douglas Engelbart | 5 | Los Alamos diff --git a/src/test/regress/expected/multi_mx_repartition_udt_prepare.out b/src/test/regress/expected/multi_mx_repartition_udt_prepare.out index fd985d68d..2ab57d35f 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_prepare.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_prepare.out @@ -156,10 +156,12 @@ INSERT INTO repartition_udt_other values (10, '(2,1)'::test_udt, 'foo'); INSERT INTO repartition_udt_other values (11, '(2,2)'::test_udt, 'foo'); INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." While removing broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; pk | udtcol | txtcol | pk | udtcol | txtcol ----+--------+--------+----+--------+-------- (0 rows) diff --git a/src/test/regress/expected/multi_mx_repartition_udt_w1.out b/src/test/regress/expected/multi_mx_repartition_udt_w1.out index 7675fc6e9..fbd94b27e 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_w1.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_w1.out @@ -7,11 +7,12 @@ SET client_min_messages = LOG; SET citus.large_table_shard_count = 1; SET citus.task_executor_type = 'task-tracker'; SET citus.log_multi_join_order = true; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; -LOG: join order: [ "repartition_udt" ][ local partition join "repartition_udt_other" ] + WHERE repartition_udt.pk = 1; pk | udtcol | txtcol | pk | udtcol | txtcol ----+--------+--------+----+--------+-------- (0 rows) diff --git a/src/test/regress/expected/multi_mx_repartition_udt_w2.out b/src/test/regress/expected/multi_mx_repartition_udt_w2.out index 362e591fb..76c8a0ef6 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_w2.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_w2.out @@ -7,11 +7,12 @@ SET client_min_messages = LOG; SET citus.large_table_shard_count = 1; SET citus.task_executor_type = 'task-tracker'; SET citus.log_multi_join_order = true; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; -LOG: join order: [ "repartition_udt" ][ local partition join "repartition_udt_other" ] + WHERE repartition_udt.pk = 1; pk | udtcol | txtcol | pk | udtcol | txtcol ----+--------+--------+----+--------+-------- (0 rows) diff --git a/src/test/regress/expected/multi_repartition_udt.out b/src/test/regress/expected/multi_repartition_udt.out index fb9fb50c8..1307fa114 100644 --- a/src/test/regress/expected/multi_repartition_udt.out +++ b/src/test/regress/expected/multi_repartition_udt.out @@ -164,10 +164,12 @@ INSERT INTO repartition_udt_other values (10, '(2,1)'::test_udt, 'foo'); INSERT INTO repartition_udt_other values (11, '(2,2)'::test_udt, 'foo'); INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; pk | udtcol | txtcol | pk | udtcol | txtcol ----+--------+--------+----+--------+-------- (0 rows) diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index 9c3f7aa26..62317e4cd 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -1667,31 +1667,81 @@ DEBUG: Plan is router executable ----+-----------+-------+------------+------+---- (0 rows) --- multi-shard join is not router plannable -SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) +-- This query was intended to test "multi-shard join is not router plannable" +-- To run it using repartition join logic we change the join columns +SET citus.task_executor_type to "task-tracker"; +SELECT * FROM articles_range ar join authors_range au on (ar.title = au.name) WHERE ar.author_id = 35; -DEBUG: join prunable for intervals [21,40] and [1,10] -DEBUG: join prunable for intervals [31,40] and [1,10] -DEBUG: join prunable for intervals [31,40] and [11,30] +DEBUG: join prunable for task partitionId 0 and 1 +DEBUG: join prunable for task partitionId 0 and 2 +DEBUG: join prunable for task partitionId 0 and 3 +DEBUG: join prunable for task partitionId 1 and 0 +DEBUG: join prunable for task partitionId 1 and 2 +DEBUG: join prunable for task partitionId 1 and 3 +DEBUG: join prunable for task partitionId 2 and 0 +DEBUG: join prunable for task partitionId 2 and 1 +DEBUG: join prunable for task partitionId 2 and 3 +DEBUG: join prunable for task partitionId 3 and 0 +DEBUG: join prunable for task partitionId 3 and 1 +DEBUG: join prunable for task partitionId 3 and 2 +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 3 +DEBUG: pruning merge fetch taskId 2 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 10 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 9 +DEBUG: pruning merge fetch taskId 8 +DETAIL: Creating dependency on merge taskId 15 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 12 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 20 id | author_id | title | word_count | name | id ----+-----------+-------+------------+------+---- (0 rows) --- this is a bug, it is a single shard join query but not router plannable -SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) +-- This query was intended to test "this is a bug, it is a single shard join +-- query but not router plannable". To run it using repartition join logic we +-- change the join columns. +SELECT * FROM articles_range ar join authors_range au on (ar.title = au.name) WHERE ar.author_id = 1 or au.id = 5; -DEBUG: join prunable for intervals [1,10] and [11,30] -DEBUG: join prunable for intervals [1,10] and [21,40] -DEBUG: join prunable for intervals [1,10] and [31,40] -DEBUG: join prunable for intervals [11,30] and [1,10] -DEBUG: join prunable for intervals [11,30] and [31,40] -DEBUG: join prunable for intervals [21,40] and [1,10] -DEBUG: join prunable for intervals [31,40] and [1,10] -DEBUG: join prunable for intervals [31,40] and [11,30] +DEBUG: join prunable for task partitionId 0 and 1 +DEBUG: join prunable for task partitionId 0 and 2 +DEBUG: join prunable for task partitionId 0 and 3 +DEBUG: join prunable for task partitionId 1 and 0 +DEBUG: join prunable for task partitionId 1 and 2 +DEBUG: join prunable for task partitionId 1 and 3 +DEBUG: join prunable for task partitionId 2 and 0 +DEBUG: join prunable for task partitionId 2 and 1 +DEBUG: join prunable for task partitionId 2 and 3 +DEBUG: join prunable for task partitionId 3 and 0 +DEBUG: join prunable for task partitionId 3 and 1 +DEBUG: join prunable for task partitionId 3 and 2 +DEBUG: pruning merge fetch taskId 1 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 2 +DETAIL: Creating dependency on merge taskId 5 +DEBUG: pruning merge fetch taskId 4 +DETAIL: Creating dependency on merge taskId 10 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 10 +DEBUG: pruning merge fetch taskId 7 +DETAIL: Creating dependency on merge taskId 15 +DEBUG: pruning merge fetch taskId 8 +DETAIL: Creating dependency on merge taskId 15 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 20 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 20 id | author_id | title | word_count | name | id ----+-----------+-------+------------+------+---- (0 rows) +RESET citus.task_executor_type; -- bogus query, join on non-partition column, but router plannable due to filters SELECT * FROM articles_range ar join authors_range au on (ar.id = au.id) WHERE ar.author_id = 1 and au.id < 10; @@ -1726,21 +1776,21 @@ DEBUG: join prunable for intervals [21,40] and [1,10] DEBUG: join prunable for intervals [31,40] and [1,10] DEBUG: join prunable for intervals [31,40] and [11,30] DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 2 +DEBUG: pruning merge fetch taskId 3 +DETAIL: Creating dependency on merge taskId 4 +DEBUG: pruning merge fetch taskId 5 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 5 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 9 +DETAIL: Creating dependency on merge taskId 6 +DEBUG: pruning merge fetch taskId 11 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 13 -DETAIL: Creating dependency on merge taskId 7 -DEBUG: pruning merge fetch taskId 16 -DETAIL: Creating dependency on merge taskId 7 -DEBUG: pruning merge fetch taskId 19 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 22 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 8 +DEBUG: pruning merge fetch taskId 15 +DETAIL: Creating dependency on merge taskId 8 ERROR: the query contains a join that requires repartitioning HINT: Set citus.enable_repartition_joins to on to enable repartitioning -- join between a range partitioned table and reference table is router plannable @@ -1794,7 +1844,7 @@ WARNING: relation "public.articles_append" does not exist CONTEXT: while executing command on localhost:57638 WARNING: relation "public.articles_append" does not exist CONTEXT: while executing command on localhost:57638 -ERROR: failed to execute task 2 +ERROR: failed to execute task 1 -- same query with where false but evaluation left to worker SELECT author_id FROM articles_append WHERE @@ -1809,7 +1859,7 @@ WARNING: relation "public.articles_append" does not exist CONTEXT: while executing command on localhost:57638 WARNING: relation "public.articles_append" does not exist CONTEXT: while executing command on localhost:57638 -ERROR: failed to execute task 2 +ERROR: failed to execute task 1 -- same query on router planner with where false but evaluation left to worker SELECT author_id FROM articles_single_shard_hash WHERE diff --git a/src/test/regress/expected/multi_simple_queries.out b/src/test/regress/expected/multi_simple_queries.out index 6f2227013..608822129 100644 --- a/src/test/regress/expected/multi_simple_queries.out +++ b/src/test/regress/expected/multi_simple_queries.out @@ -215,6 +215,7 @@ SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIK (0 rows) -- subqueries are supported in FROM clause +SET citus.large_table_shard_count TO 1; SELECT articles.id,test.word_count FROM articles, (SELECT id, word_count FROM articles) AS test WHERE test.id = articles.id ORDER BY articles.id; @@ -272,6 +273,7 @@ ORDER BY articles.id; 50 | 19519 (50 rows) +RESET citus.large_table_shard_count; -- subqueries are not supported in SELECT clause SELECT a.title AS name, (SELECT a2.id FROM articles_single_shard a2 WHERE a.id = a2.id LIMIT 1) AS special_price FROM articles a; @@ -575,21 +577,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 +DETAIL: Creating dependency on merge taskId 2 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 3 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 8 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 11 -DEBUG: pruning merge fetch taskId 10 DETAIL: Creating dependency on merge taskId 9 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 12 ERROR: the query contains a join that requires repartitioning HINT: Set citus.enable_repartition_joins to on to enable repartitioning -- system columns from shard tables can be queried and retrieved diff --git a/src/test/regress/expected/multi_simple_queries_0.out b/src/test/regress/expected/multi_simple_queries_0.out index b52940258..bad6c3232 100644 --- a/src/test/regress/expected/multi_simple_queries_0.out +++ b/src/test/regress/expected/multi_simple_queries_0.out @@ -212,10 +212,12 @@ SELECT * FROM articles, position('om' in 'Thomas') ORDER BY 2 DESC, 1 DESC, 3 DE SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); ERROR: Complex subqueries and CTEs are not supported when task_executor_type is set to 'task-tracker' -- subqueries are supported in FROM clause +SET citus.large_table_shard_count TO 1; SELECT articles.id,test.word_count FROM articles, (SELECT id, word_count FROM articles) AS test WHERE test.id = articles.id ORDER BY articles.id; ERROR: Complex subqueries and CTEs are not supported when task_executor_type is set to 'task-tracker' +RESET citus.large_table_shard_count; -- subqueries are not supported in SELECT clause SELECT a.title AS name, (SELECT a2.id FROM articles_single_shard a2 WHERE a.id = a2.id LIMIT 1) AS special_price FROM articles a; @@ -519,21 +521,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 +DETAIL: Creating dependency on merge taskId 2 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 3 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 5 +DETAIL: Creating dependency on merge taskId 4 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 8 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 7 +DETAIL: Creating dependency on merge taskId 6 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 11 -DEBUG: pruning merge fetch taskId 10 DETAIL: Creating dependency on merge taskId 9 +DEBUG: pruning merge fetch taskId 10 +DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 12 ERROR: the query contains a join that requires repartitioning HINT: Set citus.enable_repartition_joins to on to enable repartitioning -- system columns from shard tables can be queried and retrieved diff --git a/src/test/regress/expected/multi_subquery.out b/src/test/regress/expected/multi_subquery.out index 52a911d2e..736add4f8 100644 --- a/src/test/regress/expected/multi_subquery.out +++ b/src/test/regress/expected/multi_subquery.out @@ -60,53 +60,6 @@ FROM l_orderkey) AS unit_prices; ERROR: cannot perform distributed planning on this query DETAIL: Cartesian products are currently unsupported --- this query is only required to execute --- the following query given that recursive planning --- (in general real-time queries in transactions) --- do not execute shard fetch tasks and the next --- query relies on that -SELECT - l_orderkey, - avg(o_totalprice / l_quantity) AS unit_price - FROM - lineitem_subquery, - orders_subquery - WHERE - l_orderkey = o_custkey - GROUP BY - l_orderkey - ORDER BY 2 DESC, 1 DESC - LIMIT 5; -DEBUG: push down of limit count: 5 - l_orderkey | unit_price -------------+------------------------ - 421 | 102749.853333333333 - 806 | 96257.4480681818181818 - 418 | 57151.4156630824373871 - 1124 | 56102.2804738959822181 - 230 | 53847.0509778948909754 -(5 rows) - -SELECT - avg(unit_price) -FROM - (SELECT - l_orderkey, - avg(o_totalprice / l_quantity) AS unit_price - FROM - lineitem_subquery, - orders_subquery - WHERE - l_orderkey = o_custkey - GROUP BY - l_orderkey) AS unit_prices; -DEBUG: generating subplan 7_1 for subquery SELECT lineitem_subquery.l_orderkey, avg((orders_subquery.o_totalprice / lineitem_subquery.l_quantity)) AS unit_price FROM public.lineitem_subquery, public.orders_subquery WHERE (lineitem_subquery.l_orderkey = orders_subquery.o_custkey) GROUP BY lineitem_subquery.l_orderkey -DEBUG: Plan 7 query after replacing subqueries and CTEs: SELECT avg(unit_price) AS avg FROM (SELECT intermediate_result.l_orderkey, intermediate_result.unit_price FROM read_intermediate_result('7_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint, unit_price numeric)) unit_prices - avg ------------------------- - 12973.7343244916919367 -(1 row) - RESET client_min_messages; -- Subqueries without relation with a volatile functions (non-constant) are planned recursively SELECT count(*) FROM ( @@ -124,11 +77,11 @@ SELECT count(*) FROM (SELECT l_orderkey FROM lineitem_subquery) UNION ALL (SELECT 1::bigint) ) b; -DEBUG: generating subplan 10_1 for subquery SELECT l_orderkey FROM public.lineitem_subquery +DEBUG: generating subplan 7_1 for subquery SELECT l_orderkey FROM public.lineitem_subquery DEBUG: Creating router plan DEBUG: Plan is router executable -DEBUG: generating subplan 10_2 for subquery SELECT intermediate_result.l_orderkey FROM read_intermediate_result('10_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint) UNION ALL SELECT (1)::bigint AS int8 -DEBUG: Plan 10 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('10_2'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) b +DEBUG: generating subplan 7_2 for subquery SELECT intermediate_result.l_orderkey FROM read_intermediate_result('7_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint) UNION ALL SELECT (1)::bigint AS int8 +DEBUG: Plan 7 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('7_2'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) b DEBUG: Creating router plan DEBUG: Plan is router executable count @@ -142,12 +95,12 @@ SELECT count(*) FROM (SELECT l_orderkey FROM lineitem_subquery) UNION (SELECT l_partkey FROM lineitem_subquery) ) b; -DEBUG: generating subplan 13_1 for subquery SELECT l_orderkey FROM public.lineitem_subquery -DEBUG: generating subplan 13_2 for subquery SELECT l_partkey FROM public.lineitem_subquery +DEBUG: generating subplan 10_1 for subquery SELECT l_orderkey FROM public.lineitem_subquery +DEBUG: generating subplan 10_2 for subquery SELECT l_partkey FROM public.lineitem_subquery DEBUG: Creating router plan DEBUG: Plan is router executable -DEBUG: generating subplan 13_3 for subquery SELECT intermediate_result.l_orderkey FROM read_intermediate_result('13_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint) UNION SELECT intermediate_result.l_partkey FROM read_intermediate_result('13_2'::text, 'binary'::citus_copy_format) intermediate_result(l_partkey integer) -DEBUG: Plan 13 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('13_3'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) b +DEBUG: generating subplan 10_3 for subquery SELECT intermediate_result.l_orderkey FROM read_intermediate_result('10_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint) UNION SELECT intermediate_result.l_partkey FROM read_intermediate_result('10_2'::text, 'binary'::citus_copy_format) intermediate_result(l_partkey integer) +DEBUG: Plan 10 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('10_3'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) b DEBUG: Creating router plan DEBUG: Plan is router executable count diff --git a/src/test/regress/expected/multi_subquery_behavioral_analytics.out b/src/test/regress/expected/multi_subquery_behavioral_analytics.out index db4877dd8..3b35820f0 100644 --- a/src/test/regress/expected/multi_subquery_behavioral_analytics.out +++ b/src/test/regress/expected/multi_subquery_behavioral_analytics.out @@ -2228,6 +2228,27 @@ FROM 5 | 0 | 0.50000000000000000000 (5 rows) +-- Getting aggregation of value which is created by aggregation in subquery +SELECT + count(1), + avg(agg_value) +FROM + (SELECT + users_table.user_id, + avg(users_table.value_1 / events_table.value_4) AS agg_value + FROM + users_table, + events_table + WHERE + users_table.user_id = events_table.user_id + GROUP BY + 1 + ) AS temp; + count | avg +-------+----- + 6 | +(1 row) + DROP FUNCTION test_join_function_2(integer, integer); SELECT run_command_on_workers($f$ diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index a3daab850..dd0d2f4bf 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -57,9 +57,9 @@ SET client_min_messages TO DEBUG3; -- First test the default greedy task assignment policy SET citus.task_assignment_policy TO 'greedy'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -68,9 +68,9 @@ DEBUG: assigned task 4 to node localhost:57637 (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -81,9 +81,9 @@ DEBUG: assigned task 4 to node localhost:57637 -- Next test the first-replica task assignment policy SET citus.task_assignment_policy TO 'first-replica'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -92,9 +92,9 @@ DEBUG: assigned task 2 to node localhost:57638 (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -105,20 +105,9 @@ DEBUG: assigned task 2 to node localhost:57638 -- Finally test the round-robin task assignment policy SET citus.task_assignment_policy TO 'round-robin'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57638 -DEBUG: assigned task 2 to node localhost:57637 - QUERY PLAN ------------------------------------------------------------------------ - Aggregate (cost=0.00..0.00 rows=0 width=0) - -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) - explain statements for distributed queries are not enabled -(3 rows) - -EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) @@ -127,9 +116,20 @@ DEBUG: assigned task 2 to node localhost:57638 (3 rows) EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 QUERY PLAN ----------------------------------------------------------------------- Aggregate (cost=0.00..0.00 rows=0 width=0) diff --git a/src/test/regress/expected/multi_task_assignment_policy_0.out b/src/test/regress/expected/multi_task_assignment_policy_0.out index 8741e0459..fb4ffda24 100644 --- a/src/test/regress/expected/multi_task_assignment_policy_0.out +++ b/src/test/regress/expected/multi_task_assignment_policy_0.out @@ -63,9 +63,9 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- @@ -77,9 +77,9 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- @@ -96,9 +96,9 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- @@ -110,9 +110,9 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- @@ -129,23 +129,9 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57638 -DEBUG: assigned task 2 to node localhost:57637 -DEBUG: CommitTransactionCommand - QUERY PLAN ------------------------------------------------------------------------ - Aggregate (cost=0.00..0.00 rows=0 width=0) - -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) - explain statements for distributed queries are not enabled -(3 rows) - -EXPLAIN SELECT count(*) FROM task_assignment_test_table; -DEBUG: StartTransactionCommand -DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 4 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- @@ -157,9 +143,23 @@ DEBUG: CommitTransactionCommand EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: StartTransactionCommand DEBUG: ProcessUtility -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 1 to node localhost:57638 +DEBUG: CommitTransactionCommand + QUERY PLAN +----------------------------------------------------------------------- + Aggregate (cost=0.00..0.00 rows=0 width=0) + -> Custom Scan (Citus Real-Time) (cost=0.00..0.00 rows=0 width=0) + explain statements for distributed queries are not enabled +(3 rows) + +EXPLAIN SELECT count(*) FROM task_assignment_test_table; +DEBUG: StartTransactionCommand +DEBUG: ProcessUtility +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 1 to node localhost:57637 DEBUG: CommitTransactionCommand QUERY PLAN ----------------------------------------------------------------------- diff --git a/src/test/regress/expected/set_operation_and_local_tables.out b/src/test/regress/expected/set_operation_and_local_tables.out index 42113d061..cdd3a19f4 100644 --- a/src/test/regress/expected/set_operation_and_local_tables.out +++ b/src/test/regress/expected/set_operation_and_local_tables.out @@ -297,21 +297,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: cannot use real time executor with repartition jobs HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. DEBUG: generating subplan 53_1 for subquery SELECT t1.x FROM recursive_set_local.test t1, recursive_set_local.test t2 WHERE (t1.x = t2.y) LIMIT 2 diff --git a/src/test/regress/expected/set_operations.out b/src/test/regress/expected/set_operations.out index 39ba33d63..d9593cc6e 100644 --- a/src/test/regress/expected/set_operations.out +++ b/src/test/regress/expected/set_operations.out @@ -760,21 +760,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: cannot use real time executor with repartition jobs HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. DEBUG: generating subplan 164_1 for subquery SELECT t1.x FROM recursive_union.test t1, recursive_union.test t2 WHERE (t1.x = t2.y) LIMIT 0 @@ -801,21 +801,21 @@ DEBUG: join prunable for task partitionId 3 and 0 DEBUG: join prunable for task partitionId 3 and 1 DEBUG: join prunable for task partitionId 3 and 2 DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 9 +DETAIL: Creating dependency on merge taskId 5 DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 14 +DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 19 +DETAIL: Creating dependency on merge taskId 15 DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 24 +DETAIL: Creating dependency on merge taskId 20 DEBUG: cannot use real time executor with repartition jobs HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. DEBUG: generating subplan 167_1 for subquery SELECT t1.x FROM recursive_union.test t1, recursive_union.test t2 WHERE (t1.x = t2.y) diff --git a/src/test/regress/input/multi_append_table_to_shard.source b/src/test/regress/input/multi_append_table_to_shard.source index 213ef3c6b..326a8339f 100644 --- a/src/test/regress/input/multi_append_table_to_shard.source +++ b/src/test/regress/input/multi_append_table_to_shard.source @@ -34,8 +34,8 @@ SELECT set_config('citus.shard_replication_factor', '2', false); \copy multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \copy multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' --- Place 'right' table only on the primary worker -SELECT set_config('citus.shard_replication_factor', '1', false); +-- Place 'right' table on both workers +SELECT set_config('citus.shard_replication_factor', '2', false); \copy multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' -- Reset shard replication factor to ensure tasks will be assigned to both workers @@ -54,34 +54,8 @@ CREATE TABLE multi_append_table_to_shard_stage text TEXT not null ); -COPY multi_append_table_to_shard_stage FROM '@abs_srcdir@/data/agg.data'; - -SELECT master_append_table_to_shard(shardid, 'multi_append_table_to_shard_stage', 'localhost', 57636) -FROM - pg_dist_shard -WHERE 'multi_append_table_to_shard_right'::regclass::oid = logicalrelid; - --- Only the primary worker will see the new matches, as the secondary still uses a cached shard -SELECT COUNT(*) -FROM multi_append_table_to_shard_left, - multi_append_table_to_shard_right -WHERE left_number = right_number; - --- Now add a lot of data to ensure we increase the size on disk -DELETE FROM multi_append_table_to_shard_stage; COPY multi_append_table_to_shard_stage FROM '@abs_srcdir@/data/large_records.data' with delimiter '|'; -SELECT master_append_table_to_shard(shardid, 'multi_append_table_to_shard_stage', 'localhost', 57636) -FROM - pg_dist_shard -WHERE 'multi_append_table_to_shard_right'::regclass::oid = logicalrelid; - --- This join will refresh the shard on the secondary, all 8 rows in the left table will match twice (16) -SELECT COUNT(*) -FROM multi_append_table_to_shard_left, - multi_append_table_to_shard_right -WHERE left_number = right_number; - -- Check that we error out if we try to append data to a hash partitioned table. SELECT master_create_empty_shard('multi_append_table_to_shard_right_hash'); diff --git a/src/test/regress/input/multi_mx_copy_data.source b/src/test/regress/input/multi_mx_copy_data.source index 7b6a991b0..60cd23d83 100644 --- a/src/test/regress/input/multi_mx_copy_data.source +++ b/src/test/regress/input/multi_mx_copy_data.source @@ -17,10 +17,15 @@ SET search_path TO public; \COPY lineitem_mx FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \COPY lineitem_mx FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' +\c - - - :worker_2_port +-- and use second worker as well \COPY orders_mx FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' \COPY orders_mx FROM '@abs_srcdir@/data/orders.2.data' with delimiter '|' --- and use coordinator for reference tables +-- These copies were intended to test copying data to single sharded table from +-- worker nodes, yet in order to remove broadcast logic related codes we change +-- the the table to reference table and copy data from master. Should be updated +-- when worker nodes gain capability to run dml commands on reference tables. \c - - - :master_port SET search_path TO public; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 7f0007ecd..461d773dc 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -159,7 +159,8 @@ test: multi_transaction_recovery # multi_copy creates hash and range-partitioned tables and performs COPY # multi_router_planner creates hash partitioned tables. # --------- -test: multi_copy multi_router_planner +test: multi_copy +test: multi_router_planner # ---------- # multi_large_shardid loads more lineitem data using high shard identifiers @@ -198,11 +199,6 @@ test: multi_function_evaluation # ---------- test: multi_truncate -# ---------- -# multi_expire_table_cache tests for broadcast tables -# ---------- -test: multi_expire_table_cache - # ---------- # multi_colocation_utils tests utility functions written for co-location feature & internal API # multi_colocated_shard_transfer tests master_copy_shard_placement with colocated tables. diff --git a/src/test/regress/output/multi_append_table_to_shard.source b/src/test/regress/output/multi_append_table_to_shard.source index 341f5fe0f..5e6b617c9 100644 --- a/src/test/regress/output/multi_append_table_to_shard.source +++ b/src/test/regress/output/multi_append_table_to_shard.source @@ -51,11 +51,11 @@ SELECT set_config('citus.shard_replication_factor', '2', false); \copy multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \copy multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' --- Place 'right' table only on the primary worker -SELECT set_config('citus.shard_replication_factor', '1', false); +-- Place 'right' table on both workers +SELECT set_config('citus.shard_replication_factor', '2', false); set_config ------------ - 1 + 2 (1 row) \copy multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' @@ -82,48 +82,7 @@ CREATE TABLE multi_append_table_to_shard_stage number INTEGER not null, text TEXT not null ); -COPY multi_append_table_to_shard_stage FROM '@abs_srcdir@/data/agg.data'; -SELECT master_append_table_to_shard(shardid, 'multi_append_table_to_shard_stage', 'localhost', 57636) -FROM - pg_dist_shard -WHERE 'multi_append_table_to_shard_right'::regclass::oid = logicalrelid; - master_append_table_to_shard ------------------------------- - 0.0533333 -(1 row) - --- Only the primary worker will see the new matches, as the secondary still uses a cached shard -SELECT COUNT(*) -FROM multi_append_table_to_shard_left, - multi_append_table_to_shard_right -WHERE left_number = right_number; - count -------- - 12 -(1 row) - --- Now add a lot of data to ensure we increase the size on disk -DELETE FROM multi_append_table_to_shard_stage; COPY multi_append_table_to_shard_stage FROM '@abs_srcdir@/data/large_records.data' with delimiter '|'; -SELECT master_append_table_to_shard(shardid, 'multi_append_table_to_shard_stage', 'localhost', 57636) -FROM - pg_dist_shard -WHERE 'multi_append_table_to_shard_right'::regclass::oid = logicalrelid; - master_append_table_to_shard ------------------------------- - 0.106667 -(1 row) - --- This join will refresh the shard on the secondary, all 8 rows in the left table will match twice (16) -SELECT COUNT(*) -FROM multi_append_table_to_shard_left, - multi_append_table_to_shard_right -WHERE left_number = right_number; - count -------- - 16 -(1 row) - -- Check that we error out if we try to append data to a hash partitioned table. SELECT master_create_empty_shard('multi_append_table_to_shard_right_hash'); ERROR: relation "multi_append_table_to_shard_right_hash" is a hash partitioned table diff --git a/src/test/regress/output/multi_mx_copy_data.source b/src/test/regress/output/multi_mx_copy_data.source index 658f285a8..4a03b449a 100644 --- a/src/test/regress/output/multi_mx_copy_data.source +++ b/src/test/regress/output/multi_mx_copy_data.source @@ -12,9 +12,14 @@ SET search_path TO citus_mx_test_schema; SET search_path TO public; \COPY lineitem_mx FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \COPY lineitem_mx FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' +\c - - - :worker_2_port +-- and use second worker as well \COPY orders_mx FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' \COPY orders_mx FROM '@abs_srcdir@/data/orders.2.data' with delimiter '|' --- and use coordinator for reference tables +-- These copies were intended to test copying data to single sharded table from +-- worker nodes, yet in order to remove broadcast logic related codes we change +-- the the table to reference table and copy data from master. Should be updated +-- when worker nodes gain capability to run dml commands on reference tables. \c - - - :master_port SET search_path TO public; \COPY customer_mx FROM '@abs_srcdir@/data/customer.1.data' with delimiter '|' diff --git a/src/test/regress/sql/multi_expire_table_cache.sql b/src/test/regress/sql/multi_expire_table_cache.sql deleted file mode 100644 index 2e7ac3c5d..000000000 --- a/src/test/regress/sql/multi_expire_table_cache.sql +++ /dev/null @@ -1,53 +0,0 @@ ---- ---- MULTI_EXPIRE_TABLE_CACHE ---- - - -SET citus.next_shard_id TO 1220000; - --- create test table -CREATE TABLE large_table(a int, b int); -SELECT master_create_distributed_table('large_table', 'a', 'hash'); -SELECT master_create_worker_shards('large_table', 8, 1); - -CREATE TABLE broadcast_table(a int, b int); -SELECT master_create_distributed_table('broadcast_table', 'a', 'hash'); -SELECT master_create_worker_shards('broadcast_table', 2, 1); - --- verify only small tables are supported -SELECT master_expire_table_cache('large_table'); -SELECT master_expire_table_cache('broadcast_table'); - --- run a join so that broadcast tables are cached on other workers -SELECT * from large_table l, broadcast_table b where l.a = b.b; - --- insert some data -INSERT INTO large_table VALUES(1, 1); -INSERT INTO large_table VALUES(1, 2); -INSERT INTO large_table VALUES(2, 1); -INSERT INTO large_table VALUES(2, 2); -INSERT INTO large_table VALUES(3, 1); -INSERT INTO large_table VALUES(3, 2); - -INSERT INTO broadcast_table VALUES(1, 1); - --- verify returned results are wrong -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - --- expire cache and re-run, results should be correct this time -SELECT master_expire_table_cache('broadcast_table'); - -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - --- insert some more data into broadcast table -INSERT INTO broadcast_table VALUES(2, 2); - --- run the same query, get wrong results -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - --- expire cache and re-run, results should be correct this time -SELECT master_expire_table_cache('broadcast_table'); -SELECT * from large_table l, broadcast_table b WHERE l.b = b.b ORDER BY l.a, l.b; - - -DROP TABLE large_table, broadcast_table; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index a907f192b..45f6ff983 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -133,6 +133,7 @@ ALTER EXTENSION citus UPDATE TO '7.2-1'; ALTER EXTENSION citus UPDATE TO '7.2-2'; ALTER EXTENSION citus UPDATE TO '7.2-3'; ALTER EXTENSION citus UPDATE TO '7.3-3'; +ALTER EXTENSION citus UPDATE TO '7.4-1'; -- show running version SHOW citus.version; diff --git a/src/test/regress/sql/multi_modifying_xacts.sql b/src/test/regress/sql/multi_modifying_xacts.sql index 7b575136c..1a987ec11 100644 --- a/src/test/regress/sql/multi_modifying_xacts.sql +++ b/src/test/regress/sql/multi_modifying_xacts.sql @@ -111,7 +111,7 @@ INSERT INTO researchers VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers, labs WHERE labs.id = researchers.lab_id; +SELECT * FROM researchers, labs WHERE labs.id = researchers.lab_id AND researchers.lab_id = 5; -- and the other way around is also allowed BEGIN; diff --git a/src/test/regress/sql/multi_multiuser.sql b/src/test/regress/sql/multi_multiuser.sql index 8f30b70f6..c2137febd 100644 --- a/src/test/regress/sql/multi_multiuser.sql +++ b/src/test/regress/sql/multi_multiuser.sql @@ -8,7 +8,6 @@ SET citus.next_shard_id TO 1420000; ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 1420000; SET citus.shard_replication_factor TO 1; -SET citus.shard_count TO 2; CREATE TABLE test (id integer, val integer); SELECT create_distributed_table('test', 'id'); @@ -33,6 +32,9 @@ CREATE USER no_access; GRANT ALL ON TABLE test_1420000 TO full_access; GRANT SELECT ON TABLE test_1420000 TO read_access; +GRANT ALL ON TABLE test_1420002 TO full_access; +GRANT SELECT ON TABLE test_1420002 TO read_access; + \c - - - :worker_2_port CREATE USER full_access; CREATE USER read_access; @@ -41,6 +43,9 @@ CREATE USER no_access; GRANT ALL ON TABLE test_1420001 TO full_access; GRANT SELECT ON TABLE test_1420001 TO read_access; +GRANT ALL ON TABLE test_1420003 TO full_access; +GRANT SELECT ON TABLE test_1420003 TO read_access; + \c - - - :master_port -- create prepare tests diff --git a/src/test/regress/sql/multi_mx_explain.sql b/src/test/regress/sql/multi_mx_explain.sql index be396d91f..9d5f8034a 100644 --- a/src/test/regress/sql/multi_mx_explain.sql +++ b/src/test/regress/sql/multi_mx_explain.sql @@ -12,6 +12,11 @@ ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1320000; SET citus.task_executor_type TO 'real-time'; SET citus.explain_distributed_queries TO on; +VACUUM ANALYZE lineitem_mx; +VACUUM ANALYZE orders_mx; +VACUUM ANALYZE customer_mx; +VACUUM ANALYZE supplier_mx; + \c - - - :worker_1_port -- Function that parses explain output as JSON CREATE FUNCTION explain_json(query text) diff --git a/src/test/regress/sql/multi_mx_modifying_xacts.sql b/src/test/regress/sql/multi_mx_modifying_xacts.sql index 0e74d8fbc..3b4ad87b7 100644 --- a/src/test/regress/sql/multi_mx_modifying_xacts.sql +++ b/src/test/regress/sql/multi_mx_modifying_xacts.sql @@ -116,7 +116,7 @@ INSERT INTO researchers_mx VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs_mx VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id; +SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id and researchers_mx.lab_id = 5;; -- and the other way around is also allowed BEGIN; @@ -132,7 +132,7 @@ INSERT INTO researchers_mx VALUES (8, 5, 'Douglas Engelbart'); INSERT INTO labs_mx VALUES (5, 'Los Alamos'); COMMIT; -SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id; +SELECT * FROM researchers_mx, labs_mx WHERE labs_mx.id = researchers_mx.lab_id and researchers_mx.lab_id = 5; -- and the other way around is also allowed BEGIN; diff --git a/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql b/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql index db415ac7d..e48d9945b 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql @@ -192,10 +192,12 @@ INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." While removing broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; -- Query that should result in a repartition join on UDT column. SET citus.large_table_shard_count = 1; diff --git a/src/test/regress/sql/multi_mx_repartition_udt_w1.sql b/src/test/regress/sql/multi_mx_repartition_udt_w1.sql index 9b91f7d9a..6a7bcabe5 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_w1.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_w1.sql @@ -9,10 +9,12 @@ SET citus.large_table_shard_count = 1; SET citus.task_executor_type = 'task-tracker'; SET citus.log_multi_join_order = true; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.udtcol = repartition_udt_other.udtcol diff --git a/src/test/regress/sql/multi_mx_repartition_udt_w2.sql b/src/test/regress/sql/multi_mx_repartition_udt_w2.sql index 26b45a631..528bc50b9 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_w2.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_w2.sql @@ -9,10 +9,12 @@ SET citus.large_table_shard_count = 1; SET citus.task_executor_type = 'task-tracker'; SET citus.log_multi_join_order = true; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.udtcol = repartition_udt_other.udtcol diff --git a/src/test/regress/sql/multi_repartition_udt.sql b/src/test/regress/sql/multi_repartition_udt.sql index 359d63d01..968492c51 100644 --- a/src/test/regress/sql/multi_repartition_udt.sql +++ b/src/test/regress/sql/multi_repartition_udt.sql @@ -189,10 +189,12 @@ INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; --- Query that should result in a repartition join on int column, and be empty. +-- This query was intended to test "Query that should result in a repartition +-- join on int column, and be empty." In order to remove broadcast logic, we +-- manually make the query router plannable. SELECT * FROM repartition_udt JOIN repartition_udt_other ON repartition_udt.pk = repartition_udt_other.pk - WHERE repartition_udt.pk > 1; + WHERE repartition_udt.pk = 1; -- Query that should result in a repartition join on UDT column. SET citus.large_table_shard_count = 1; diff --git a/src/test/regress/sql/multi_router_planner.sql b/src/test/regress/sql/multi_router_planner.sql index 0f0bc9e2a..83b960d4c 100644 --- a/src/test/regress/sql/multi_router_planner.sql +++ b/src/test/regress/sql/multi_router_planner.sql @@ -788,13 +788,18 @@ SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) WHERE ar.author_id = 1 and au.id = 2; --- multi-shard join is not router plannable -SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) +-- This query was intended to test "multi-shard join is not router plannable" +-- To run it using repartition join logic we change the join columns +SET citus.task_executor_type to "task-tracker"; +SELECT * FROM articles_range ar join authors_range au on (ar.title = au.name) WHERE ar.author_id = 35; --- this is a bug, it is a single shard join query but not router plannable -SELECT * FROM articles_range ar join authors_range au on (ar.author_id = au.id) +-- This query was intended to test "this is a bug, it is a single shard join +-- query but not router plannable". To run it using repartition join logic we +-- change the join columns. +SELECT * FROM articles_range ar join authors_range au on (ar.title = au.name) WHERE ar.author_id = 1 or au.id = 5; +RESET citus.task_executor_type; -- bogus query, join on non-partition column, but router plannable due to filters SELECT * FROM articles_range ar join authors_range au on (ar.id = au.id) diff --git a/src/test/regress/sql/multi_simple_queries.sql b/src/test/regress/sql/multi_simple_queries.sql index 18dcab0e8..43250c520 100644 --- a/src/test/regress/sql/multi_simple_queries.sql +++ b/src/test/regress/sql/multi_simple_queries.sql @@ -132,10 +132,15 @@ SELECT * FROM articles, position('om' in 'Thomas') ORDER BY 2 DESC, 1 DESC, 3 DE SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); -- subqueries are supported in FROM clause + +SET citus.large_table_shard_count TO 1; + SELECT articles.id,test.word_count FROM articles, (SELECT id, word_count FROM articles) AS test WHERE test.id = articles.id ORDER BY articles.id; +RESET citus.large_table_shard_count; + -- subqueries are not supported in SELECT clause SELECT a.title AS name, (SELECT a2.id FROM articles_single_shard a2 WHERE a.id = a2.id LIMIT 1) AS special_price FROM articles a; diff --git a/src/test/regress/sql/multi_subquery.sql b/src/test/regress/sql/multi_subquery.sql index c22f436e7..ea7aeb390 100644 --- a/src/test/regress/sql/multi_subquery.sql +++ b/src/test/regress/sql/multi_subquery.sql @@ -55,38 +55,6 @@ FROM GROUP BY l_orderkey) AS unit_prices; --- this query is only required to execute --- the following query given that recursive planning --- (in general real-time queries in transactions) --- do not execute shard fetch tasks and the next --- query relies on that -SELECT - l_orderkey, - avg(o_totalprice / l_quantity) AS unit_price - FROM - lineitem_subquery, - orders_subquery - WHERE - l_orderkey = o_custkey - GROUP BY - l_orderkey - ORDER BY 2 DESC, 1 DESC - LIMIT 5; - -SELECT - avg(unit_price) -FROM - (SELECT - l_orderkey, - avg(o_totalprice / l_quantity) AS unit_price - FROM - lineitem_subquery, - orders_subquery - WHERE - l_orderkey = o_custkey - GROUP BY - l_orderkey) AS unit_prices; - RESET client_min_messages; -- Subqueries without relation with a volatile functions (non-constant) are planned recursively diff --git a/src/test/regress/sql/multi_subquery_behavioral_analytics.sql b/src/test/regress/sql/multi_subquery_behavioral_analytics.sql index 74ad8d8b7..53b66e0ce 100644 --- a/src/test/regress/sql/multi_subquery_behavioral_analytics.sql +++ b/src/test/regress/sql/multi_subquery_behavioral_analytics.sql @@ -1821,6 +1821,24 @@ FROM ORDER BY 1,2 LIMIT 5; +-- Getting aggregation of value which is created by aggregation in subquery +SELECT + count(1), + avg(agg_value) +FROM + (SELECT + users_table.user_id, + avg(users_table.value_1 / events_table.value_4) AS agg_value + FROM + users_table, + events_table + WHERE + users_table.user_id = events_table.user_id + GROUP BY + 1 + ) AS temp; + + DROP FUNCTION test_join_function_2(integer, integer);