From c4a0d55b4586e4a2166f9b6082de20c0c2f4064f Mon Sep 17 00:00:00 2001 From: Nitish Upreti Date: Tue, 12 Jul 2022 13:42:35 -0700 Subject: [PATCH] Limit max shards that can be created with split --- src/backend/distributed/operations/shard_split.c | 10 +++++++++- .../citus_split_shard_by_split_points_negative.out | 9 +++++++++ .../sql/citus_split_shard_by_split_points_negative.sql | 9 +++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/operations/shard_split.c b/src/backend/distributed/operations/shard_split.c index a2ba24df4..3578b00e0 100644 --- a/src/backend/distributed/operations/shard_split.c +++ b/src/backend/distributed/operations/shard_split.c @@ -16,6 +16,7 @@ #include "lib/stringinfo.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "distributed/shared_library_init.h" #include "distributed/adaptive_executor.h" #include "distributed/colocation_utils.h" #include "distributed/metadata_cache.h" @@ -223,7 +224,8 @@ ErrorIfCannotSplitShardExtended(SplitOperation splitOperation, int splitPointsCount = list_length(shardSplitPointsList); int nodeIdsCount = list_length(nodeIdsForPlacementList); - if (nodeIdsCount != splitPointsCount + 1) + int shardsCount = splitPointsCount + 1; + if (nodeIdsCount != shardsCount) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -234,6 +236,12 @@ ErrorIfCannotSplitShardExtended(SplitOperation splitOperation, splitPointsCount))); } + if (shardsCount > MAX_SHARD_COUNT) + { + ereport(ERROR, (errmsg( + "Resulting shard count '%d' with split is greater than max shard count '%d' limit.", + shardsCount, MAX_SHARD_COUNT))); + } Assert(shardIntervalToSplit->minValueExists); Assert(shardIntervalToSplit->maxValueExists); diff --git a/src/test/regress/expected/citus_split_shard_by_split_points_negative.out b/src/test/regress/expected/citus_split_shard_by_split_points_negative.out index d8b006741..148e3a142 100644 --- a/src/test/regress/expected/citus_split_shard_by_split_points_negative.out +++ b/src/test/regress/expected/citus_split_shard_by_split_points_negative.out @@ -83,6 +83,15 @@ SELECT citus_split_shard_by_split_points( ARRAY['-1073741825'], -- Split point equals shard's max value. ARRAY[:worker_1_node, :worker_2_node]); ERROR: Invalid split point -1073741825, as split points should be inclusive. Please use -1073741826 instead. +-- UDF fails if resulting shard count from split greater than MAX_SHARD_COUNT (64000) +-- 64000 split point definee 64000+1 way split (64001 worker nodes needed). +WITH shard_ranges AS (SELECT ((-2147483648 + indx))::text as split_points, :worker_1_node as node_ids FROM generate_series(1,64000) indx ) +SELECT citus_split_shard_by_split_points( + 49761300, + array_agg(split_points), + array_agg(node_ids) || :worker_1_node) --placement node list should exceed split points by one. +FROM shard_ranges; +ERROR: Resulting shard count '64001' with split is greater than max shard count '64000' limit. -- UDF fails where source shard cannot be split further i.e min and max range is equal. -- Create a Shard where range cannot be split further SELECT isolate_tenant_to_new_shard('table_to_split', 1); diff --git a/src/test/regress/sql/citus_split_shard_by_split_points_negative.sql b/src/test/regress/sql/citus_split_shard_by_split_points_negative.sql index bdaf32682..7f310c055 100644 --- a/src/test/regress/sql/citus_split_shard_by_split_points_negative.sql +++ b/src/test/regress/sql/citus_split_shard_by_split_points_negative.sql @@ -74,6 +74,15 @@ SELECT citus_split_shard_by_split_points( ARRAY['-1073741825'], -- Split point equals shard's max value. ARRAY[:worker_1_node, :worker_2_node]); +-- UDF fails if resulting shard count from split greater than MAX_SHARD_COUNT (64000) +-- 64000 split point definee 64000+1 way split (64001 worker nodes needed). +WITH shard_ranges AS (SELECT ((-2147483648 + indx))::text as split_points, :worker_1_node as node_ids FROM generate_series(1,64000) indx ) +SELECT citus_split_shard_by_split_points( + 49761300, + array_agg(split_points), + array_agg(node_ids) || :worker_1_node) --placement node list should exceed split points by one. +FROM shard_ranges; + -- UDF fails where source shard cannot be split further i.e min and max range is equal. -- Create a Shard where range cannot be split further SELECT isolate_tenant_to_new_shard('table_to_split', 1);