external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify code that tests the shard rebalancer algorithm (#4925) 

This modifies the test code to use sane defaults instead of requiring
all values to be specified in the test.
pull/4470/merge
Jelte Fennema 2021-05-03 15:47:19 +02:00 committed by GitHub
parent 23a505d41f
commit 50357db957
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 584 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
src/backend/distributed/test/shard_rebalancer.c
View File

@ -34,9 +34,12 @@
static List * JsonArrayToShardPlacementTestInfoList(
ArrayType *shardPlacementJsonArrayObject);
static List * JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject);
static bool JsonFieldValueBool(Datum jsonDocument, const char *key);
static uint32 JsonFieldValueUInt32(Datum jsonDocument, const char *key);
static uint64 JsonFieldValueUInt64(Datum jsonDocument, const char *key);
static bool JsonFieldValueBoolDefault(Datum jsonDocument, const char *key,
bool defaultValue);
static uint32 JsonFieldValueUInt32Default(Datum jsonDocument, const char *key,
uint32 defaultValue);
static uint64 JsonFieldValueUInt64Default(Datum jsonDocument, const char *key,
uint64 defaultValue);
static char * JsonFieldValueString(Datum jsonDocument, const char *key);
static ArrayType * PlacementUpdateListToJsonArray(List *placementUpdateList);
static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *context);
@ -319,12 +322,25 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject)
MemoryContext oldContext = MemoryContextSwitchTo(functionCallContext);
uint64 shardId = JsonFieldValueUInt64(placementJson, FIELD_NAME_SHARD_ID);
uint64 shardLength = JsonFieldValueUInt64(placementJson, FIELD_NAME_SHARD_LENGTH);
int shardState = JsonFieldValueUInt32(placementJson, FIELD_NAME_SHARD_STATE);
uint64 shardId = JsonFieldValueUInt64Default(
placementJson, FIELD_NAME_SHARD_ID, placementIndex + 1);
uint64 shardLength = JsonFieldValueUInt64Default(
placementJson, FIELD_NAME_SHARD_LENGTH, 1);
int shardState = JsonFieldValueUInt32Default(
placementJson, FIELD_NAME_SHARD_STATE, SHARD_STATE_ACTIVE);
char *nodeName = JsonFieldValueString(placementJson, FIELD_NAME_NODE_NAME);
int nodePort = JsonFieldValueUInt32(placementJson, FIELD_NAME_NODE_PORT);
uint64 placementId = JsonFieldValueUInt64(placementJson, FIELD_NAME_PLACEMENT_ID);
if (nodeName == NULL)
{
ereport(ERROR, (errmsg(FIELD_NAME_NODE_NAME " needs be set")));
}
int nodePort = JsonFieldValueUInt32Default(
placementJson, FIELD_NAME_NODE_PORT, 5432);
uint64 placementId = JsonFieldValueUInt64Default(
placementJson, FIELD_NAME_PLACEMENT_ID, placementIndex + 1);
uint64 cost = JsonFieldValueUInt64Default(placementJson, "cost", 1);
bool nextColocationGroup =
JsonFieldValueBoolDefault(placementJson, "next_colocation", false);
MemoryContextSwitchTo(oldContext);
@ -335,6 +351,8 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject)
placementTestInfo->placement->nodeName = pstrdup(nodeName);
placementTestInfo->placement->nodePort = nodePort;
placementTestInfo->placement->placementId = placementId;
placementTestInfo->cost = cost;
placementTestInfo->nextColocationGroup = nextColocationGroup;
/*
* We have copied whatever we needed from the UDF calls, so we can free
@ -345,33 +363,6 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject)
shardPlacementTestInfoList = lappend(shardPlacementTestInfoList,
placementTestInfo);
PG_TRY();
{
placementTestInfo->cost = JsonFieldValueUInt64(placementJson,
"cost");
}
PG_CATCH();
{
/* Ignore errors about not being able to find the key in that case cost is 1 */
FlushErrorState();
MemoryContextSwitchTo(oldContext);
placementTestInfo->cost = 1;
}
PG_END_TRY();
PG_TRY();
{
placementTestInfo->nextColocationGroup = JsonFieldValueBool(
placementJson, "next_colocation");
}
PG_CATCH();
{
/* Ignore errors about not being able to find the key in that case cost is 1 */
FlushErrorState();
MemoryContextSwitchTo(oldContext);
}
PG_END_TRY();
}
pfree(shardPlacementJsonArray);
@ -398,11 +389,13 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject)
{
Datum workerNodeJson = workerNodeJsonArray[workerNodeIndex];
char *workerName = JsonFieldValueString(workerNodeJson, FIELD_NAME_WORKER_NAME);
uint32 workerPort = JsonFieldValueUInt32(workerNodeJson,
FIELD_NAME_WORKER_PORT);
if (workerName == NULL)
{
ereport(ERROR, (errmsg(FIELD_NAME_WORKER_NAME " needs be set")));
}
uint32 workerPort = JsonFieldValueUInt32Default(workerNodeJson,
FIELD_NAME_WORKER_PORT, 5432);
List *disallowedShardIdList = NIL;
char *disallowedShardsString = NULL;
MemoryContext savedContext = CurrentMemoryContext;
WorkerTestInfo *workerTestInfo = palloc0(sizeof(WorkerTestInfo));
@ -415,35 +408,12 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject)
workerNode->nodeRole = PrimaryNodeRoleId();
workerTestInfo->node = workerNode;
PG_TRY();
{
workerTestInfo->capacity = JsonFieldValueUInt64(workerNodeJson,
"capacity");
}
PG_CATCH();
{
/* Ignore errors about not being able to find the key in that case capacity is 1 */
FlushErrorState();
MemoryContextSwitchTo(savedContext);
workerTestInfo->capacity = 1;
}
PG_END_TRY();
workerTestInfo->capacity = JsonFieldValueUInt64Default(workerNodeJson,
"capacity", 1);
workerTestInfoList = lappend(workerTestInfoList, workerTestInfo);
PG_TRY();
{
disallowedShardsString = JsonFieldValueString(workerNodeJson,
"disallowed_shards");
}
PG_CATCH();
{
/* Ignore errors about not being able to find the key in that case all shards are allowed */
FlushErrorState();
MemoryContextSwitchTo(savedContext);
disallowedShardsString = NULL;
}
PG_END_TRY();
char *disallowedShardsString = JsonFieldValueString(
workerNodeJson, "disallowed_shards");
if (disallowedShardsString == NULL)
{
@ -467,13 +437,18 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject)
/*
* JsonFieldValueBool gets the value of the given key in the given json
* document and returns it as a boolean.
* JsonFieldValueBoolDefault gets the value of the given key in the given json
* document and returns it as a boolean. If the field does not exist in the
* JSON it returns defaultValue.
*/
static bool
JsonFieldValueBool(Datum jsonDocument, const char *key)
JsonFieldValueBoolDefault(Datum jsonDocument, const char *key, bool defaultValue)
{
char *valueString = JsonFieldValueString(jsonDocument, key);
if (valueString == NULL)
{
return defaultValue;
}
Datum valueBoolDatum = DirectFunctionCall1(boolin, CStringGetDatum(valueString));
return DatumGetBool(valueBoolDatum);
@ -481,13 +456,18 @@ JsonFieldValueBool(Datum jsonDocument, const char *key)
/*
* JsonFieldValueUInt32 gets the value of the given key in the given json
* document and returns it as an unsigned 32-bit integer.
* JsonFieldValueUInt32Default gets the value of the given key in the given json
* document and returns it as an unsigned 32-bit integer. If the field does not
* exist in the JSON it returns defaultValue.
*/
static uint32
JsonFieldValueUInt32(Datum jsonDocument, const char *key)
JsonFieldValueUInt32Default(Datum jsonDocument, const char *key, uint32 defaultValue)
{
char *valueString = JsonFieldValueString(jsonDocument, key);
if (valueString == NULL)
{
return defaultValue;
}
Datum valueInt4Datum = DirectFunctionCall1(int4in, CStringGetDatum(valueString));
uint32 valueUInt32 = DatumGetInt32(valueInt4Datum);
@ -497,12 +477,17 @@ JsonFieldValueUInt32(Datum jsonDocument, const char *key)
/*
* JsonFieldValueUInt64 gets the value of the given key in the given json
* document and returns it as an unsigned 64-bit integer.
* document and returns it as an unsigned 64-bit integer. If the field does not
* exist in the JSON it returns defaultValue.
*/
static uint64
JsonFieldValueUInt64(Datum jsonDocument, const char *key)
JsonFieldValueUInt64Default(Datum jsonDocument, const char *key, uint64 defaultValue)
{
char *valueString = JsonFieldValueString(jsonDocument, key);
if (valueString == NULL)
{
return defaultValue;
}
Datum valueInt8Datum = DirectFunctionCall1(int8in, CStringGetDatum(valueString));
uint64 valueUInt64 = DatumGetInt64(valueInt8Datum);
@ -510,38 +495,50 @@ JsonFieldValueUInt64(Datum jsonDocument, const char *key)
}
/*
* DirectFunctionalCall2Null is a version of DirectFunctionCall2 that can
* return NULL. It still does not support NULL arguments though.
*/
static Datum
DirectFunctionCall2Null(PGFunction func, bool *isnull, Datum arg1, Datum arg2)
{
LOCAL_FCINFO(fcinfo, 2);
InitFunctionCallInfoData(*fcinfo, NULL, 2, InvalidOid, NULL, NULL);
fcinfo->args[0].value = arg1;
fcinfo->args[0].isnull = false;
fcinfo->args[1].value = arg2;
fcinfo->args[1].isnull = false;
Datum result = (*func)(fcinfo);
if (fcinfo->isnull)
{
*isnull = true;
return 0;
}
*isnull = false;
return result;
}
/*
* JsonFieldValueString gets the value of the given key in the given json
* document and returns it as a string.
* document and returns it as a string. If the field does not exist in the JSON
* it returns NULL.
*/
static char *
JsonFieldValueString(Datum jsonDocument, const char *key)
{
Datum valueTextDatum = 0;
bool valueFetched = false;
bool isnull = false;
Datum keyDatum = PointerGetDatum(cstring_to_text(key));
/*
* json_object_field_text can return NULL, but DirectFunctionalCall2 raises
* cryptic errors when the function returns NULL. We catch this error and
* raise a more meaningful error.
*/
PG_TRY();
Datum valueTextDatum = DirectFunctionCall2Null(
json_object_field_text, &isnull, jsonDocument, keyDatum);
if (isnull)
{
valueTextDatum = DirectFunctionCall2(json_object_field_text,
jsonDocument, keyDatum);
valueFetched = true;
}
PG_CATCH();
{
FlushErrorState();
valueFetched = false;
}
PG_END_TRY();
if (!valueFetched)
{
ereport(ERROR, (errmsg("could not get value for '%s'", key)));
return NULL;
}
char *valueString = text_to_cstring(DatumGetTextP(valueTextDatum));

100
src/test/regress/expected/shard_rebalancer.out
View File

@ -189,14 +189,14 @@ SELECT worker_node_responsive('localhost', 1);
-- Check that with threshold=0.0 shard_placement_rebalance_array returns enough
-- moves to make the cluster completely balanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1"}',
'{"shardid":5, "nodename":"hostname1"}',
'{"shardid":6, "nodename":"hostname2"}']::json[],
0.0
));
unnest
@ -208,11 +208,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that with two nodes and threshold=1.0 shard_placement_rebalance_array
-- doesn't return any moves, even if it is completely unbalanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
1.0
));
unnest
@ -222,13 +222,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that with three nodes and threshold=1.0
-- shard_placement_rebalance_array returns moves when it is completely unbalanced
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
1.0
));
unnest
@ -240,13 +240,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- shard_placement_rebalance_array doesn't return any moves, even if it is
-- completely unbalanced. (with three nodes)
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
2.0
));
unnest
@ -256,14 +256,14 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that with threshold=0.0 shard_placement_rebalance_array doesn't return
-- any moves if the cluster is already balanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}']::json[],
0.0
));
unnest
@ -273,12 +273,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that shard_placement_replication_array returns a shard copy operation
-- for each of the shards in an inactive node.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":1, "nodename":"hostname3"}',
'{"shardid":2, "nodename":"hostname3"}']::json[],
2
));
unnest
@ -290,12 +290,12 @@ SELECT unnest(shard_placement_replication_array(
-- Check that shard_placement_replication_array returns a shard copy operation
-- for each of the inactive shards.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":3, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":1, "shardstate":3, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "shardstate":3, "nodename":"hostname1"}',
'{"shardid":1, "shardstate":3, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}']::json[],
2
));
unnest
@ -307,17 +307,17 @@ SELECT unnest(shard_placement_replication_array(
-- Check that shard_placement_replication_array errors out if all placements of
-- a shard are placed on inactive nodes.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":1, "nodename":"hostname3"}']::json[],
2
));
ERROR: could not find a source for shard xxxxx
-- Check that shard_placement_replication_array errors out if replication factor
-- is more than number of active nodes.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}']::json[],
2
));
ERROR: could not find a target for shard xxxxx

390
src/test/regress/expected/shard_rebalancer_unit.out
View File

@ -11,11 +11,11 @@ LANGUAGE C STRICT VOLATILE;
-- Check that even with threshold=0.0 shard_placement_rebalance_array returns
-- something when there's no completely balanced solution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -24,12 +24,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that a node can be drained in a balanced cluster
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
unnest
@ -41,12 +41,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that an already drained node won't be filled again after a second
-- rebalance
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
unnest
@ -56,16 +56,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that even when shards are already balanced, but shard xxxxx is on a node
-- where it is not allowed it will be moved and there will be rebalancing later
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "4"}',
'{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "4"}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "disallowed_shards": "4"}',
'{"node_name": "hostname3", "disallowed_shards": "4"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname3"}',
'{"shardid":6, "nodename":"hostname3"}'
]::json[]
));
unnest
@ -79,16 +79,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- moved away from hostname1 and the only shard that is allowed there will be
-- moved there
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname3"}',
'{"shardid":6, "nodename":"hostname3"}'
]::json[]
));
unnest
@ -100,12 +100,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that an error is returned when a shard is not allowed anywhere
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "2,4"}',
'{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "1,4"}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "2,4"}',
'{"node_name": "hostname2", "disallowed_shards": "1,4"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
WARNING: Not allowed to move shard xxxxx anywhere from hostname2:5432
@ -117,12 +117,12 @@ WARNING: Not allowed to move shard xxxxx anywhere from hostname2:5432
-- Check that cost is taken into account when rebalancing
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[]
));
unnest
---------------------------------------------------------------------
@ -131,13 +131,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that cost is taken into account when rebalancing disallowed placements
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[]
));
unnest
---------------------------------------------------------------------
@ -149,12 +149,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that node capacacity is taken into account.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -170,11 +170,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- of 2/3 now. Since load is spread more fairly with utilization 2/3 than 0 it
-- should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -187,11 +187,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 0 now. Since load is spread more fairly with utilization 2/3
-- than 0 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -205,13 +205,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 1. Since load is spread more fairly with utilization 1.5 than
-- 1 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 2}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -225,13 +225,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 1.5. Since load is spread more fairly with utilization 1.5
-- than 1 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 2}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -239,11 +239,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that all shards will be moved to 1 node if its capacity is big enough
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 4}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 4}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -254,12 +254,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that shards will be moved to a smaller node node if utilization improves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}']::json[]
));
unnest
---------------------------------------------------------------------
@ -268,10 +268,10 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that node capacity works with different shard costs
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2", "cost": 3}']::json[]
));
unnest
---------------------------------------------------------------------
@ -280,11 +280,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that node capacity works with different shard costs again
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[]
));
unnest
---------------------------------------------------------------------
@ -294,11 +294,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that max_shard_moves works and that we get a NOTICE that it is hit
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[],
max_shard_moves := 1
));
NOTICE: Stopped searching before we were out of moves. Please rerun the rebalancer after it's finished for a more optimal placement.
@ -311,11 +311,11 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan
-- NOTE: these moves are not optimal, once we implement merging of updates this
-- output should change.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 5}',
'{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "1,2"}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432, "cost": 2}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 5}',
'{"node_name": "hostname3", "disallowed_shards": "1,2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname3"}',
'{"shardid":2, "nodename":"hostname3", "cost": 2}']::json[]
));
unnest
---------------------------------------------------------------------
@ -326,16 +326,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that draining + rebalancing nodes works
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[]
));
unnest
@ -347,16 +347,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that draining nodes with drain only works
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
drain_only := true
));
@ -367,16 +367,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that draining nodes has priority over max_shard_moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 0
));
@ -389,16 +389,16 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan
-- Check that drained moves are counted towards shard moves and thus use up the
-- limit when doing normal rebalance moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 2
));
@ -411,22 +411,22 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan
-- Check that draining for all colocation groups is done before rebalancing
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[]
));
unnest
@ -442,22 +442,22 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that max_shard_moves warning is only shown once even if more than one
-- colocation group its placement updates are ignored because of it
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 1
));
@ -471,22 +471,22 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan
-- Check that moves for different colocation groups are added together when
-- taking into account max_shard_moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 5
));

100
src/test/regress/sql/shard_rebalancer.sql
View File

@ -124,14 +124,14 @@ SELECT worker_node_responsive('localhost', 1);
-- moves to make the cluster completely balanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1"}',
'{"shardid":5, "nodename":"hostname1"}',
'{"shardid":6, "nodename":"hostname2"}']::json[],
0.0
));
@ -139,24 +139,24 @@ SELECT unnest(shard_placement_rebalance_array(
-- doesn't return any moves, even if it is completely unbalanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
1.0
));
-- Check that with three nodes and threshold=1.0
-- shard_placement_rebalance_array returns moves when it is completely unbalanced
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
1.0
));
@ -167,13 +167,13 @@ SELECT unnest(shard_placement_rebalance_array(
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[],
2.0
));
@ -181,14 +181,14 @@ SELECT unnest(shard_placement_rebalance_array(
-- any moves if the cluster is already balanced.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}']::json[],
0.0
));
@ -196,12 +196,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- for each of the shards in an inactive node.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":1, "nodename":"hostname3"}',
'{"shardid":2, "nodename":"hostname3"}']::json[],
2
));
@ -209,12 +209,12 @@ SELECT unnest(shard_placement_replication_array(
-- for each of the inactive shards.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":3, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":1, "shardstate":3, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "shardstate":3, "nodename":"hostname1"}',
'{"shardid":1, "shardstate":3, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}']::json[],
2
));
@ -222,9 +222,9 @@ SELECT unnest(shard_placement_replication_array(
-- a shard are placed on inactive nodes.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":1, "nodename":"hostname3"}']::json[],
2
));
@ -232,8 +232,8 @@ SELECT unnest(shard_placement_replication_array(
-- is more than number of active nodes.
SELECT unnest(shard_placement_replication_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[],
ARRAY['{"node_name": "hostname1"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}']::json[],
2
));

390
src/test/regress/sql/shard_rebalancer_unit.sql
View File

@ -14,22 +14,22 @@ LANGUAGE C STRICT VOLATILE;
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[]
));
-- Check that a node can be drained in a balanced cluster
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
@ -37,12 +37,12 @@ SELECT unnest(shard_placement_rebalance_array(
-- rebalance
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
@ -51,16 +51,16 @@ SELECT unnest(shard_placement_rebalance_array(
-- where it is not allowed it will be moved and there will be rebalancing later
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "4"}',
'{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "4"}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "disallowed_shards": "4"}',
'{"node_name": "hostname3", "disallowed_shards": "4"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname3"}',
'{"shardid":6, "nodename":"hostname3"}'
]::json[]
));
@ -69,65 +69,65 @@ SELECT unnest(shard_placement_rebalance_array(
-- moved there
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname3"}',
'{"shardid":6, "nodename":"hostname3"}'
]::json[]
));
-- Check that an error is returned when a shard is not allowed anywhere
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "2,4"}',
'{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "1,4"}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "2,4"}',
'{"node_name": "hostname2", "disallowed_shards": "1,4"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}'
]::json[]
));
-- Check that cost is taken into account when rebalancing
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[]
));
-- Check that cost is taken into account when rebalancing disallowed placements
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[]
));
-- Check that node capacacity is taken into account.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}',
'{"shardid":4, "nodename":"hostname1"}']::json[]
));
-- Check that shards are not moved when target utilization stays the same and
@ -137,11 +137,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- of 2/3 now. Since load is spread more fairly with utilization 2/3 than 0 it
-- should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}']::json[]
));
@ -152,11 +152,11 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 0 now. Since load is spread more fairly with utilization 2/3
-- than 0 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}']::json[]
));
-- Check that shards are moved even when target utilization stays the same, but
@ -166,13 +166,13 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 1. Since load is spread more fairly with utilization 1.5 than
-- 1 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 2}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}']::json[]
));
-- Check that shards are moved even when target utilization stays the same, but
@ -182,59 +182,59 @@ SELECT unnest(shard_placement_rebalance_array(
-- utilization of 1.5. Since load is spread more fairly with utilization 1.5
-- than 1 it should choose that distribution.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 2}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}']::json[]
));
-- Check that all shards will be moved to 1 node if its capacity is big enough
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 4}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 4}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1"}']::json[]
));
-- Check that shards will be moved to a smaller node node if utilization improves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}']::json[]
));
-- Check that node capacity works with different shard costs
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432, "cost": 3}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname2"}',
'{"shardid":2, "nodename":"hostname2", "cost": 3}']::json[]
));
-- Check that node capacity works with different shard costs again
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[]
));
-- Check that max_shard_moves works and that we get a NOTICE that it is hit
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[],
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 3}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname1"}',
'{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[],
max_shard_moves := 1
));
@ -243,57 +243,57 @@ SELECT unnest(shard_placement_rebalance_array(
-- NOTE: these moves are not optimal, once we implement merging of updates this
-- output should change.
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432}',
'{"node_name": "hostname2", "node_port": 5432, "capacity": 5}',
'{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "1,2"}']::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432, "cost": 2}']::json[]
ARRAY['{"node_name": "hostname1"}',
'{"node_name": "hostname2", "capacity": 5}',
'{"node_name": "hostname3", "disallowed_shards": "1,2"}']::json[],
ARRAY['{"shardid":1, "nodename":"hostname3"}',
'{"shardid":2, "nodename":"hostname3", "cost": 2}']::json[]
));
-- Check that draining + rebalancing nodes works
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[]
));
-- Check that draining nodes with drain only works
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
drain_only := true
));
-- Check that draining nodes has priority over max_shard_moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 0
));
@ -301,60 +301,60 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that drained moves are counted towards shard moves and thus use up the
-- limit when doing normal rebalance moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 2
));
-- Check that draining for all colocation groups is done before rebalancing
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[]
));
-- Check that max_shard_moves warning is only shown once even if more than one
-- colocation group its placement updates are ignored because of it
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 1
));
@ -362,22 +362,22 @@ SELECT unnest(shard_placement_rebalance_array(
-- Check that moves for different colocation groups are added together when
-- taking into account max_shard_moves
SELECT unnest(shard_placement_rebalance_array(
ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2", "node_port": 5432}',
'{"node_name": "hostname3", "node_port": 5432}'
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}',
'{"node_name": "hostname2"}',
'{"node_name": "hostname3"}'
]::json[],
ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}',
'{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}',
'{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}',
'{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}'
ARRAY['{"shardid":1, "nodename":"hostname1"}',
'{"shardid":2, "nodename":"hostname2"}',
'{"shardid":3, "nodename":"hostname2"}',
'{"shardid":4, "nodename":"hostname2"}',
'{"shardid":5, "nodename":"hostname2"}',
'{"shardid":6, "nodename":"hostname2"}',
'{"shardid":7, "nodename":"hostname1", "next_colocation": true}',
'{"shardid":8, "nodename":"hostname2"}',
'{"shardid":9, "nodename":"hostname2"}',
'{"shardid":10, "nodename":"hostname2"}',
'{"shardid":11, "nodename":"hostname2"}',
'{"shardid":12, "nodename":"hostname2"}'
]::json[],
max_shard_moves := 5
));