diff --git a/src/backend/distributed/test/shard_rebalancer.c b/src/backend/distributed/test/shard_rebalancer.c index a402792f0..67785c30e 100644 --- a/src/backend/distributed/test/shard_rebalancer.c +++ b/src/backend/distributed/test/shard_rebalancer.c @@ -34,9 +34,12 @@ static List * JsonArrayToShardPlacementTestInfoList( ArrayType *shardPlacementJsonArrayObject); static List * JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject); -static bool JsonFieldValueBool(Datum jsonDocument, const char *key); -static uint32 JsonFieldValueUInt32(Datum jsonDocument, const char *key); -static uint64 JsonFieldValueUInt64(Datum jsonDocument, const char *key); +static bool JsonFieldValueBoolDefault(Datum jsonDocument, const char *key, + bool defaultValue); +static uint32 JsonFieldValueUInt32Default(Datum jsonDocument, const char *key, + uint32 defaultValue); +static uint64 JsonFieldValueUInt64Default(Datum jsonDocument, const char *key, + uint64 defaultValue); static char * JsonFieldValueString(Datum jsonDocument, const char *key); static ArrayType * PlacementUpdateListToJsonArray(List *placementUpdateList); static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *context); @@ -319,12 +322,25 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject) MemoryContext oldContext = MemoryContextSwitchTo(functionCallContext); - uint64 shardId = JsonFieldValueUInt64(placementJson, FIELD_NAME_SHARD_ID); - uint64 shardLength = JsonFieldValueUInt64(placementJson, FIELD_NAME_SHARD_LENGTH); - int shardState = JsonFieldValueUInt32(placementJson, FIELD_NAME_SHARD_STATE); + uint64 shardId = JsonFieldValueUInt64Default( + placementJson, FIELD_NAME_SHARD_ID, placementIndex + 1); + uint64 shardLength = JsonFieldValueUInt64Default( + placementJson, FIELD_NAME_SHARD_LENGTH, 1); + int shardState = JsonFieldValueUInt32Default( + placementJson, FIELD_NAME_SHARD_STATE, SHARD_STATE_ACTIVE); char *nodeName = JsonFieldValueString(placementJson, FIELD_NAME_NODE_NAME); - int nodePort = JsonFieldValueUInt32(placementJson, FIELD_NAME_NODE_PORT); - uint64 placementId = JsonFieldValueUInt64(placementJson, FIELD_NAME_PLACEMENT_ID); + if (nodeName == NULL) + { + ereport(ERROR, (errmsg(FIELD_NAME_NODE_NAME " needs be set"))); + } + int nodePort = JsonFieldValueUInt32Default( + placementJson, FIELD_NAME_NODE_PORT, 5432); + uint64 placementId = JsonFieldValueUInt64Default( + placementJson, FIELD_NAME_PLACEMENT_ID, placementIndex + 1); + + uint64 cost = JsonFieldValueUInt64Default(placementJson, "cost", 1); + bool nextColocationGroup = + JsonFieldValueBoolDefault(placementJson, "next_colocation", false); MemoryContextSwitchTo(oldContext); @@ -335,6 +351,8 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject) placementTestInfo->placement->nodeName = pstrdup(nodeName); placementTestInfo->placement->nodePort = nodePort; placementTestInfo->placement->placementId = placementId; + placementTestInfo->cost = cost; + placementTestInfo->nextColocationGroup = nextColocationGroup; /* * We have copied whatever we needed from the UDF calls, so we can free @@ -345,33 +363,6 @@ JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject) shardPlacementTestInfoList = lappend(shardPlacementTestInfoList, placementTestInfo); - - PG_TRY(); - { - placementTestInfo->cost = JsonFieldValueUInt64(placementJson, - "cost"); - } - PG_CATCH(); - { - /* Ignore errors about not being able to find the key in that case cost is 1 */ - FlushErrorState(); - MemoryContextSwitchTo(oldContext); - placementTestInfo->cost = 1; - } - PG_END_TRY(); - - PG_TRY(); - { - placementTestInfo->nextColocationGroup = JsonFieldValueBool( - placementJson, "next_colocation"); - } - PG_CATCH(); - { - /* Ignore errors about not being able to find the key in that case cost is 1 */ - FlushErrorState(); - MemoryContextSwitchTo(oldContext); - } - PG_END_TRY(); } pfree(shardPlacementJsonArray); @@ -398,11 +389,13 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject) { Datum workerNodeJson = workerNodeJsonArray[workerNodeIndex]; char *workerName = JsonFieldValueString(workerNodeJson, FIELD_NAME_WORKER_NAME); - uint32 workerPort = JsonFieldValueUInt32(workerNodeJson, - FIELD_NAME_WORKER_PORT); + if (workerName == NULL) + { + ereport(ERROR, (errmsg(FIELD_NAME_WORKER_NAME " needs be set"))); + } + uint32 workerPort = JsonFieldValueUInt32Default(workerNodeJson, + FIELD_NAME_WORKER_PORT, 5432); List *disallowedShardIdList = NIL; - char *disallowedShardsString = NULL; - MemoryContext savedContext = CurrentMemoryContext; WorkerTestInfo *workerTestInfo = palloc0(sizeof(WorkerTestInfo)); @@ -415,35 +408,12 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject) workerNode->nodeRole = PrimaryNodeRoleId(); workerTestInfo->node = workerNode; - PG_TRY(); - { - workerTestInfo->capacity = JsonFieldValueUInt64(workerNodeJson, - "capacity"); - } - PG_CATCH(); - { - /* Ignore errors about not being able to find the key in that case capacity is 1 */ - FlushErrorState(); - MemoryContextSwitchTo(savedContext); - workerTestInfo->capacity = 1; - } - PG_END_TRY(); - + workerTestInfo->capacity = JsonFieldValueUInt64Default(workerNodeJson, + "capacity", 1); workerTestInfoList = lappend(workerTestInfoList, workerTestInfo); - PG_TRY(); - { - disallowedShardsString = JsonFieldValueString(workerNodeJson, - "disallowed_shards"); - } - PG_CATCH(); - { - /* Ignore errors about not being able to find the key in that case all shards are allowed */ - FlushErrorState(); - MemoryContextSwitchTo(savedContext); - disallowedShardsString = NULL; - } - PG_END_TRY(); + char *disallowedShardsString = JsonFieldValueString( + workerNodeJson, "disallowed_shards"); if (disallowedShardsString == NULL) { @@ -467,13 +437,18 @@ JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject) /* - * JsonFieldValueBool gets the value of the given key in the given json - * document and returns it as a boolean. + * JsonFieldValueBoolDefault gets the value of the given key in the given json + * document and returns it as a boolean. If the field does not exist in the + * JSON it returns defaultValue. */ static bool -JsonFieldValueBool(Datum jsonDocument, const char *key) +JsonFieldValueBoolDefault(Datum jsonDocument, const char *key, bool defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); + if (valueString == NULL) + { + return defaultValue; + } Datum valueBoolDatum = DirectFunctionCall1(boolin, CStringGetDatum(valueString)); return DatumGetBool(valueBoolDatum); @@ -481,13 +456,18 @@ JsonFieldValueBool(Datum jsonDocument, const char *key) /* - * JsonFieldValueUInt32 gets the value of the given key in the given json - * document and returns it as an unsigned 32-bit integer. + * JsonFieldValueUInt32Default gets the value of the given key in the given json + * document and returns it as an unsigned 32-bit integer. If the field does not + * exist in the JSON it returns defaultValue. */ static uint32 -JsonFieldValueUInt32(Datum jsonDocument, const char *key) +JsonFieldValueUInt32Default(Datum jsonDocument, const char *key, uint32 defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); + if (valueString == NULL) + { + return defaultValue; + } Datum valueInt4Datum = DirectFunctionCall1(int4in, CStringGetDatum(valueString)); uint32 valueUInt32 = DatumGetInt32(valueInt4Datum); @@ -497,12 +477,17 @@ JsonFieldValueUInt32(Datum jsonDocument, const char *key) /* * JsonFieldValueUInt64 gets the value of the given key in the given json - * document and returns it as an unsigned 64-bit integer. + * document and returns it as an unsigned 64-bit integer. If the field does not + * exist in the JSON it returns defaultValue. */ static uint64 -JsonFieldValueUInt64(Datum jsonDocument, const char *key) +JsonFieldValueUInt64Default(Datum jsonDocument, const char *key, uint64 defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); + if (valueString == NULL) + { + return defaultValue; + } Datum valueInt8Datum = DirectFunctionCall1(int8in, CStringGetDatum(valueString)); uint64 valueUInt64 = DatumGetInt64(valueInt8Datum); @@ -510,38 +495,50 @@ JsonFieldValueUInt64(Datum jsonDocument, const char *key) } +/* + * DirectFunctionalCall2Null is a version of DirectFunctionCall2 that can + * return NULL. It still does not support NULL arguments though. + */ +static Datum +DirectFunctionCall2Null(PGFunction func, bool *isnull, Datum arg1, Datum arg2) +{ + LOCAL_FCINFO(fcinfo, 2); + + InitFunctionCallInfoData(*fcinfo, NULL, 2, InvalidOid, NULL, NULL); + + fcinfo->args[0].value = arg1; + fcinfo->args[0].isnull = false; + fcinfo->args[1].value = arg2; + fcinfo->args[1].isnull = false; + + Datum result = (*func)(fcinfo); + if (fcinfo->isnull) + { + *isnull = true; + return 0; + } + + *isnull = false; + return result; +} + + /* * JsonFieldValueString gets the value of the given key in the given json - * document and returns it as a string. + * document and returns it as a string. If the field does not exist in the JSON + * it returns NULL. */ static char * JsonFieldValueString(Datum jsonDocument, const char *key) { - Datum valueTextDatum = 0; - bool valueFetched = false; + bool isnull = false; Datum keyDatum = PointerGetDatum(cstring_to_text(key)); - /* - * json_object_field_text can return NULL, but DirectFunctionalCall2 raises - * cryptic errors when the function returns NULL. We catch this error and - * raise a more meaningful error. - */ - PG_TRY(); + Datum valueTextDatum = DirectFunctionCall2Null( + json_object_field_text, &isnull, jsonDocument, keyDatum); + if (isnull) { - valueTextDatum = DirectFunctionCall2(json_object_field_text, - jsonDocument, keyDatum); - valueFetched = true; - } - PG_CATCH(); - { - FlushErrorState(); - valueFetched = false; - } - PG_END_TRY(); - - if (!valueFetched) - { - ereport(ERROR, (errmsg("could not get value for '%s'", key))); + return NULL; } char *valueString = text_to_cstring(DatumGetTextP(valueTextDatum)); diff --git a/src/test/regress/expected/shard_rebalancer.out b/src/test/regress/expected/shard_rebalancer.out index c3086d166..843d7c170 100644 --- a/src/test/regress/expected/shard_rebalancer.out +++ b/src/test/regress/expected/shard_rebalancer.out @@ -189,14 +189,14 @@ SELECT worker_node_responsive('localhost', 1); -- Check that with threshold=0.0 shard_placement_rebalance_array returns enough -- moves to make the cluster completely balanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1"}', + '{"shardid":5, "nodename":"hostname1"}', + '{"shardid":6, "nodename":"hostname2"}']::json[], 0.0 )); unnest @@ -208,11 +208,11 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that with two nodes and threshold=1.0 shard_placement_rebalance_array -- doesn't return any moves, even if it is completely unbalanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 1.0 )); unnest @@ -222,13 +222,13 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that with three nodes and threshold=1.0 -- shard_placement_rebalance_array returns moves when it is completely unbalanced SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 1.0 )); unnest @@ -240,13 +240,13 @@ SELECT unnest(shard_placement_rebalance_array( -- shard_placement_rebalance_array doesn't return any moves, even if it is -- completely unbalanced. (with three nodes) SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 2.0 )); unnest @@ -256,14 +256,14 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that with threshold=0.0 shard_placement_rebalance_array doesn't return -- any moves if the cluster is already balanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}']::json[], 0.0 )); unnest @@ -273,12 +273,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that shard_placement_replication_array returns a shard copy operation -- for each of the shards in an inactive node. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":1, "nodename":"hostname3"}', + '{"shardid":2, "nodename":"hostname3"}']::json[], 2 )); unnest @@ -290,12 +290,12 @@ SELECT unnest(shard_placement_replication_array( -- Check that shard_placement_replication_array returns a shard copy operation -- for each of the inactive shards. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":3, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":1, "shardstate":3, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "shardstate":3, "nodename":"hostname1"}', + '{"shardid":1, "shardstate":3, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}']::json[], 2 )); unnest @@ -307,17 +307,17 @@ SELECT unnest(shard_placement_replication_array( -- Check that shard_placement_replication_array errors out if all placements of -- a shard are placed on inactive nodes. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":1, "nodename":"hostname3"}']::json[], 2 )); ERROR: could not find a source for shard xxxxx -- Check that shard_placement_replication_array errors out if replication factor -- is more than number of active nodes. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}']::json[], 2 )); ERROR: could not find a target for shard xxxxx diff --git a/src/test/regress/expected/shard_rebalancer_unit.out b/src/test/regress/expected/shard_rebalancer_unit.out index 3308dfc3e..dcbf29df3 100644 --- a/src/test/regress/expected/shard_rebalancer_unit.out +++ b/src/test/regress/expected/shard_rebalancer_unit.out @@ -11,11 +11,11 @@ LANGUAGE C STRICT VOLATILE; -- Check that even with threshold=0.0 shard_placement_rebalance_array returns -- something when there's no completely balanced solution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[] )); unnest --------------------------------------------------------------------- @@ -24,12 +24,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that a node can be drained in a balanced cluster SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); unnest @@ -41,12 +41,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that an already drained node won't be filled again after a second -- rebalance SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); unnest @@ -56,16 +56,16 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that even when shards are already balanced, but shard xxxxx is on a node -- where it is not allowed it will be moved and there will be rebalancing later SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}', - '{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "4"}', - '{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "4"}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}', + '{"node_name": "hostname2", "disallowed_shards": "4"}', + '{"node_name": "hostname3", "disallowed_shards": "4"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname3"}', + '{"shardid":6, "nodename":"hostname3"}' ]::json[] )); unnest @@ -79,16 +79,16 @@ SELECT unnest(shard_placement_rebalance_array( -- moved away from hostname1 and the only shard that is allowed there will be -- moved there SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname3"}', + '{"shardid":6, "nodename":"hostname3"}' ]::json[] )); unnest @@ -100,12 +100,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that an error is returned when a shard is not allowed anywhere SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "2,4"}', - '{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "1,4"}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "2,4"}', + '{"node_name": "hostname2", "disallowed_shards": "1,4"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); WARNING: Not allowed to move shard xxxxx anywhere from hostname2:5432 @@ -117,12 +117,12 @@ WARNING: Not allowed to move shard xxxxx anywhere from hostname2:5432 -- Check that cost is taken into account when rebalancing SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[] )); unnest --------------------------------------------------------------------- @@ -131,13 +131,13 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that cost is taken into account when rebalancing disallowed placements SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[] )); unnest --------------------------------------------------------------------- @@ -149,12 +149,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that node capacacity is taken into account. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1"}']::json[] )); unnest --------------------------------------------------------------------- @@ -170,11 +170,11 @@ SELECT unnest(shard_placement_rebalance_array( -- of 2/3 now. Since load is spread more fairly with utilization 2/3 than 0 it -- should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}']::json[] )); unnest --------------------------------------------------------------------- @@ -187,11 +187,11 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 0 now. Since load is spread more fairly with utilization 2/3 -- than 0 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}']::json[] )); unnest --------------------------------------------------------------------- @@ -205,13 +205,13 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 1. Since load is spread more fairly with utilization 1.5 than -- 1 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 2}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}']::json[] )); unnest --------------------------------------------------------------------- @@ -225,13 +225,13 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 1.5. Since load is spread more fairly with utilization 1.5 -- than 1 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 2}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}']::json[] )); unnest --------------------------------------------------------------------- @@ -239,11 +239,11 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that all shards will be moved to 1 node if its capacity is big enough SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 4}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 4}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[] )); unnest --------------------------------------------------------------------- @@ -254,12 +254,12 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that shards will be moved to a smaller node node if utilization improves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}']::json[] )); unnest --------------------------------------------------------------------- @@ -268,10 +268,10 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that node capacity works with different shard costs SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2", "cost": 3}']::json[] )); unnest --------------------------------------------------------------------- @@ -280,11 +280,11 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that node capacity works with different shard costs again SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[] )); unnest --------------------------------------------------------------------- @@ -294,11 +294,11 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that max_shard_moves works and that we get a NOTICE that it is hit SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[], max_shard_moves := 1 )); NOTICE: Stopped searching before we were out of moves. Please rerun the rebalancer after it's finished for a more optimal placement. @@ -311,11 +311,11 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan -- NOTE: these moves are not optimal, once we implement merging of updates this -- output should change. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 5}', - '{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "1,2"}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432, "cost": 2}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 5}', + '{"node_name": "hostname3", "disallowed_shards": "1,2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname3"}', + '{"shardid":2, "nodename":"hostname3", "cost": 2}']::json[] )); unnest --------------------------------------------------------------------- @@ -326,16 +326,16 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that draining + rebalancing nodes works SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[] )); unnest @@ -347,16 +347,16 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that draining nodes with drain only works SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], drain_only := true )); @@ -367,16 +367,16 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that draining nodes has priority over max_shard_moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], max_shard_moves := 0 )); @@ -389,16 +389,16 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan -- Check that drained moves are counted towards shard moves and thus use up the -- limit when doing normal rebalance moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], max_shard_moves := 2 )); @@ -411,22 +411,22 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan -- Check that draining for all colocation groups is done before rebalancing SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[] )); unnest @@ -442,22 +442,22 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that max_shard_moves warning is only shown once even if more than one -- colocation group its placement updates are ignored because of it SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[], max_shard_moves := 1 )); @@ -471,22 +471,22 @@ NOTICE: Stopped searching before we were out of moves. Please rerun the rebalan -- Check that moves for different colocation groups are added together when -- taking into account max_shard_moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[], max_shard_moves := 5 )); diff --git a/src/test/regress/sql/shard_rebalancer.sql b/src/test/regress/sql/shard_rebalancer.sql index ce6c998a2..ec8751d3b 100644 --- a/src/test/regress/sql/shard_rebalancer.sql +++ b/src/test/regress/sql/shard_rebalancer.sql @@ -124,14 +124,14 @@ SELECT worker_node_responsive('localhost', 1); -- moves to make the cluster completely balanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1"}', + '{"shardid":5, "nodename":"hostname1"}', + '{"shardid":6, "nodename":"hostname2"}']::json[], 0.0 )); @@ -139,24 +139,24 @@ SELECT unnest(shard_placement_rebalance_array( -- doesn't return any moves, even if it is completely unbalanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 1.0 )); -- Check that with three nodes and threshold=1.0 -- shard_placement_rebalance_array returns moves when it is completely unbalanced SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 1.0 )); @@ -167,13 +167,13 @@ SELECT unnest(shard_placement_rebalance_array( SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[], 2.0 )); @@ -181,14 +181,14 @@ SELECT unnest(shard_placement_rebalance_array( -- any moves if the cluster is already balanced. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}']::json[], 0.0 )); @@ -196,12 +196,12 @@ SELECT unnest(shard_placement_rebalance_array( -- for each of the shards in an inactive node. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":1, "nodename":"hostname3"}', + '{"shardid":2, "nodename":"hostname3"}']::json[], 2 )); @@ -209,12 +209,12 @@ SELECT unnest(shard_placement_replication_array( -- for each of the inactive shards. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":3, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":1, "shardstate":3, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "shardstate":3, "nodename":"hostname1"}', + '{"shardid":1, "shardstate":3, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}']::json[], 2 )); @@ -222,9 +222,9 @@ SELECT unnest(shard_placement_replication_array( -- a shard are placed on inactive nodes. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":1, "nodename":"hostname3"}']::json[], 2 )); @@ -232,8 +232,8 @@ SELECT unnest(shard_placement_replication_array( -- is more than number of active nodes. SELECT unnest(shard_placement_replication_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[], + ARRAY['{"node_name": "hostname1"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}']::json[], 2 )); diff --git a/src/test/regress/sql/shard_rebalancer_unit.sql b/src/test/regress/sql/shard_rebalancer_unit.sql index d6159cbd2..8907ad1ec 100644 --- a/src/test/regress/sql/shard_rebalancer_unit.sql +++ b/src/test/regress/sql/shard_rebalancer_unit.sql @@ -14,22 +14,22 @@ LANGUAGE C STRICT VOLATILE; SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[] )); -- Check that a node can be drained in a balanced cluster SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); @@ -37,12 +37,12 @@ SELECT unnest(shard_placement_rebalance_array( -- rebalance SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); @@ -51,16 +51,16 @@ SELECT unnest(shard_placement_rebalance_array( -- where it is not allowed it will be moved and there will be rebalancing later SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}', - '{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "4"}', - '{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "4"}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}', + '{"node_name": "hostname2", "disallowed_shards": "4"}', + '{"node_name": "hostname3", "disallowed_shards": "4"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname3"}', + '{"shardid":6, "nodename":"hostname3"}' ]::json[] )); @@ -69,65 +69,65 @@ SELECT unnest(shard_placement_rebalance_array( -- moved there SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,5,6"}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname3"}', + '{"shardid":6, "nodename":"hostname3"}' ]::json[] )); -- Check that an error is returned when a shard is not allowed anywhere SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "2,4"}', - '{"node_name": "hostname2", "node_port": 5432, "disallowed_shards": "1,4"}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "2,4"}', + '{"node_name": "hostname2", "disallowed_shards": "1,4"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}' ]::json[] )); -- Check that cost is taken into account when rebalancing SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[] )); -- Check that cost is taken into account when rebalancing disallowed placements SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4"}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4"}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1", "cost": 3}']::json[] )); -- Check that node capacacity is taken into account. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}', + '{"shardid":4, "nodename":"hostname1"}']::json[] )); -- Check that shards are not moved when target utilization stays the same and @@ -137,11 +137,11 @@ SELECT unnest(shard_placement_rebalance_array( -- of 2/3 now. Since load is spread more fairly with utilization 2/3 than 0 it -- should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}']::json[] )); @@ -152,11 +152,11 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 0 now. Since load is spread more fairly with utilization 2/3 -- than 0 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}']::json[] )); -- Check that shards are moved even when target utilization stays the same, but @@ -166,13 +166,13 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 1. Since load is spread more fairly with utilization 1.5 than -- 1 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 2}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}']::json[] )); -- Check that shards are moved even when target utilization stays the same, but @@ -182,59 +182,59 @@ SELECT unnest(shard_placement_rebalance_array( -- utilization of 1.5. Since load is spread more fairly with utilization 1.5 -- than 1 it should choose that distribution. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 2}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 2}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}']::json[] )); -- Check that all shards will be moved to 1 node if its capacity is big enough SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 4}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 4}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1"}']::json[] )); -- Check that shards will be moved to a smaller node node if utilization improves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}']::json[] )); -- Check that node capacity works with different shard costs SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432, "cost": 3}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname2"}', + '{"shardid":2, "nodename":"hostname2", "cost": 3}']::json[] )); -- Check that node capacity works with different shard costs again SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[] )); -- Check that max_shard_moves works and that we get a NOTICE that it is hit SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 3}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "cost": 2}']::json[], + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 3}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname1"}', + '{"shardid":3, "nodename":"hostname1", "cost": 2}']::json[], max_shard_moves := 1 )); @@ -243,57 +243,57 @@ SELECT unnest(shard_placement_rebalance_array( -- NOTE: these moves are not optimal, once we implement merging of updates this -- output should change. SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432}', - '{"node_name": "hostname2", "node_port": 5432, "capacity": 5}', - '{"node_name": "hostname3", "node_port": 5432, "disallowed_shards": "1,2"}']::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname3", "nodeport":5432, "cost": 2}']::json[] + ARRAY['{"node_name": "hostname1"}', + '{"node_name": "hostname2", "capacity": 5}', + '{"node_name": "hostname3", "disallowed_shards": "1,2"}']::json[], + ARRAY['{"shardid":1, "nodename":"hostname3"}', + '{"shardid":2, "nodename":"hostname3", "cost": 2}']::json[] )); -- Check that draining + rebalancing nodes works SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[] )); -- Check that draining nodes with drain only works SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], drain_only := true )); -- Check that draining nodes has priority over max_shard_moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], max_shard_moves := 0 )); @@ -301,60 +301,60 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that drained moves are counted towards shard moves and thus use up the -- limit when doing normal rebalance moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}' ]::json[], max_shard_moves := 2 )); -- Check that draining for all colocation groups is done before rebalancing SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[] )); -- Check that max_shard_moves warning is only shown once even if more than one -- colocation group its placement updates are ignored because of it SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[], max_shard_moves := 1 )); @@ -362,22 +362,22 @@ SELECT unnest(shard_placement_rebalance_array( -- Check that moves for different colocation groups are added together when -- taking into account max_shard_moves SELECT unnest(shard_placement_rebalance_array( - ARRAY['{"node_name": "hostname1", "node_port": 5432, "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', - '{"node_name": "hostname2", "node_port": 5432}', - '{"node_name": "hostname3", "node_port": 5432}' + ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,4,5,6,7,8,9,10,11,12", "capacity": 0}', + '{"node_name": "hostname2"}', + '{"node_name": "hostname3"}' ]::json[], - ARRAY['{"placementid":1, "shardid":1, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432}', - '{"placementid":2, "shardid":2, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":3, "shardid":3, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":4, "shardid":4, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":5, "shardid":5, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":6, "shardid":6, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":7, "shardid":7, "shardstate":1, "shardlength":1, "nodename":"hostname1", "nodeport":5432, "next_colocation": true}', - '{"placementid":8, "shardid":8, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":9, "shardid":9, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":10, "shardid":10, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":11, "shardid":11, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}', - '{"placementid":12, "shardid":12, "shardstate":1, "shardlength":1, "nodename":"hostname2", "nodeport":5432}' + ARRAY['{"shardid":1, "nodename":"hostname1"}', + '{"shardid":2, "nodename":"hostname2"}', + '{"shardid":3, "nodename":"hostname2"}', + '{"shardid":4, "nodename":"hostname2"}', + '{"shardid":5, "nodename":"hostname2"}', + '{"shardid":6, "nodename":"hostname2"}', + '{"shardid":7, "nodename":"hostname1", "next_colocation": true}', + '{"shardid":8, "nodename":"hostname2"}', + '{"shardid":9, "nodename":"hostname2"}', + '{"shardid":10, "nodename":"hostname2"}', + '{"shardid":11, "nodename":"hostname2"}', + '{"shardid":12, "nodename":"hostname2"}' ]::json[], max_shard_moves := 5 ));