From 28a16beba71abb65a83f289ad57fca0d619c6ab5 Mon Sep 17 00:00:00 2001 From: Metin Doslu Date: Thu, 2 Jun 2016 13:49:33 +0300 Subject: [PATCH] Update only shard length on statistics update for hash-partitioned Update only the shard length on master_update_shard_statistics() call for hash-partitioned tables. Fixes #519. --- .../master/master_stage_protocol.c | 9 +- src/test/regress/input/multi_copy.source | 40 ++++++++ src/test/regress/output/multi_copy.source | 96 +++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/master/master_stage_protocol.c b/src/backend/distributed/master/master_stage_protocol.c index b527f545d..8b11d3733 100644 --- a/src/backend/distributed/master/master_stage_protocol.c +++ b/src/backend/distributed/master/master_stage_protocol.c @@ -447,6 +447,7 @@ UpdateShardStatistics(int64 shardId) ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; char storageType = shardInterval->storageType; + char partitionType = PartitionMethod(relationId); char *shardQualifiedName = NULL; List *shardPlacementList = NIL; ListCell *shardPlacementCell = NULL; @@ -516,8 +517,12 @@ UpdateShardStatistics(int64 shardId) workerName, workerPort); } - DeleteShardRow(shardId); - InsertShardRow(relationId, shardId, storageType, minValue, maxValue); + /* only update shard min/max values for append-partitioned tables */ + if (partitionType == DISTRIBUTE_BY_APPEND) + { + DeleteShardRow(shardId); + InsertShardRow(relationId, shardId, storageType, minValue, maxValue); + } if (QueryCancelPending) { diff --git a/src/test/regress/input/multi_copy.source b/src/test/regress/input/multi_copy.source index daba9d182..425f58514 100644 --- a/src/test/regress/input/multi_copy.source +++ b/src/test/regress/input/multi_copy.source @@ -110,6 +110,15 @@ SELECT count(*) FROM customer_copy_hash; -- Confirm that data was copied SELECT count(*) FROM customer_copy_hash; +-- Make sure that master_update_shard_statistics() only updates shard length for +-- hash-partitioned tables +SELECT master_update_shard_statistics(560000); + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560000; + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560000; + + -- Create a new hash-partitioned table with default now() function CREATE TABLE customer_with_default( c_custkey integer, @@ -195,6 +204,24 @@ FROM customer_copy_range WHERE c_custkey <= 500; -- Check whether data was copied SELECT count(*) FROM customer_copy_range; +-- Manipulate min/max values and check shard statistics for new shard +UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 +WHERE shardid = :new_shard_id; + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; + +-- Update shard statistics for range-partitioned shard and check that only the +-- shard length is updated. +SELECT master_update_shard_statistics(:new_shard_id); + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; + +-- Revert back min/max value updates +UPDATE pg_dist_shard SET shardminvalue = 501, shardmaxvalue = 1000 +WHERE shardid = :new_shard_id; + -- Create a new append-partitioned table into which to COPY CREATE TABLE customer_copy_append ( c_custkey integer, @@ -233,6 +260,19 @@ COPY customer_copy_append(c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv'); -- Check whether data was copied properly SELECT * FROM customer_copy_append; +-- Manipulate manipulate and check shard statistics for append-partitioned table shard +UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardid = 560131; +UPDATE pg_dist_shard_placement SET shardlength = 0 WHERE shardid = 560131; + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; + +-- Update shard statistics for append-partitioned shard +SELECT master_update_shard_statistics(560131); + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; + -- Create lineitem table CREATE TABLE lineitem_copy_append ( l_orderkey bigint not null, diff --git a/src/test/regress/output/multi_copy.source b/src/test/regress/output/multi_copy.source index ff3990b46..c7021550d 100644 --- a/src/test/regress/output/multi_copy.source +++ b/src/test/regress/output/multi_copy.source @@ -126,6 +126,26 @@ SELECT count(*) FROM customer_copy_hash; 2006 (1 row) +-- Make sure that master_update_shard_statistics() only updates shard length for +-- hash-partitioned tables +SELECT master_update_shard_statistics(560000); + master_update_shard_statistics +-------------------------------- + 8192 +(1 row) + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560000; + shardid | shardminvalue | shardmaxvalue +---------+---------------+--------------- + 560000 | -2147483648 | -2080374785 +(1 row) + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560000; + shardid | shardlength +---------+------------- + 560000 | 8192 +(1 row) + -- Create a new hash-partitioned table with default now() function CREATE TABLE customer_with_default( c_custkey integer, @@ -227,6 +247,46 @@ SELECT count(*) FROM customer_copy_range; 1000 (1 row) +-- Manipulate min/max values and check shard statistics for new shard +UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 +WHERE shardid = :new_shard_id; +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; + shardid | shardminvalue | shardmaxvalue +---------+---------------+--------------- + 560129 | 1501 | 2000 +(1 row) + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; + shardid | shardlength +---------+------------- + 560129 | 0 + 560129 | 0 +(2 rows) + +-- Update shard statistics for range-partitioned shard and check that only the +-- shard length is updated. +SELECT master_update_shard_statistics(:new_shard_id); + master_update_shard_statistics +-------------------------------- + 131072 +(1 row) + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; + shardid | shardminvalue | shardmaxvalue +---------+---------------+--------------- + 560129 | 1501 | 2000 +(1 row) + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; + shardid | shardlength +---------+------------- + 560129 | 131072 + 560129 | 131072 +(2 rows) + +-- Revert back min/max value updates +UPDATE pg_dist_shard SET shardminvalue = 501, shardmaxvalue = 1000 +WHERE shardid = :new_shard_id; -- Create a new append-partitioned table into which to COPY CREATE TABLE customer_copy_append ( c_custkey integer, @@ -273,6 +333,42 @@ SELECT * FROM customer_copy_append; 2 | customer2 | | | | | | (2 rows) +-- Manipulate manipulate and check shard statistics for append-partitioned table shard +UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardid = 560131; +UPDATE pg_dist_shard_placement SET shardlength = 0 WHERE shardid = 560131; +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; + shardid | shardminvalue | shardmaxvalue +---------+---------------+--------------- + 560131 | 1501 | 2000 +(1 row) + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; + shardid | shardlength +---------+------------- + 560131 | 0 + 560131 | 0 +(2 rows) + +-- Update shard statistics for append-partitioned shard +SELECT master_update_shard_statistics(560131); + master_update_shard_statistics +-------------------------------- + 8192 +(1 row) + +SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; + shardid | shardminvalue | shardmaxvalue +---------+---------------+--------------- + 560131 | 1 | 2 +(1 row) + +SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; + shardid | shardlength +---------+------------- + 560131 | 8192 + 560131 | 8192 +(2 rows) + -- Create lineitem table CREATE TABLE lineitem_copy_append ( l_orderkey bigint not null,