external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to sequential mode on long partition names 

This commit adds support for long partition names for distributed tables:
- ALTER TABLE dist_table ATTACH PARTITION ..
- CREATE TABLE .. PARTITION OF dist_table ..

Note: create_distributed_table UDF does not support long table and
partition names, and is not covered in this commit

(cherry picked from commit 9919fbe3f8)
pull/5110/head
Hanefi Onaldi 2021-02-23 13:32:23 +03:00
parent 11d5d21fd8
commit 6640c76bde
No known key found for this signature in database
GPG Key ID: F18CDB10BA0DFDC7
8 changed files with 183 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/backend/distributed/commands/alter_table.c
View File

@ -1655,8 +1655,23 @@ SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId,
}
char *longestPartitionName = get_rel_name(longestNamePartitionId);
char *longestPartitionShardName = GetLongestShardName(longestNamePartitionId,
longestPartitionName);
char *longestPartitionShardName = NULL;
/*
* Use the shardId values of the partition if it is distributed, otherwise use
* hypothetical values
*/
if (IsCitusTable(longestNamePartitionId) &&
ShardIntervalCount(longestNamePartitionId) > 0)
{
longestPartitionShardName =
GetLongestShardName(longestNamePartitionId, longestPartitionName);
}
else
{
longestPartitionShardName =
GetLongestShardNameForLocalPartition(relationId, longestPartitionName);
}
SwitchToSequentialAndLocalExecutionIfShardNameTooLong(longestPartitionName,
longestPartitionShardName);
@ -1708,3 +1723,16 @@ SwitchToSequentialAndLocalExecutionIfShardNameTooLong(char *relationName,
return false;
}
/*
* SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong is a wrapper for new
* partitions that will be distributed after attaching to a distributed partitioned table
*/
void
SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid parentRelationId,
Oid partitionRelationId)
{
SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(
parentRelationId, get_rel_name(partitionRelationId));
}

6
src/backend/distributed/commands/table.c
View File

@ -324,6 +324,9 @@ PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const
char *parentRelationName = generate_qualified_relation_name(parentRelationId);
bool viaDeprecatedAPI = false;
SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(parentRelationId,
relationId);
CreateDistributedTable(relationId, parentDistributionColumn,
parentDistributionMethod, ShardCount,
parentRelationName, viaDeprecatedAPI);
@ -398,6 +401,9 @@ PostprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement,
char *parentRelationName = generate_qualified_relation_name(relationId);
bool viaDeprecatedAPI = false;
SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(
relationId, partitionRelationId);
CreateDistributedTable(partitionRelationId, distributionColumn,
distributionMethod, ShardCount,
parentRelationName, viaDeprecatedAPI);

66
src/backend/distributed/utils/shard_utils.c
View File

@ -11,11 +11,17 @@
#include "postgres.h"
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
#include "distributed/coordinator_protocol.h"
#include "distributed/metadata_utility.h"
#include "distributed/relay_utility.h"
#include "distributed/shard_utils.h"
static int GetLargestShardId(void);
/*
* GetTableLocalShardOid returns the oid of the shard from the given distributed
* relation with the shardId.
@ -55,3 +61,63 @@ GetLongestShardName(Oid citusTableOid, char *finalRelationName)
return longestShardName;
}
/*
* GetLongestShardNameForLocalPartition is a utility function that creates a hypothetical shard
* name for a partition table that is not distributed yet.
*/
char *
GetLongestShardNameForLocalPartition(Oid parentTableOid, char *partitionRelationName)
{
char *longestShardName = pstrdup(partitionRelationName);
CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(parentTableOid);
int shardIntervalCount = cacheEntry->shardIntervalArrayLength;
int newShardId = GetLargestShardId() + shardIntervalCount;
AppendShardIdToName(&longestShardName, newShardId);
return longestShardName;
}
/*
* GetLargestShardId returns the biggest shard id, and returns a 10^6 in case of failure
* to get the last value from the sequence.
*/
int
GetLargestShardId()
{
Oid savedUserId = InvalidOid;
int savedSecurityContext = 0;
GetUserIdAndSecContext(&savedUserId, &savedSecurityContext);
SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE);
text *sequenceName = cstring_to_text(SHARDID_SEQUENCE_NAME);
Oid sequenceId = ResolveRelationId(sequenceName, false);
Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId);
volatile int64 largestShardId = 0;
/*
* pg_sequence_last_value() returns NULL if the sequence value is not yet used.
* DirectFunctionCall1() gives an ERROR message on NULL return values, and that's why we
* need a PG_TRY block.
*/
PG_TRY();
{
Datum lastShardIdDatum = DirectFunctionCall1(pg_sequence_last_value,
sequenceIdDatum);
largestShardId = DatumGetInt64(lastShardIdDatum);
}
PG_CATCH();
{
/* assume that we have a shardId with 7 digits */
largestShardId = 1000000;
}
PG_END_TRY();
SetUserIdAndSecContext(savedUserId, savedSecurityContext);
return largestShardId;
}

4
src/include/distributed/commands.h
View File

@ -27,6 +27,10 @@ extern bool EnableLocalReferenceForeignKeys;
extern void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId,
char *
finalRelationName);
extern void SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid
parentRelationId,
Oid
partitionRelationId);
/*

2
src/include/distributed/shard_utils.h
View File

@ -15,5 +15,7 @@
extern Oid GetTableLocalShardOid(Oid citusTableOid, uint64 shardId);
extern char * GetLongestShardName(Oid citusTableOid, char *finalRelationName);
extern char * GetLongestShardNameForLocalPartition(Oid parentTableOid,
char *partitionRelationName);
#endif /* SHARD_UTILS_H */

63
src/test/regress/expected/multi_name_lengths.out
View File

@ -186,42 +186,28 @@ NOTICE: identifier "partition_lengths_12345678901234567890123456789012345678901
-- verify that we can rename partitioned tables and partitions with too-long names
ALTER TABLE partition_lengths_12345678901234567890123456789012345678901234567890 RENAME TO partition_lengths;
NOTICE: identifier "partition_lengths_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_123456789012345678901234567890123456789012345"
-- Placeholders for unsupported operations
\set VERBOSITY TERSE
-- renaming distributed table partitions
ALTER TABLE partition_lengths_p2020_09_28 RENAME TO partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890;
NOTICE: identifier "partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_28_123456789012345678901234567890123"
-- creating or attaching new partitions with long names create deadlocks
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890);
-- creating or attaching new partitions with long names
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28);
NOTICE: identifier "partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_29_123456789012345678901234567890123"
NOTICE: identifier "partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_28_123456789012345678901234567890123"
ALTER TABLE partition_lengths
ATTACH PARTITION partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890
FOR VALUES FROM ('2020-09-29 00:00:00') TO ('2020-09-30 00:00:00');
NOTICE: identifier "partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_29_123456789012345678901234567890123"
ERROR: canceling the transaction since it was involved in a distributed deadlock
CREATE TABLE partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-09-30 00:00:00') TO ('2020-10-01 00:00:00');
NOTICE: identifier "partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_30_123456789012345678901234567890123"
ERROR: canceling the transaction since it was involved in a distributed deadlock
CREATE TABLE partition_lengths_p2020_10_01_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-10-01 00:00:00') TO ('2020-10-02 00:00:00');
NOTICE: identifier "partition_lengths_p2020_10_01_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_10_01_123456789012345678901234567890123"
DROP TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890;
NOTICE: identifier "partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_29_123456789012345678901234567890123"
-- creating or attaching new partitions with long names work when using sequential shard modify mode
BEGIN;
SET LOCAL citus.multi_shard_modify_mode = sequential;
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890);
NOTICE: identifier "partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_29_123456789012345678901234567890123"
-- Placeholders for unsupported operations
\set VERBOSITY TERSE
-- renaming distributed table partitions are not supported
ALTER TABLE partition_lengths_p2020_09_28 RENAME TO partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890;
NOTICE: identifier "partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_28_123456789012345678901234567890123"
ALTER TABLE partition_lengths
ATTACH PARTITION partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890
FOR VALUES FROM ('2020-09-29 00:00:00') TO ('2020-09-30 00:00:00');
NOTICE: identifier "partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_29_123456789012345678901234567890123"
CREATE TABLE partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-09-30 00:00:00') TO ('2020-10-01 00:00:00');
NOTICE: identifier "partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890" will be truncated to "partition_lengths_p2020_09_30_123456789012345678901234567890123"
ROLLBACK;
-- renaming distributed table constraints are not supported
ALTER TABLE name_lengths RENAME CONSTRAINT unique_12345678901234567890123456789012345678901234567890 TO unique2_12345678901234567890123456789012345678901234567890;
ERROR: renaming constraints belonging to distributed tables is currently unsupported
@ -311,14 +297,24 @@ SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2');
(1 row)
\c - - :public_worker_1_host :worker_1_port
\di public.sneaky*225030
SELECT c1.relname AS sneaky_index_name,
c2.oid AS sneaky_shard_oid
FROM pg_class c1
JOIN pg_index i ON i.indexrelid = c1.oid
JOIN pg_class c2 ON i.indrelid = c2.oid
WHERE c1.relname LIKE 'sneaky_name_lengths_int_col_%'
AND c2.relname LIKE 'sneaky_name_lengths_%'
AND c1.relkind = 'i'
ORDER BY 1 ASC, 2 ASC
LIMIT 1 \gset
\di :sneaky_index_name
List of relations
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
public | sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225030 | index | postgres | sneaky_name_lengths_225030
public | sneaky_name_lengths_int_col_1234567890123456789_6402d2cd_225026 | index | postgres | sneaky_name_lengths_225026
(1 row)
SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths_225030'::regclass ORDER BY 1 DESC, 2 DESC;
SELECT "Constraint", "Definition" FROM table_checks WHERE relid= :sneaky_shard_oid ORDER BY 1 DESC, 2 DESC;
Constraint | Definition
---------------------------------------------------------------------
checky_12345678901234567890123456789012345678901234567890 | CHECK (int_col_123456789012345678901234567890123456789012345678901234 > 100)
@ -342,11 +338,20 @@ SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash');
(1 row)
\c - - :public_worker_1_host :worker_1_port
\di unique*225032
SELECT c1.relname AS unique_index_name
FROM pg_class c1
JOIN pg_index i ON i.indexrelid = c1.oid
JOIN pg_class c2 ON i.indrelid = c2.oid
WHERE c1.relname LIKE 'unique_123456789%'
AND c2.relname LIKE 'sneaky_name_lengths_%'
AND c1.relkind = 'i'
ORDER BY 1 ASC
LIMIT 1 \gset
\di :unique_index_name
List of relations
Schema | Name | Type | Owner | Table
---------------------------------------------------------------------
public | unique_1234567890123456789012345678901234567890_a5986f27_225032 | index | postgres | sneaky_name_lengths_225032
public | unique_1234567890123456789012345678901234567890_a5986f27_225028 | index | postgres | sneaky_name_lengths_225028
(1 row)
\c - - :master_host :master_port

2
src/test/regress/expected/multi_partitioning.out
View File

@ -621,7 +621,7 @@ SELECT name, type FROM table_attrs WHERE relid = 'partitioning_test_2010'::regcl
-- test add PRIMARY KEY
-- add PRIMARY KEY to partitioned table - this will error out
ALTER TABLE partitioning_test ADD CONSTRAINT partitioning_primary PRIMARY KEY (id);
ERROR: insufficient columns in PRIMARY KEY constraint definition
ERROR: unique constraint on partitioned table must include all partitioning columns
DETAIL: PRIMARY KEY constraint on table "partitioning_test" lacks column "time" which is part of the partition key.
-- ADD PRIMARY KEY to partition
ALTER TABLE partitioning_test_2009 ADD CONSTRAINT partitioning_2009_primary PRIMARY KEY (id);

66
src/test/regress/sql/multi_name_lengths.sql
View File

@ -130,34 +130,26 @@ ALTER TABLE partition_lengths RENAME TO partition_lengths_1234567890123456789012
-- verify that we can rename partitioned tables and partitions with too-long names
ALTER TABLE partition_lengths_12345678901234567890123456789012345678901234567890 RENAME TO partition_lengths;
-- creating or attaching new partitions with long names
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28);
ALTER TABLE partition_lengths
ATTACH PARTITION partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890
FOR VALUES FROM ('2020-09-29 00:00:00') TO ('2020-09-30 00:00:00');
CREATE TABLE partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-09-30 00:00:00') TO ('2020-10-01 00:00:00');
CREATE TABLE partition_lengths_p2020_10_01_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-10-01 00:00:00') TO ('2020-10-02 00:00:00');
DROP TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890;
-- Placeholders for unsupported operations
\set VERBOSITY TERSE
-- renaming distributed table partitions
-- renaming distributed table partitions are not supported
ALTER TABLE partition_lengths_p2020_09_28 RENAME TO partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890;
-- creating or attaching new partitions with long names create deadlocks
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890);
ALTER TABLE partition_lengths
ATTACH PARTITION partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890
FOR VALUES FROM ('2020-09-29 00:00:00') TO ('2020-09-30 00:00:00');
CREATE TABLE partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-09-30 00:00:00') TO ('2020-10-01 00:00:00');
DROP TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890;
-- creating or attaching new partitions with long names work when using sequential shard modify mode
BEGIN;
SET LOCAL citus.multi_shard_modify_mode = sequential;
CREATE TABLE partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890 (LIKE partition_lengths_p2020_09_28_12345678901234567890123456789012345678901234567890);
ALTER TABLE partition_lengths
ATTACH PARTITION partition_lengths_p2020_09_29_12345678901234567890123456789012345678901234567890
FOR VALUES FROM ('2020-09-29 00:00:00') TO ('2020-09-30 00:00:00');
CREATE TABLE partition_lengths_p2020_09_30_12345678901234567890123456789012345678901234567890
PARTITION OF partition_lengths
FOR VALUES FROM ('2020-09-30 00:00:00') TO ('2020-10-01 00:00:00');
ROLLBACK;
-- renaming distributed table constraints are not supported
ALTER TABLE name_lengths RENAME CONSTRAINT unique_12345678901234567890123456789012345678901234567890 TO unique2_12345678901234567890123456789012345678901234567890;
@ -215,8 +207,19 @@ SELECT master_create_distributed_table('sneaky_name_lengths', 'int_col_123456789
SELECT master_create_worker_shards('sneaky_name_lengths', '2', '2');
\c - - :public_worker_1_host :worker_1_port
\di public.sneaky*225030
SELECT "Constraint", "Definition" FROM table_checks WHERE relid='public.sneaky_name_lengths_225030'::regclass ORDER BY 1 DESC, 2 DESC;
SELECT c1.relname AS sneaky_index_name,
c2.oid AS sneaky_shard_oid
FROM pg_class c1
JOIN pg_index i ON i.indexrelid = c1.oid
JOIN pg_class c2 ON i.indrelid = c2.oid
WHERE c1.relname LIKE 'sneaky_name_lengths_int_col_%'
AND c2.relname LIKE 'sneaky_name_lengths_%'
AND c1.relkind = 'i'
ORDER BY 1 ASC, 2 ASC
LIMIT 1 \gset
\di :sneaky_index_name
SELECT "Constraint", "Definition" FROM table_checks WHERE relid= :sneaky_shard_oid ORDER BY 1 DESC, 2 DESC;
\c - - :master_host :master_port
SET citus.shard_count TO 2;
@ -234,7 +237,18 @@ CREATE TABLE sneaky_name_lengths (
SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash');
\c - - :public_worker_1_host :worker_1_port
\di unique*225032
SELECT c1.relname AS unique_index_name
FROM pg_class c1
JOIN pg_index i ON i.indexrelid = c1.oid
JOIN pg_class c2 ON i.indrelid = c2.oid
WHERE c1.relname LIKE 'unique_123456789%'
AND c2.relname LIKE 'sneaky_name_lengths_%'
AND c1.relkind = 'i'
ORDER BY 1 ASC
LIMIT 1 \gset
\di :unique_index_name
\c - - :master_host :master_port
SET citus.shard_count TO 2;