From 63efc140de3c7d97e77393a36adef7df57b6a93b Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 10 May 2021 00:20:04 +0300 Subject: [PATCH] Create partitions before copying data --- .../distributed/operations/repair_shards.c | 16 +++---- .../expected/partitioning_issue_4949.out | 46 +++++++++++++++++++ src/test/regress/multi_schedule | 4 +- .../regress/sql/partitioning_issue_4949.sql | 44 ++++++++++++++++++ 4 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 src/test/regress/expected/partitioning_issue_4949.out create mode 100644 src/test/regress/sql/partitioning_issue_4949.sql diff --git a/src/backend/distributed/operations/repair_shards.c b/src/backend/distributed/operations/repair_shards.c index b5b30606e..155cb6418 100644 --- a/src/backend/distributed/operations/repair_shards.c +++ b/src/backend/distributed/operations/repair_shards.c @@ -849,6 +849,14 @@ CopyShardTables(List *shardIntervalList, char *sourceNodeName, int32 sourceNodeP List *ddlCommandList = CopyShardCommandList(shardInterval, sourceNodeName, sourceNodePort, includeDataCopy); + + if (PartitionTable(shardInterval->relationId)) + { + char *attachPartitionCommand = + GenerateAttachShardPartitionCommand(shardInterval); + + ddlCommandList = lappend(ddlCommandList, attachPartitionCommand); + } char *tableOwner = TableOwner(shardInterval->relationId); SendCommandListToWorkerInSingleTransaction(targetNodeName, targetNodePort, @@ -878,14 +886,6 @@ CopyShardTables(List *shardIntervalList, char *sourceNodeName, int32 sourceNodeP List *commandList = list_concat(shardForeignConstraintCommandList, referenceTableForeignConstraintList); - if (PartitionTable(shardInterval->relationId)) - { - char *attachPartitionCommand = - GenerateAttachShardPartitionCommand(shardInterval); - - commandList = lappend(commandList, attachPartitionCommand); - } - SendCommandListToWorkerInSingleTransaction(targetNodeName, targetNodePort, tableOwner, commandList); MemoryContextReset(localContext); diff --git a/src/test/regress/expected/partitioning_issue_4949.out b/src/test/regress/expected/partitioning_issue_4949.out new file mode 100644 index 000000000..032986b9c --- /dev/null +++ b/src/test/regress/expected/partitioning_issue_4949.out @@ -0,0 +1,46 @@ +CREATE SCHEMA shard_move_fkeys_indexes; +SET search_path TO shard_move_fkeys_indexes; +SET citus.next_shard_id TO 8970000; +SET citus.next_placement_id TO 8770000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +CREATE TABLE sensors( +measureid integer, +eventdatetime date, +measure_data jsonb, +PRIMARY KEY (measureid, eventdatetime, measure_data)) +PARTITION BY RANGE(eventdatetime); +CREATE TABLE sensors_old PARTITION OF sensors FOR VALUES FROM ('2000-01-01') TO ('2020-01-01'); +CREATE TABLE sensors_2020_01_01 PARTITION OF sensors FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); +CREATE TABLE sensors_news PARTITION OF sensors FOR VALUES FROM ('2020-05-01') TO ('2025-01-01'); +CREATE INDEX index_on_parent ON sensors(measureid); +CREATE INDEX index_on_child ON sensors_2020_01_01(measureid); +SELECT create_distributed_table('sensors', 'measureid'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE colocated_partitioned_table( + measureid integer, + eventdatetime date, + PRIMARY KEY (measureid, eventdatetime)) +PARTITION BY RANGE(eventdatetime); +CREATE TABLE colocated_partitioned_table_2020_01_01 PARTITION OF colocated_partitioned_table FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); +SELECT create_distributed_table('colocated_partitioned_table', 'measureid'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- from child to parent +ALTER TABLE sensors_2020_01_01 ADD CONSTRAINT fkey_from_child_to_parent FOREIGN KEY (measureid,eventdatetime) REFERENCES colocated_partitioned_table(measureid,eventdatetime); +-- load some data +INSERT INTO colocated_partitioned_table SELECT i, '2020-01-05' FROM generate_series(0,1000)i; +INSERT INTO sensors SELECT i, '2020-01-05', '{}' FROM generate_series(0,1000)i; +SELECT citus_move_shard_placement(8970000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); + citus_move_shard_placement +--------------------------------------------------------------------- + +(1 row) + diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 338b14968..1b383fe8a 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -62,8 +62,8 @@ test: ensure_no_intermediate_data_leak # ---------- # Tests for partitioning support # ---------- -test: multi_partitioning_utils multi_partitioning partitioning_issue_3970 replicated_partitioned_table - +test: multi_partitioning_utils multi_partitioning replicated_partitioned_table +test: partitioning_issue_4949 partitioning_issue_3970 # ---------- # Tests for foreign data wrapper support # ---------- diff --git a/src/test/regress/sql/partitioning_issue_4949.sql b/src/test/regress/sql/partitioning_issue_4949.sql new file mode 100644 index 000000000..a6e627374 --- /dev/null +++ b/src/test/regress/sql/partitioning_issue_4949.sql @@ -0,0 +1,44 @@ +CREATE SCHEMA shard_move_fkeys_indexes; +SET search_path TO shard_move_fkeys_indexes; +SET citus.next_shard_id TO 8970000; +SET citus.next_placement_id TO 8770000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; + +CREATE TABLE sensors( +measureid integer, +eventdatetime date, +measure_data jsonb, +PRIMARY KEY (measureid, eventdatetime, measure_data)) +PARTITION BY RANGE(eventdatetime); + +CREATE TABLE sensors_old PARTITION OF sensors FOR VALUES FROM ('2000-01-01') TO ('2020-01-01'); +CREATE TABLE sensors_2020_01_01 PARTITION OF sensors FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); +CREATE TABLE sensors_news PARTITION OF sensors FOR VALUES FROM ('2020-05-01') TO ('2025-01-01'); + +CREATE INDEX index_on_parent ON sensors(measureid); +CREATE INDEX index_on_child ON sensors_2020_01_01(measureid); + +SELECT create_distributed_table('sensors', 'measureid'); + + +CREATE TABLE colocated_partitioned_table( + measureid integer, + eventdatetime date, + PRIMARY KEY (measureid, eventdatetime)) +PARTITION BY RANGE(eventdatetime); + +CREATE TABLE colocated_partitioned_table_2020_01_01 PARTITION OF colocated_partitioned_table FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); +SELECT create_distributed_table('colocated_partitioned_table', 'measureid'); + + + +-- from child to parent +ALTER TABLE sensors_2020_01_01 ADD CONSTRAINT fkey_from_child_to_parent FOREIGN KEY (measureid,eventdatetime) REFERENCES colocated_partitioned_table(measureid,eventdatetime); + +-- load some data +INSERT INTO colocated_partitioned_table SELECT i, '2020-01-05' FROM generate_series(0,1000)i; +INSERT INTO sensors SELECT i, '2020-01-05', '{}' FROM generate_series(0,1000)i; + + +SELECT citus_move_shard_placement(8970000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); \ No newline at end of file