external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert postgres tables to citus local when creating reference table having fkeys

pull/4539/head
Onur Tirtir 2021-01-22 18:26:23 +03:00
parent 8e02375aa3
commit b5ea033a0b
5 changed files with 514 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/backend/distributed/commands/create_distributed_table.c
View File

@ -44,6 +44,7 @@
#include "distributed/deparser.h"
#include "distributed/distribution_column.h"
#include "distributed/listutils.h"
#include "distributed/local_executor.h"
#include "distributed/metadata_utility.h"
#include "distributed/coordinator_protocol.h"
#include "distributed/metadata/dependency.h"
@ -365,6 +366,40 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
relationId = DropFKeysAndUndistributeTable(relationId);
}
/*
* To support foreign keys between reference tables and local tables,
* we drop & re-define foreign keys at the end of this function so
* that ALTER TABLE hook does the necessary job, which means converting
* local tables to citus local tables to properly support such foreign
* keys.
*
* This function does not expect to create Citus local table, so we blindly
* create reference table when the method is DISTRIBUTE_BY_NONE.
*/
else if (distributionMethod == DISTRIBUTE_BY_NONE &&
ShouldEnableLocalReferenceForeignKeys() &&
HasForeignKeyWithLocalTable(relationId))
{
/*
* Store foreign key creation commands for foreign key relationships
* that relation has with postgres tables.
*/
originalForeignKeyRecreationCommands =
GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
INCLUDE_LOCAL_TABLES);
/*
* Soon we will convert local tables to citus local tables. As
* CreateCitusLocalTable needs to use local execution, now we
* switch to local execution beforehand so that reference table
* creation doesn't use remote execution and we don't error out
* in CreateCitusLocalTable.
*/
SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED);
DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES);
}
/*
* distributed tables might have dependencies on different objects, since we create
* shards for a distributed table via multiple sessions these objects will be created
@ -428,6 +463,10 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
}
else if (distributionMethod == DISTRIBUTE_BY_NONE)
{
/*
* This function does not expect to create Citus local table, so we blindly
* create reference table when the method is DISTRIBUTE_BY_NONE.
*/
CreateReferenceTableShard(relationId);
}
@ -474,7 +513,8 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
* we can skip the validation of the foreign keys.
*/
bool skip_validation = true;
ExecuteForeignKeyCreateCommandList(droppedFKeyCreationCommands, skip_validation);
ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands,
skip_validation);
}

33
src/backend/distributed/commands/foreign_constraint.c
View File

@ -78,6 +78,7 @@ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple,
static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId,
int searchForeignKeyColumnFlags);
static Oid get_relation_constraint_oid_compat(HeapTuple heapTuple);
static List * GetForeignKeysWithLocalTables(Oid relationId);
static bool IsTableTypeIncluded(Oid relationId, int flags);
static void UpdateConstraintIsValid(Oid constraintId, bool isValid);
@ -711,6 +712,38 @@ get_relation_constraint_oid_compat(HeapTuple heapTuple)
}
/*
* HasForeignKeyToLocalTable returns true if relation has foreign key
* relationship with a local table.
*/
bool
HasForeignKeyWithLocalTable(Oid relationId)
{
List *foreignKeysWithLocalTables = GetForeignKeysWithLocalTables(relationId);
return list_length(foreignKeysWithLocalTables) > 0;
}
/*
* GetForeignKeysWithLocalTables returns a list foreign keys for foreign key
* relationaships that relation has with local tables.
*/
static List *
GetForeignKeysWithLocalTables(Oid relationId)
{
int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
INCLUDE_LOCAL_TABLES;
List *referencingFKeyList = GetForeignKeyOids(relationId, referencingFKeysFlag);
/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
EXCLUDE_SELF_REFERENCES |
INCLUDE_LOCAL_TABLES;
List *referencedFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag);
return list_concat(referencingFKeyList, referencedFKeyList);
}
/*
* HasForeignKeyToCitusLocalTable returns true if any of the foreign key constraints
* on the relation with relationId references to a citus local table.

1
src/include/distributed/commands.h
View File

@ -187,6 +187,7 @@ extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId);
extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId);
extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId);
extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags);
extern bool HasForeignKeyWithLocalTable(Oid relationId);
extern bool HasForeignKeyToCitusLocalTable(Oid relationId);
extern bool HasForeignKeyToReferenceTable(Oid relationOid);
extern bool TableReferenced(Oid relationOid);

282
src/test/regress/expected/fkeys_between_local_ref.out
View File

@ -481,6 +481,145 @@ BEGIN;
(2 rows)
ROLLBACK;
-- converting any local table to a citus local table in graph converts
-- other tables to citus local tables, test this in below xact blocks
BEGIN;
SELECT create_reference_table('local_table_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('local_table_2', 'col_1');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
distributed_table | h | c
local_table_1 | n | t
local_table_2 | h | c
local_table_3 | n | c
local_table_4 | n | c
reference_table_1 | n | t
(6 rows)
ROLLBACK;
BEGIN;
SELECT create_reference_table('local_table_4');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('local_table_3');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
distributed_table | h | c
local_table_1 | n | c
local_table_2 | n | c
local_table_3 | n | t
local_table_4 | n | t
reference_table_1 | n | t
(6 rows)
ROLLBACK;
BEGIN;
CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
SELECT create_reference_table('local_table_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('local_table_2', 'col_1');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('local_table_5', 'col_1');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
distributed_table | h | c
local_table_1 | n | t
local_table_2 | h | c
local_table_3 | n | c
local_table_4 | n | c
local_table_5 | h | c
reference_table_1 | n | t
(7 rows)
ROLLBACK;
BEGIN;
ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
-- errors out as foreign keys from reference tables to citus local tables
-- cannot have CASCADE behavior
SELECT create_reference_table('local_table_1');
ERROR: cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
ROLLBACK;
SET citus.enable_local_execution TO OFF;
-- show that this errors out as it tries to convert connected relations to citus
-- local tables and creating citus local table requires local execution but local
-- execution is disabled
SELECT create_reference_table('local_table_1');
ERROR: cannot execute command because a local execution has accessed a placement in the transaction
SET citus.enable_local_execution TO ON;
-- test behavior when outside of the xact block
CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
SELECT create_reference_table('local_table_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('local_table_2', 'col_1');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('local_table_6', 'col_1');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
distributed_table | h | c
local_table_1 | n | t
local_table_2 | h | c
local_table_3 | n | c
local_table_4 | n | c
local_table_6 | h | c
reference_table_1 | n | t
(7 rows)
-- this errors out as we don't support creating citus local
-- tables from partitioned tables
CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@ -493,5 +632,148 @@ ERROR: column "col_99" referenced in foreign key constraint does not exist
-- fails as referenced table does not exist
CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
ERROR: relation "table_does_not_exist" does not exist
-- drop & recreate schema to prevent noise in next test outputs
DROP SCHEMA fkeys_between_local_ref CASCADE;
CREATE SCHEMA fkeys_between_local_ref;
SET search_path TO fkeys_between_local_ref;
-- now have some tests to test behavior before/after enabling foreign keys
-- between local tables & reference tables
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY);
SELECT create_reference_table('ref_2');
create_reference_table
---------------------------------------------------------------------
(1 row)
ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
pg_local_1 | n | c
ref_1 | n | t
ref_2 | n | t
(3 rows)
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
SELECT create_reference_table('ref_2');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
pg_local_1 | n | c
ref_1 | n | t
ref_2 | n | t
(3 rows)
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
SELECT create_reference_table('ref_2');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
pg_local_1 | n | c
ref_1 | n | t
ref_2 | n | t
(3 rows)
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
create_reference_table
---------------------------------------------------------------------
(1 row)
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
-- we still didn't convert local tables to citus local tables
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
ref_1 | n | t
(1 row)
CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
-- pg_local_3 is not connected to other local tables, so we will just
-- convert pg_local_3 to a citus local table
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
pg_local_3 | n | c
ref_1 | n | t
(2 rows)
CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
-- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
-- so we will convert those two local tables to citus local tables too
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
tablename | partmethod | repmodel
---------------------------------------------------------------------
pg_local_1 | n | c
pg_local_2 | n | c
pg_local_3 | n | c
pg_local_4 | n | c
ref_1 | n | t
(5 rows)
ROLLBACK;
-- cleanup at exit
DROP SCHEMA fkeys_between_local_ref CASCADE;

157
src/test/regress/sql/fkeys_between_local_ref.sql
View File

@ -341,6 +341,70 @@ BEGIN;
ORDER BY tablename;
ROLLBACK;
-- converting any local table to a citus local table in graph converts
-- other tables to citus local tables, test this in below xact blocks
BEGIN;
SELECT create_reference_table('local_table_1');
SELECT create_distributed_table('local_table_2', 'col_1');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
SELECT create_reference_table('local_table_4');
SELECT create_reference_table('local_table_3');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
SELECT create_reference_table('local_table_1');
SELECT create_distributed_table('local_table_2', 'col_1');
SELECT create_distributed_table('local_table_5', 'col_1');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
-- errors out as foreign keys from reference tables to citus local tables
-- cannot have CASCADE behavior
SELECT create_reference_table('local_table_1');
ROLLBACK;
SET citus.enable_local_execution TO OFF;
-- show that this errors out as it tries to convert connected relations to citus
-- local tables and creating citus local table requires local execution but local
-- execution is disabled
SELECT create_reference_table('local_table_1');
SET citus.enable_local_execution TO ON;
-- test behavior when outside of the xact block
CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
SELECT create_reference_table('local_table_1');
SELECT create_distributed_table('local_table_2', 'col_1');
SELECT create_distributed_table('local_table_6', 'col_1');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
-- this errors out as we don't support creating citus local
-- tables from partitioned tables
CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@ -352,5 +416,98 @@ CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_
-- fails as referenced table does not exist
CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
-- drop & recreate schema to prevent noise in next test outputs
DROP SCHEMA fkeys_between_local_ref CASCADE;
CREATE SCHEMA fkeys_between_local_ref;
SET search_path TO fkeys_between_local_ref;
-- now have some tests to test behavior before/after enabling foreign keys
-- between local tables & reference tables
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY);
SELECT create_reference_table('ref_2');
ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
SELECT create_reference_table('ref_2');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
SELECT create_reference_table('ref_2');
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
BEGIN;
SET citus.enable_local_reference_table_foreign_keys TO OFF;
CREATE TABLE ref_1(a int PRIMARY KEY);
CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
SELECT create_reference_table('ref_1');
SET citus.enable_local_reference_table_foreign_keys TO ON;
CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
-- we still didn't convert local tables to citus local tables
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
-- pg_local_3 is not connected to other local tables, so we will just
-- convert pg_local_3 to a citus local table
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
-- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
-- so we will convert those two local tables to citus local tables too
SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
ORDER BY tablename;
ROLLBACK;
-- cleanup at exit
DROP SCHEMA fkeys_between_local_ref CASCADE;