diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index df105c1e8..a2898736d 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -527,8 +527,8 @@ ConvertTable(TableConversionState *con) * Acquire ExclusiveLock as UndistributeTable does in order to * make sure that no modifications happen on the relations. */ - CascadeOperationForConnectedRelations(con->relationId, ExclusiveLock, - CASCADE_FKEY_UNDISTRIBUTE_TABLE); + CascadeOperationForFkeyConnectedRelations(con->relationId, ExclusiveLock, + CASCADE_FKEY_UNDISTRIBUTE_TABLE); /* * Undistributed every foreign key connected relation in our foreign key diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index 079bc702b..e048c4d64 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -33,9 +33,9 @@ static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList); +static List * GetPartitionRelationIds(List *relationIdList); static void LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode); -static List * RemovePartitionRelationIds(List *relationIdList); -static List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList); +static void ErrorIfConvertingMultiLevelPartitionedTable(List *relationIdList); static void DropRelationIdListForeignKeys(List *relationIdList, int fKeyFlags); static List * GetRelationDropFkeyCommands(Oid relationId, int fKeyFlags); static char * GetDropFkeyCascadeCommand(Oid foreignKeyId); @@ -46,17 +46,14 @@ static void ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation); /* - * CascadeOperationForConnectedRelations executes citus table function specified - * by CascadeOperationType argument on each relation that relation - * with relationId is connected via it's foreign key graph, which includes - * input relation itself. - * Also see CascadeOperationType enum definition for supported - * citus table functions. + * CascadeOperationForFkeyConnectedRelations is a wrapper function which calls + * CascadeOperationForRelationIdList for the foreign key connected relations, for + * the given relationId. */ void -CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE lockMode, - CascadeOperationType - cascadeOperationType) +CascadeOperationForFkeyConnectedRelations(Oid relationId, LOCKMODE lockMode, + CascadeOperationType + cascadeOperationType) { /* * As we will operate on foreign key connected relations, here we @@ -72,7 +69,38 @@ CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE lockMode, return; } - LockRelationsWithLockMode(fKeyConnectedRelationIdList, lockMode); + CascadeOperationForRelationIdList(fKeyConnectedRelationIdList, lockMode, + cascadeOperationType); +} + + +/* + * CascadeOperationForRelationIdList executes citus table function specified + * by CascadeOperationType argument on each relation in the relationIdList; + * Also see CascadeOperationType enum definition for supported + * citus table functions. + */ +void +CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, + CascadeOperationType + cascadeOperationType) +{ + LockRelationsWithLockMode(relationIdList, lockMode); + + if (cascadeOperationType == CASCADE_ADD_LOCAL_TABLE_TO_METADATA) + { + /* + * In CreateCitusLocalTable function, this check would never error out, + * since CreateCitusLocalTable gets called with partition relations, *after* + * they are detached. + * Instead, here, it would error out if the user tries to convert a multi-level + * partitioned table, since partitioned table conversions always go through here. + * Also, there can be a multi-level partitioned table, to be cascaded via foreign + * keys, and they are hard to detect in CreateCitusLocalTable. + * Therefore, we put this check here. + */ + ErrorIfConvertingMultiLevelPartitionedTable(relationIdList); + } /* * Before removing any partition relations, we should error out here if any @@ -81,25 +109,29 @@ CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE lockMode, * We should handle this case here as we remove partition relations in this * function before ExecuteCascadeOperationForRelationIdList. */ - ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(fKeyConnectedRelationIdList); + ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(relationIdList); + + List *partitonRelationList = GetPartitionRelationIds(relationIdList); /* - * We shouldn't cascade through foreign keys on partition tables as citus - * table functions already have their own logics to handle partition relations. + * Here we generate detach/attach commands, if there are any partition tables + * in our "relations-to-cascade" list. */ - List *nonPartitionRelationIdList = - RemovePartitionRelationIds(fKeyConnectedRelationIdList); + List *detachPartitionCommands = + GenerateDetachPartitionCommandRelationIdList(partitonRelationList); + List *attachPartitionCommands = + GenerateAttachPartitionCommandRelationIdList(partitonRelationList); /* * Our foreign key subgraph can have distributed tables which might already * be modified in current transaction. So switch to sequential execution * before executing any ddl's to prevent erroring out later in this function. */ - EnsureSequentialModeForCitusTableCascadeFunction(nonPartitionRelationIdList); + EnsureSequentialModeForCitusTableCascadeFunction(relationIdList); /* store foreign key creation commands before dropping them */ List *fKeyCreationCommands = - GetFKeyCreationCommandsForRelationIdList(nonPartitionRelationIdList); + GetFKeyCreationCommandsForRelationIdList(relationIdList); /* * Note that here we only drop referencing foreign keys for each relation. @@ -107,16 +139,43 @@ CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE lockMode, * relations' referencing foreign keys. */ int fKeyFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; - DropRelationIdListForeignKeys(nonPartitionRelationIdList, fKeyFlags); - ExecuteCascadeOperationForRelationIdList(nonPartitionRelationIdList, + DropRelationIdListForeignKeys(relationIdList, fKeyFlags); + + ExecuteAndLogUtilityCommandList(detachPartitionCommands); + + ExecuteCascadeOperationForRelationIdList(relationIdList, cascadeOperationType); + ExecuteAndLogUtilityCommandList(attachPartitionCommands); + /* now recreate foreign keys on tables */ bool skip_validation = true; ExecuteForeignKeyCreateCommandList(fKeyCreationCommands, skip_validation); } +/* + * GetPartitionRelationIds returns a list of relation id's by picking + * partition relation id's from given relationIdList. + */ +static List * +GetPartitionRelationIds(List *relationIdList) +{ + List *partitionRelationIdList = NIL; + + Oid relationId = InvalidOid; + foreach_oid(relationId, relationIdList) + { + if (PartitionTable(relationId)) + { + partitionRelationIdList = lappend_oid(partitionRelationIdList, relationId); + } + } + + return partitionRelationIdList; +} + + /* * LockRelationsWithLockMode sorts given relationIdList and then acquires * specified lockMode on those relations. @@ -133,6 +192,36 @@ LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode) } +/* + * ErrorIfConvertingMultiLevelPartitionedTable iterates given relationIdList and checks + * if there's a multi-level partitioned table involved or not. As we currently don't + * support converting multi-level partitioned tables into Citus Local Tables, + * this function errors out for such a case. We detect the multi-level partitioned + * table if one of the relations is both partition and partitioned table. + */ +static void +ErrorIfConvertingMultiLevelPartitionedTable(List *relationIdList) +{ + Oid relationId; + foreach_oid(relationId, relationIdList) + { + if (PartitionedTable(relationId) && PartitionTable(relationId)) + { + Oid parentRelId = PartitionParentOid(relationId); + char *parentRelationName = get_rel_name(parentRelId); + char *relationName = get_rel_name(relationId); + + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Citus does not support multi-level " + "partitioned tables"), + errdetail("Relation \"%s\" is partitioned table itself so " + "cannot be partition of relation \"%s\".", + relationName, parentRelationName))); + } + } +} + + /* * ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey searches given * relationIdList for a partition relation involved in a foreign key relationship @@ -167,30 +256,6 @@ ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList) } -/* - * RemovePartitionRelationIds returns a list of relation id's by removing - * partition relation id's from given relationIdList. - */ -static List * -RemovePartitionRelationIds(List *relationIdList) -{ - List *nonPartitionRelationIdList = NIL; - - Oid relationId = InvalidOid; - foreach_oid(relationId, relationIdList) - { - if (PartitionTable(relationId)) - { - continue; - } - - nonPartitionRelationIdList = lappend_oid(nonPartitionRelationIdList, relationId); - } - - return nonPartitionRelationIdList; -} - - /* * EnsureSequentialModeForCitusTableCascadeFunction switches to sequential * execution mode if needed. If it's not possible, then errors out. @@ -247,7 +312,7 @@ RelationIdListHasReferenceTable(List *relationIdList) * GetFKeyCreationCommandsForRelationIdList returns a list of DDL commands to * create foreign keys for each relation in relationIdList. */ -static List * +List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList) { List *fKeyCreationCommands = NIL; @@ -409,7 +474,7 @@ ExecuteCascadeOperationForRelationIdList(List *relationIdList, break; } - case CASCADE_FKEY_ADD_LOCAL_TABLE_TO_METADATA: + case CASCADE_ADD_LOCAL_TABLE_TO_METADATA: { if (!IsCitusTable(relationId)) { diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index 303f2ea3a..8218c68da 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -30,6 +30,7 @@ #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/commands/utility_hook.h" +#include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_sync.h" @@ -47,6 +48,7 @@ static void citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeignKeys); +static void ErrorIfAddingPartitionTableToMetadata(Oid relationId); static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation); static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId); static void ErrorIfUnsupportedCitusLocalColumnDefinition(Relation relation); @@ -223,6 +225,8 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) ErrorIfUnsupportedCreateCitusLocalTable(relation); + ErrorIfAddingPartitionTableToMetadata(relationId); + /* * We immediately close relation with NoLock right after opening it. This is * because, in this function, we may execute ALTER TABLE commands modifying @@ -232,15 +236,35 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) */ relation_close(relation, NoLock); - bool tableHasExternalForeignKeys = TableHasExternalForeignKeys(relationId); - if (tableHasExternalForeignKeys && cascadeViaForeignKeys) + if (TableHasExternalForeignKeys(relationId)) { + if (!cascadeViaForeignKeys) + { + /* + * We do not allow creating citus local table if the table is involved in a + * foreign key relationship with "any other table", unless the option + * cascadeViaForeignKeys is given true. + * Note that we allow self references. + */ + char *qualifiedRelationName = generate_qualified_relation_name(relationId); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("relation %s is involved in a foreign key " + "relationship with another table", + qualifiedRelationName), + errhint("Use cascade_via_foreign_keys option to add " + "all the relations involved in a foreign key " + "relationship with %s to citus metadata by " + "executing SELECT citus_add_local_table_to_metadata($$%s$$, " + "cascade_via_foreign_keys=>true)", + qualifiedRelationName, qualifiedRelationName))); + } + /* * By acquiring AccessExclusiveLock, make sure that no modifications happen * on the relations. */ - CascadeOperationForConnectedRelations(relationId, lockMode, - CASCADE_FKEY_ADD_LOCAL_TABLE_TO_METADATA); + CascadeOperationForFkeyConnectedRelations(relationId, lockMode, + CASCADE_ADD_LOCAL_TABLE_TO_METADATA); /* * We converted every foreign key connected table in our subgraph @@ -248,23 +272,18 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) */ return; } - else if (tableHasExternalForeignKeys) + + if (PartitionedTable(relationId)) { - /* - * We do not allow creating citus local table if the table is involved in a - * foreign key relationship with "any other table". Note that we allow self - * references. - */ - char *qualifiedRelationName = generate_qualified_relation_name(relationId); - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("relation %s is involved in a foreign key " - "relationship with another table", qualifiedRelationName), - errhint("Use cascade_via_foreign_keys option to add " - "all the relations involved in a foreign key " - "relationship with %s to citus metadata by " - "executing SELECT citus_add_local_table_to_metadata($$%s$$, " - "cascade_via_foreign_keys=>true)", - qualifiedRelationName, qualifiedRelationName))); + List *relationList = PartitionList(relationId); + if (list_length(relationList) > 0) + { + relationList = lappend_oid(relationList, relationId); + + CascadeOperationForRelationIdList(relationList, AccessExclusiveLock, + CASCADE_ADD_LOCAL_TABLE_TO_METADATA); + return; + } } ObjectAddress tableAddress = { 0 }; @@ -333,6 +352,88 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) } +/* + * CreateCitusLocalTablePartitionOf generates and executes the necessary commands + * to create a table as partition of a partitioned Citus Local Table. + * The conversion is done by CreateCitusLocalTable. + */ +void +CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, + Oid parentRelationId) +{ + if (createStatement->partspec) + { + /* + * Since partspec represents "PARTITION BY" clause, being different than + * NULL means that given CreateStmt attempts to create a parent table + * at the same time. That means multi-level partitioning within this + * function's context. We don't support this currently. + */ + char *parentRelationName = get_rel_name(parentRelationId); + char *relationName = get_rel_name(relationId); + + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("distributing multi-level partitioned tables " + "is not supported"), + errdetail("Relation \"%s\" is partitioned table itself " + "and it is also partition of relation \"%s\".", + relationName, parentRelationName))); + } + + /* + * Since the shell table for the partition is not created yet on MX workers, + * we should disable DDL propagation before the DETACH command, to avoid + * getting an error on the worker. + */ + List *detachCommands = list_make3(DISABLE_DDL_PROPAGATION, + GenerateDetachPartitionCommand(relationId), + ENABLE_DDL_PROPAGATION); + char *attachCommand = GenerateAlterTableAttachPartitionCommand(relationId); + ExecuteAndLogUtilityCommandList(detachCommands); + int fKeyFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; + + /* + * When cascadeViaForeignKeys is false, CreateCitusLocalTable doesn't expect + * any foreign keys on given relation. Note that we don't want to pass + * cascadeViaForeignKeys to be true here since we don't already allow non-inherited + * foreign keys on child relations, and for the inherited ones, we should have already + * cascaded to the other relations when creating a citus local table from parent. + * + * For this reason, we drop inherited foreign keys here, they'll anyway get created + * again with the attach command + */ + DropRelationForeignKeys(relationId, fKeyFlags); + CreateCitusLocalTable(relationId, false); + ExecuteAndLogUtilityCommand(attachCommand); +} + + +/* + * ErrorIfAddingPartitionTableToMetadata errors out if we try to create the + * citus local table from a partition table. + */ +static void +ErrorIfAddingPartitionTableToMetadata(Oid relationId) +{ + if (PartitionTable(relationId)) + { + /* + * We do not allow converting only partitions into Citus Local Tables. + * Users should call the UDF citus_add_local_table_to_metadata with the + * parent table; then the whole partitioned table will be converted. + */ + char *relationName = get_rel_name(relationId); + Oid parentRelationId = PartitionParentOid(relationId); + char *parentRelationName = get_rel_name(parentRelationId); + ereport(ERROR, (errmsg("cannot add local table %s to metadata since " + "it is a partition of %s. Instead, add the parent " + "table %s to metadata.", + relationName, parentRelationName, + parentRelationName))); + } +} + + /* * ErrorIfUnsupportedCreateCitusLocalTable errors out if we cannot create the * citus local table from the relation. @@ -387,20 +488,14 @@ ErrorIfUnsupportedCitusLocalTableKind(Oid relationId) "relationships", relationName))); } - if (PartitionTable(relationId)) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot add local table \"%s\" to metadata, local tables " - "added to metadata cannot be partition of other tables ", - relationName))); - } - char relationKind = get_rel_relkind(relationId); - if (!(relationKind == RELKIND_RELATION || relationKind == RELKIND_FOREIGN_TABLE)) + if (!(relationKind == RELKIND_RELATION || relationKind == RELKIND_FOREIGN_TABLE || + relationKind == RELKIND_PARTITIONED_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add local table \"%s\" to metadata, only regular " - "tables and foreign tables can be added to citus metadata ", + "tables, partitioned tables and foreign tables" + " can be added to citus metadata ", relationName))); } @@ -948,7 +1043,13 @@ DropDefaultColumnDefinition(Oid relationId, char *columnName) "ALTER TABLE %s ALTER COLUMN %s DROP DEFAULT", qualifiedRelationName, quotedColumnName); - ExecuteAndLogUtilityCommand(sequenceDropCommand->data); + /* + * We need to disable/enable ddl propagation for this command, to prevent + * sending unnecessary ALTER COLUMN commands for partitions, to MX workers. + */ + ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, + sequenceDropCommand->data, + ENABLE_DDL_PROPAGATION)); } @@ -971,7 +1072,15 @@ TransferSequenceOwnership(Oid sequenceId, Oid targetRelationId, char *targetColu qualifiedSequenceName, qualifiedTargetRelationName, quotedTargetColumnName); - ExecuteAndLogUtilityCommand(sequenceOwnershipCommand->data); + /* + * We need to disable/enable ddl propagation for this command, to prevent + * sending unnecessary ALTER SEQUENCE commands for partitions, to MX workers. + * Especially for partitioned tables, where the same sequence is used for + * all partitions, this might cause errors. + */ + ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, + sequenceOwnershipCommand->data, + ENABLE_DDL_PROPAGATION)); } diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 3fa4e2343..383495621 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -78,10 +78,6 @@ static List * GetRelationIdListFromRangeVarList(List *rangeVarList, LOCKMODE loc static bool AlterTableCommandTypeIsTrigger(AlterTableType alterTableType); static bool AlterTableDropsForeignKey(AlterTableStmt *alterTableStatement); static void ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement); -static void ErrorIfCitusLocalTablePartitionCommand(AlterTableCmd *alterTableCmd, - Oid parentRelationId); -static Oid GetPartitionCommandChildRelationId(AlterTableCmd *alterTableCmd, - bool missingOk); static List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, const char *commandString); static bool AlterInvolvesPartitionColumn(AlterTableStmt *alterTableStatement, @@ -360,6 +356,13 @@ PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const */ if (IsCitusTable(parentRelationId)) { + if (IsCitusTableType(parentRelationId, CITUS_LOCAL_TABLE)) + { + CreateCitusLocalTablePartitionOf(createStatement, relationId, + parentRelationId); + return; + } + Var *parentDistributionColumn = DistPartitionKeyOrError(parentRelationId); char parentDistributionMethod = DISTRIBUTE_BY_HASH; char *parentRelationName = generate_qualified_relation_name(parentRelationId); @@ -442,6 +445,24 @@ PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, if (IsCitusTable(relationId) && !IsCitusTable(partitionRelationId)) { + if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) + { + if (PartitionedTable(partitionRelationId)) + { + char *relationName = get_rel_name(partitionRelationId); + char *parentRelationName = get_rel_name(relationId); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("distributing multi-level partitioned " + "tables is not supported"), + errdetail("Relation \"%s\" is partitioned table " + "itself and it is also partition of " + "relation \"%s\".", + relationName, parentRelationName))); + } + CreateCitusLocalTable(partitionRelationId, false); + return NIL; + } + Var *distributionColumn = DistPartitionKeyOrError(relationId); char *distributionColumnName = ColumnToColumnName(relationId, nodeToString( @@ -1113,7 +1134,23 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement PG_TRY(); { bool cascade = true; - CreateCitusLocalTable(relationId, cascade); + + /* + * Withoud this check, we would be erroring out in CreateCitusLocalTable + * for this case anyway. The purpose of this check&error is to provide + * a more meaningful message for the user. + */ + if (PartitionTable(relationId)) + { + ereport(ERROR, (errmsg("cannot build foreign key between" + " reference table and a partition"), + errhint("Try using parent table: %s", + get_rel_name(PartitionParentOid(relationId))))); + } + else + { + CreateCitusLocalTable(relationId, cascade); + } } PG_CATCH(); { @@ -2477,7 +2514,17 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) "separately."))); } - ErrorIfCitusLocalTablePartitionCommand(command, relationId); + if (IsCitusTableType(partitionRelationId, CITUS_LOCAL_TABLE) || + IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) + { + /* + * Citus Local Tables cannot be colocated with other tables. + * If either of two relations is not a Citus Local Table, then we + * don't need to check colocation since CreateCitusLocalTable would + * anyway throw an error. + */ + break; + } if (IsCitusTable(partitionRelationId) && !TablesColocated(relationId, partitionRelationId)) @@ -2521,7 +2568,6 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) "unsupported."))); } #endif - ErrorIfCitusLocalTablePartitionCommand(command, relationId); break; } @@ -2596,52 +2642,6 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) } -/* - * ErrorIfCitusLocalTablePartitionCommand errors out if given alter table subcommand is - * an ALTER TABLE ATTACH / DETACH PARTITION command run for a citus local table. - */ -static void -ErrorIfCitusLocalTablePartitionCommand(AlterTableCmd *alterTableCmd, Oid parentRelationId) -{ - AlterTableType alterTableType = alterTableCmd->subtype; - if (alterTableType != AT_AttachPartition && alterTableType != AT_DetachPartition) - { - return; - } - - bool missingOK = false; - Oid childRelationId = GetPartitionCommandChildRelationId(alterTableCmd, missingOK); - if (!IsCitusTableType(parentRelationId, CITUS_LOCAL_TABLE) && - !IsCitusTableType(childRelationId, CITUS_LOCAL_TABLE)) - { - return; - } - - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot execute ATTACH/DETACH PARTITION command as " - "local tables added to metadata cannot be involved in " - "partition relationships with other tables"))); -} - - -/* - * GetPartitionCommandChildRelationId returns child relationId for given - * ALTER TABLE ATTACH / DETACH PARTITION subcommand. - */ -static Oid -GetPartitionCommandChildRelationId(AlterTableCmd *alterTableCmd, bool missingOk) -{ - AlterTableType alterTableType PG_USED_FOR_ASSERTS_ONLY = alterTableCmd->subtype; - Assert(alterTableType == AT_AttachPartition || alterTableType == AT_DetachPartition); - - PartitionCmd *partitionCommand = (PartitionCmd *) alterTableCmd->def; - RangeVar *childRelationRangeVar = partitionCommand->name; - Oid childRelationId = RangeVarGetRelid(childRelationRangeVar, AccessExclusiveLock, - missingOk); - return childRelationId; -} - - /* * SetupExecutionModeForAlterTable is the function that is responsible * for two things for practical purpose for not doing the same checks diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index cae66049f..990d67ac0 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -44,13 +44,14 @@ #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" /* IWYU pragma: keep */ +#include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/deparse_shard_query.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/maintenanced.h" -#include "distributed/coordinator_protocol.h" +#include "distributed/multi_partitioning_utils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" @@ -713,6 +714,12 @@ UndistributeDisconnectedCitusLocalTables(void) } ReleaseSysCache(heapTuple); + if (PartitionTable(citusLocalTableId)) + { + /* we skip here, we'll undistribute from the parent if necessary */ + continue; + } + if (ConnectedToReferenceTableViaFKey(citusLocalTableId)) { /* still connected to a reference table, skip it */ diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 6ce5859f2..f242a687f 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1716,15 +1716,10 @@ DetachPartitionCommandList(void) } List *partitionList = PartitionList(cacheEntry->relationId); - Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) - { - char *detachPartitionCommand = - GenerateDetachPartitionCommand(partitionRelationId); - - detachPartitionCommandList = lappend(detachPartitionCommandList, - detachPartitionCommand); - } + List *detachCommands = + GenerateDetachPartitionCommandRelationIdList(partitionList); + detachPartitionCommandList = list_concat(detachPartitionCommandList, + detachCommands); } if (list_length(detachPartitionCommandList) == 0) diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index e534cb1af..522717375 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -227,11 +227,13 @@ PlacementAccessTypeToText(ShardPlacementAccessType accessType) static void RecordRelationAccessBase(Oid relationId, ShardPlacementAccessType accessType) { - /* - * We call this only for reference tables, and we don't support partitioned - * reference tables. - */ - Assert(!PartitionedTable(relationId) && !PartitionTable(relationId)); + if (IsCitusTableType(relationId, REFERENCE_TABLE)) + { + /* + * We don't support partitioned reference tables. + */ + Assert(!PartitionedTable(relationId) && !PartitionTable(relationId)); + } /* make sure that this is not a conflicting access */ CheckConflictingRelationAccesses(relationId, accessType); diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index 34737da27..ee6e13571 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -1006,6 +1006,26 @@ GenerateDetachPartitionCommand(Oid partitionTableId) } +/* + * GenerateDetachPartitionCommandRelationIdList returns the necessary command list to + * detach the given partitions from their parents. + */ +List * +GenerateDetachPartitionCommandRelationIdList(List *relationIds) +{ + List *detachPartitionCommands = NIL; + Oid relationId = InvalidOid; + foreach_oid(relationId, relationIds) + { + Assert(PartitionTable(relationId)); + char *detachCommand = GenerateDetachPartitionCommand(relationId); + detachPartitionCommands = lappend(detachPartitionCommands, detachCommand); + } + + return detachPartitionCommands; +} + + /* * GenereatePartitioningInformation returns the partitioning type and partition column * for the given parent table in the form of "PARTITION TYPE (partitioning column(s)/expression(s))". @@ -1102,6 +1122,25 @@ GenerateAlterTableAttachPartitionCommand(Oid partitionTableId) } +/* + * GenerateAttachPartitionCommandRelationIdList returns the necessary command list to + * attach the given partitions to their parents. + */ +List * +GenerateAttachPartitionCommandRelationIdList(List *relationIds) +{ + List *attachPartitionCommands = NIL; + Oid relationId = InvalidOid; + foreach_oid(relationId, relationIds) + { + char *attachCommand = GenerateAlterTableAttachPartitionCommand(relationId); + attachPartitionCommands = lappend(attachPartitionCommands, attachCommand); + } + + return attachPartitionCommands; +} + + /* * This function heaviliy inspired from RelationBuildPartitionDesc() * which is avaliable in src/backend/catalog/partition.c. diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 7b0b2f5f2..448eeeaf6 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -502,7 +502,8 @@ extern Oid GetTriggerFunctionId(Oid triggerId); /* cascade_table_operation_for_connected_relations.c */ /* - * Flags that can be passed to CascadeOperationForConnectedRelations to specify + * Flags that can be passed to CascadeOperationForFkeyConnectedRelations, and + * CascadeOperationForRelationIdList to specify * citus table function to be executed in cascading mode. */ typedef enum CascadeOperationType @@ -513,14 +514,18 @@ typedef enum CascadeOperationType CASCADE_FKEY_UNDISTRIBUTE_TABLE = 1 << 1, /* execute CreateCitusLocalTable on each relation */ - CASCADE_FKEY_ADD_LOCAL_TABLE_TO_METADATA = 1 << 2, + CASCADE_ADD_LOCAL_TABLE_TO_METADATA = 1 << 2, } CascadeOperationType; -extern void CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE relLockMode, - CascadeOperationType - cascadeOperationType); +extern void CascadeOperationForFkeyConnectedRelations(Oid relationId, + LOCKMODE relLockMode, + CascadeOperationType + cascadeOperationType); +extern void CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, + CascadeOperationType cascadeOperationType); extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList); extern bool RelationIdListHasReferenceTable(List *relationIdList); +extern List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList); extern void DropRelationForeignKeys(Oid relationId, int flags); extern void SetLocalEnableLocalReferenceForeignKeys(bool state); extern void ExecuteAndLogUtilityCommandList(List *ddlCommandList); @@ -538,5 +543,7 @@ extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *set /* create_citus_local_table.c */ extern void CreateCitusLocalTable(Oid relationId, bool cascade); +extern void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, + Oid relationId, Oid parentRelationId); #endif /*CITUS_COMMANDS_H */ diff --git a/src/include/distributed/multi_partitioning_utils.h b/src/include/distributed/multi_partitioning_utils.h index bb08fb1c4..f04cbd584 100644 --- a/src/include/distributed/multi_partitioning_utils.h +++ b/src/include/distributed/multi_partitioning_utils.h @@ -22,8 +22,10 @@ extern Oid PartitionParentOid(Oid partitionOid); extern Oid PartitionWithLongestNameRelationId(Oid parentRelationId); extern List * PartitionList(Oid parentRelationId); extern char * GenerateDetachPartitionCommand(Oid partitionTableId); +extern List * GenerateDetachPartitionCommandRelationIdList(List *relationIds); extern char * GenerateAttachShardPartitionCommand(ShardInterval *shardInterval); extern char * GenerateAlterTableAttachPartitionCommand(Oid partitionTableId); +extern List * GenerateAttachPartitionCommandRelationIdList(List *relationIds); extern char * GeneratePartitioningInformation(Oid tableId); extern void FixPartitionConstraintsOnWorkers(Oid relationId); extern void FixLocalPartitionConstraints(Oid relationId, int64 shardId); diff --git a/src/test/regress/expected/citus_local_table_triggers.out b/src/test/regress/expected/citus_local_table_triggers.out index db1512590..546a16dc9 100644 --- a/src/test/regress/expected/citus_local_table_triggers.out +++ b/src/test/regress/expected/citus_local_table_triggers.out @@ -447,7 +447,68 @@ NOTICE: executing the command locally: SELECT value FROM citus_local_table_trig 100 (2 rows) +ROLLBACK; +-- test on partitioned citus local tables +CREATE TABLE par_citus_local_table (val int) PARTITION BY RANGE(val); +CREATE TABLE par_citus_local_table_1 PARTITION OF par_citus_local_table FOR VALUES FROM (1) TO (10000); +CREATE TABLE par_another_citus_local_table (val int unique) PARTITION BY RANGE(val); +CREATE TABLE par_another_citus_local_table_1 PARTITION OF par_another_citus_local_table FOR VALUES FROM (1) TO (10000); +ALTER TABLE par_another_citus_local_table ADD CONSTRAINT fkey_self FOREIGN KEY(val) REFERENCES par_another_citus_local_table(val); +ALTER TABLE par_citus_local_table ADD CONSTRAINT fkey_c_to_c FOREIGN KEY(val) REFERENCES par_another_citus_local_table(val) ON UPDATE CASCADE; +SELECT citus_add_local_table_to_metadata('par_another_citus_local_table', cascade_via_foreign_keys=>true); +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1507011, 'citus_local_table_triggers', 1507012, 'citus_local_table_triggers', 'ALTER TABLE citus_local_table_triggers.par_another_citus_local_table ATTACH PARTITION citus_local_table_triggers.par_another_citus_local_table_1 FOR VALUES FROM (1) TO (10000);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1507013, 'citus_local_table_triggers', 1507014, 'citus_local_table_triggers', 'ALTER TABLE citus_local_table_triggers.par_citus_local_table ATTACH PARTITION citus_local_table_triggers.par_citus_local_table_1 FOR VALUES FROM (1) TO (10000);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1507011, 'citus_local_table_triggers', 1507011, 'citus_local_table_triggers', 'ALTER TABLE citus_local_table_triggers.par_another_citus_local_table ADD CONSTRAINT fkey_self FOREIGN KEY (val) REFERENCES citus_local_table_triggers.par_another_citus_local_table(val)') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1507013, 'citus_local_table_triggers', 1507011, 'citus_local_table_triggers', 'ALTER TABLE citus_local_table_triggers.par_citus_local_table ADD CONSTRAINT fkey_c_to_c FOREIGN KEY (val) REFERENCES citus_local_table_triggers.par_another_citus_local_table(val) ON UPDATE CASCADE') + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE par_reference_table(val int); +SELECT create_reference_table('par_reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE FUNCTION par_insert_100() RETURNS trigger AS $par_insert_100$ +BEGIN + INSERT INTO par_reference_table VALUES (100); + RETURN NEW; +END; +$par_insert_100$ LANGUAGE plpgsql; +BEGIN; + CREATE TRIGGER par_insert_100_trigger + AFTER TRUNCATE ON par_another_citus_local_table + FOR EACH STATEMENT EXECUTE FUNCTION par_insert_100(); + CREATE TRIGGER insert_100_trigger + AFTER TRUNCATE ON par_citus_local_table + FOR EACH STATEMENT EXECUTE FUNCTION par_insert_100(); + TRUNCATE par_another_citus_local_table CASCADE; +NOTICE: truncate cascades to table "par_citus_local_table" +NOTICE: truncate cascades to table "par_citus_local_table_1" +NOTICE: executing the command locally: INSERT INTO citus_local_table_triggers.par_reference_table_1507015 (val) VALUES (100) +NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_triggers.par_another_citus_local_table_xxxxx CASCADE +NOTICE: truncate cascades to table "par_citus_local_table_xxxxx" +NOTICE: truncate cascades to table "par_citus_local_table_1_xxxxx" +NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_triggers.par_another_citus_local_table_1_xxxxx CASCADE +NOTICE: truncate cascades to table "par_citus_local_table_xxxxx" +NOTICE: truncate cascades to table "par_citus_local_table_1_xxxxx" +NOTICE: truncate cascades to table "par_another_citus_local_table_xxxxx" +NOTICE: executing the command locally: INSERT INTO citus_local_table_triggers.par_reference_table_1507015 (val) VALUES (100) +NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_triggers.par_citus_local_table_xxxxx CASCADE +NOTICE: executing the command locally: TRUNCATE TABLE citus_local_table_triggers.par_citus_local_table_1_xxxxx CASCADE + -- we should see two rows with "100" + SELECT * FROM par_reference_table; +NOTICE: executing the command locally: SELECT val FROM citus_local_table_triggers.par_reference_table_1507015 par_reference_table + val +--------------------------------------------------------------------- + 100 + 100 +(2 rows) + ROLLBACK; -- cleanup at exit DROP SCHEMA citus_local_table_triggers, "interesting!schema" CASCADE; -NOTICE: drop cascades to 13 other objects +NOTICE: drop cascades to 20 other objects diff --git a/src/test/regress/expected/citus_local_tables.out b/src/test/regress/expected/citus_local_tables.out index 6ed2c98b1..17e28a12e 100644 --- a/src/test/regress/expected/citus_local_tables.out +++ b/src/test/regress/expected/citus_local_tables.out @@ -123,50 +123,6 @@ SELECT create_distributed_table('distributed_table', 'a'); -- cannot create citus local table from an existing citus table SELECT citus_add_local_table_to_metadata('distributed_table'); ERROR: table "distributed_table" is already distributed --- partitioned table tests -- -CREATE TABLE partitioned_table(a int, b int) PARTITION BY RANGE (a); -CREATE TABLE partitioned_table_1 PARTITION OF partitioned_table FOR VALUES FROM (0) TO (10); -CREATE TABLE partitioned_table_2 PARTITION OF partitioned_table FOR VALUES FROM (10) TO (20); --- cannot create partitioned citus local tables -SELECT citus_add_local_table_to_metadata('partitioned_table'); -ERROR: cannot add local table "partitioned_table" to metadata, only regular tables and foreign tables can be added to citus metadata -BEGIN; - CREATE TABLE citus_local_table PARTITION OF partitioned_table FOR VALUES FROM (20) TO (30); - -- cannot create citus local table as a partition of a local table - SELECT citus_add_local_table_to_metadata('citus_local_table'); -ERROR: cannot add local table "citus_local_table" to metadata, local tables added to metadata cannot be partition of other tables -ROLLBACK; -BEGIN; - CREATE TABLE citus_local_table (a int, b int); - SELECT citus_add_local_table_to_metadata('citus_local_table'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - - -- cannot create citus local table as a partition of a local table - -- via ALTER TABLE commands as well - ALTER TABLE partitioned_table ATTACH PARTITION citus_local_table FOR VALUES FROM (20) TO (30); -ERROR: non-distributed tables cannot have distributed partitions -ROLLBACK; -BEGIN; - SELECT create_distributed_table('partitioned_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - - CREATE TABLE citus_local_table (a int, b int); - SELECT citus_add_local_table_to_metadata('citus_local_table'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - - -- cannot attach citus local table to a partitioned distributed table - ALTER TABLE partitioned_table ATTACH PARTITION citus_local_table FOR VALUES FROM (20) TO (30); -ERROR: cannot execute ATTACH/DETACH PARTITION command as local tables added to metadata cannot be involved in partition relationships with other tables -ROLLBACK; -- show that we do not support inheritance relationships -- CREATE TABLE parent_table (a int, b text); CREATE TABLE child_table () INHERITS (parent_table); @@ -203,11 +159,11 @@ BEGIN; (1 row) INSERT INTO citus_local_table_3 VALUES (1); -NOTICE: executing the command locally: INSERT INTO citus_local_tables_test_schema.citus_local_table_3_1504024 (value) VALUES (1) -NOTICE: executing the command locally: UPDATE citus_local_tables_test_schema.citus_local_table_3_1504024 citus_local_table_3 SET value = (value OPERATOR(pg_catalog.+) 1) +NOTICE: executing the command locally: INSERT INTO citus_local_tables_test_schema.citus_local_table_3_1504010 (value) VALUES (1) +NOTICE: executing the command locally: UPDATE citus_local_tables_test_schema.citus_local_table_3_1504010 citus_local_table_3 SET value = (value OPERATOR(pg_catalog.+) 1) -- show that trigger is executed only once, we should see "2" (not "3") SELECT * FROM citus_local_table_3; -NOTICE: executing the command locally: SELECT value FROM citus_local_tables_test_schema.citus_local_table_3_1504024 citus_local_table_3 +NOTICE: executing the command locally: SELECT value FROM citus_local_tables_test_schema.citus_local_table_3_1504010 citus_local_table_3 value --------------------------------------------------------------------- 2 @@ -334,13 +290,13 @@ ERROR: relation "citus_local_table_1" is a local table SELECT get_shard_id_for_distribution_column('citus_local_table_1', 'not_checking_this_arg_for_non_dist_tables'); get_shard_id_for_distribution_column --------------------------------------------------------------------- - 1504027 + 1504013 (1 row) SELECT get_shard_id_for_distribution_column('citus_local_table_1'); get_shard_id_for_distribution_column --------------------------------------------------------------------- - 1504027 + 1504013 (1 row) -- master_copy_shard_placement is not supported @@ -352,7 +308,7 @@ BEGIN; SELECT undistribute_table('citus_local_table_1'); NOTICE: creating a new table for citus_local_tables_test_schema.citus_local_table_1 NOTICE: moving the data of citus_local_tables_test_schema.citus_local_table_1 -NOTICE: executing the command locally: SELECT a FROM citus_local_tables_test_schema.citus_local_table_1_1504027 citus_local_table_1 +NOTICE: executing the command locally: SELECT a FROM citus_local_tables_test_schema.citus_local_table_1_1504013 citus_local_table_1 NOTICE: dropping the old citus_local_tables_test_schema.citus_local_table_1 NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.citus_local_table_1_xxxxx CASCADE NOTICE: renaming the new table to citus_local_tables_test_schema.citus_local_table_1 @@ -395,7 +351,7 @@ SELECT citus_add_local_table_to_metadata('"CiTUS!LocalTables"."LocalTabLE.1!?!"' -- drop the table before creating it when the search path is set SET search_path to "CiTUS!LocalTables" ; DROP TABLE "LocalTabLE.1!?!"; -NOTICE: executing the command locally: DROP TABLE IF EXISTS "CiTUS!LocalTables"."LocalTabLE.1!?!_1504035" CASCADE +NOTICE: executing the command locally: DROP TABLE IF EXISTS "CiTUS!LocalTables"."LocalTabLE.1!?!_1504021" CASCADE -- have a custom type in the local table CREATE TYPE local_type AS (key int, value jsonb); -- create btree_gist for GiST index @@ -445,10 +401,10 @@ SELECT citus_add_local_table_to_metadata('"LocalTabLE.1!?!9012345678901234567890 -- create some objects after citus_add_local_table_to_metadata CREATE INDEX "my!Index2" ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789"(id) WITH ( fillfactor = 90 ) WHERE id < 20; NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -NOTICE: executing the command locally: CREATE INDEX "my!Index2_1504036" ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) WITH (fillfactor = '90' )WHERE (id < 20) +NOTICE: executing the command locally: CREATE INDEX "my!Index2_1504022" ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504022" USING btree (id ) WITH (fillfactor = '90' )WHERE (id < 20) CREATE UNIQUE INDEX uniqueIndex2 ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789"(id); NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -NOTICE: executing the command locally: CREATE UNIQUE INDEX uniqueindex2_1504036 ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) +NOTICE: executing the command locally: CREATE UNIQUE INDEX uniqueindex2_1504022 ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504022" USING btree (id ) --------------------------------------------------------------------- ---- utility command execution ---- --------------------------------------------------------------------- @@ -537,7 +493,7 @@ CREATE TABLE local_table_4 ( b int references local_table_4(a)); NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_tables_test_schema.local_table_4 ADD CONSTRAINT local_table_4_a_fkey FOREIGN KEY (a) REFERENCES citus_local_tables_test_schema.citus_local_table_1(a)') ALTER TABLE citus_local_table_1 ADD COLUMN b int NOT NULL; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504027, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD COLUMN b int NOT NULL;') +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504013, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD COLUMN b int NOT NULL;') -- show that we added column with NOT NULL SELECT table_name, column_name, is_nullable FROM INFORMATION_SCHEMA.COLUMNS @@ -546,11 +502,11 @@ ORDER BY 1; table_name | column_name | is_nullable --------------------------------------------------------------------- citus_local_table_1 | b | NO - citus_local_table_1_1504027 | b | NO + citus_local_table_1_1504013 | b | NO (2 rows) ALTER TABLE citus_local_table_1 ADD CONSTRAINT unique_a_b UNIQUE (a, b); -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504027, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT unique_a_b UNIQUE (a, b);') +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504013, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT unique_a_b UNIQUE (a, b);') -- show that we defined unique constraints SELECT conrelid::regclass, conname, conkey FROM pg_constraint @@ -558,12 +514,12 @@ WHERE conrelid::regclass::text LIKE 'citus_local_table_1%' AND contype = 'u' ORDER BY 1; conrelid | conname | conkey --------------------------------------------------------------------- - citus_local_table_1_1504027 | unique_a_b_1504027 | {1,2} + citus_local_table_1_1504013 | unique_a_b_1504013 | {1,2} citus_local_table_1 | unique_a_b | {1,2} (2 rows) CREATE UNIQUE INDEX citus_local_table_1_idx ON citus_local_table_1(b); -NOTICE: executing the command locally: CREATE UNIQUE INDEX citus_local_table_1_idx_1504027 ON citus_local_tables_test_schema.citus_local_table_1_1504027 USING btree (b ) +NOTICE: executing the command locally: CREATE UNIQUE INDEX citus_local_table_1_idx_1504013 ON citus_local_tables_test_schema.citus_local_table_1_1504013 USING btree (b ) -- show that we successfully defined the unique index SELECT indexrelid::regclass, indrelid::regclass, indkey FROM pg_index @@ -572,7 +528,7 @@ ORDER BY 1; indexrelid | indrelid | indkey --------------------------------------------------------------------- unique_a_b | citus_local_table_1 | 1 2 - unique_a_b_1504027 | citus_local_table_1_1504027 | 1 2 + unique_a_b_1504013 | citus_local_table_1_1504013 | 1 2 (2 rows) -- test creating citus local table with an index from non-default schema @@ -580,7 +536,7 @@ CREATE SCHEMA "test_\'index_schema"; CREATE TABLE "test_\'index_schema".testindex (a int, b int); CREATE INDEX ind ON "test_\'index_schema".testindex (a); ALTER TABLE "test_\'index_schema".testindex ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1504040, E'test_\\''index_schema', 1504037, 'citus_local_tables_test_schema', E'ALTER TABLE "test_\\''index_schema".testindex ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1504026, E'test_\\''index_schema', 1504023, 'citus_local_tables_test_schema', E'ALTER TABLE "test_\\''index_schema".testindex ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);') SELECT COUNT(*)=2 FROM pg_indexes WHERE tablename LIKE 'testindex%' AND indexname LIKE 'ind%'; ?column? --------------------------------------------------------------------- @@ -605,7 +561,7 @@ DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table, local_ta NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.local_table_4_xxxxx CASCADE NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.reference_table_xxxxx CASCADE NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.local_table_xxxxx CASCADE -NOTICE: drop cascades to constraint fkey_c_to_local_1504027 on table citus_local_tables_test_schema.citus_local_table_1_1504027 +NOTICE: drop cascades to constraint fkey_c_to_local_1504013 on table citus_local_tables_test_schema.citus_local_table_1_1504013 NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.citus_local_table_2_xxxxx CASCADE NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.citus_local_table_1_xxxxx CASCADE -- test some other udf's with citus local tables @@ -793,7 +749,7 @@ CREATE STATISTICS stx1 ON a, b FROM test_citus_local_table_with_stats; ALTER TABLE test_citus_local_table_with_stats ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a); NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE test_citus_local_table_with_stats ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);') CREATE STATISTICS "CiTUS!LocalTables"."Bad\'StatName" ON a, b FROM test_citus_local_table_with_stats; -NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504044, 'citus_local_tables_test_schema', E'CREATE STATISTICS "CiTUS!LocalTables"."Bad\\''StatName" ON a, b FROM citus_local_tables_test_schema.test_citus_local_table_with_stats') +NOTICE: executing the command locally: SELECT worker_apply_shard_ddl_command (1504030, 'citus_local_tables_test_schema', E'CREATE STATISTICS "CiTUS!LocalTables"."Bad\\''StatName" ON a, b FROM citus_local_tables_test_schema.test_citus_local_table_with_stats') SELECT COUNT(*)=4 FROM pg_statistic_ext WHERE stxname LIKE 'stx1%' or stxname LIKE 'Bad\\''StatName%' ; ?column? --------------------------------------------------------------------- @@ -814,6 +770,205 @@ NOTICE: truncate cascades to table "referencing_table_xxxxxxx" NOTICE: executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.referencing_table_xxxxx CASCADE RESET client_min_messages; \set VERBOSITY terse +-- test for partitioned tables +SET client_min_messages TO ERROR; +-- verify we can convert partitioned tables into Citus Local Tables +CREATE TABLE partitioned (user_id int, time timestamp with time zone, data jsonb, PRIMARY KEY (user_id, time )) PARTITION BY RANGE ("time"); +CREATE TABLE partition1 PARTITION OF partitioned FOR VALUES FROM ('2018-04-13 00:00:00+00') TO ('2018-04-14 00:00:00+00'); +CREATE TABLE partition2 PARTITION OF partitioned FOR VALUES FROM ('2018-04-14 00:00:00+00') TO ('2018-04-15 00:00:00+00'); +SELECT citus_add_local_table_to_metadata('partitioned'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- partitions added after the conversion get converted into CLT as well +CREATE TABLE partition3 PARTITION OF partitioned FOR VALUES FROM ('2018-04-15 00:00:00+00') TO ('2018-04-16 00:00:00+00'); +--verify partitioning hierarchy is preserved after conversion +select inhrelid::regclass from pg_inherits where inhparent='partitioned'::regclass order by 1; + inhrelid +--------------------------------------------------------------------- + partition1 + partition2 + partition3 +(3 rows) + +SELECT partition, from_value, to_value, access_method + FROM time_partitions + WHERE partition::text LIKE '%partition%' + ORDER BY partition::text; + partition | from_value | to_value | access_method +--------------------------------------------------------------------- + partition1 | Thu Apr 12 17:00:00 2018 PDT | Fri Apr 13 17:00:00 2018 PDT | heap + partition1_1504031 | Thu Apr 12 17:00:00 2018 PDT | Fri Apr 13 17:00:00 2018 PDT | heap + partition2 | Fri Apr 13 17:00:00 2018 PDT | Sat Apr 14 17:00:00 2018 PDT | heap + partition2_1504032 | Fri Apr 13 17:00:00 2018 PDT | Sat Apr 14 17:00:00 2018 PDT | heap + partition3 | Sat Apr 14 17:00:00 2018 PDT | Sun Apr 15 17:00:00 2018 PDT | heap + partition3_1504034 | Sat Apr 14 17:00:00 2018 PDT | Sun Apr 15 17:00:00 2018 PDT | heap +(6 rows) + +-- undistribute succesfully +SELECT undistribute_table('partitioned'); + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +-- verify the partitioning hierarchy is preserved after undistributing +select inhrelid::regclass from pg_inherits where inhparent='partitioned'::regclass order by 1; + inhrelid +--------------------------------------------------------------------- + partition1 + partition2 + partition3 +(3 rows) + +-- verify table is undistributed +SELECT relname FROM pg_class WHERE relname LIKE 'partition3%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_test_schema') + ORDER BY relname; + relname +--------------------------------------------------------------------- + partition3 + partition3_pkey +(2 rows) + +-- drop successfully +DROP TABLE partitioned; +-- test creating distributed tables from partitioned citus local tables +CREATE TABLE partitioned_distributed (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE partitioned_distributed_1 PARTITION OF partitioned_distributed FOR VALUES FROM (1) TO (4); +CREATE TABLE partitioned_distributed_2 PARTITION OF partitioned_distributed FOR VALUES FROM (5) TO (8); +SELECT citus_add_local_table_to_metadata('partitioned_distributed'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioned_distributed','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\c - - - :worker_1_port +SELECT relname FROM pg_class + WHERE relname LIKE 'partitioned_distributed%' + AND relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_test_schema') + ORDER BY relname; + relname +--------------------------------------------------------------------- + partitioned_distributed_1504038 + partitioned_distributed_1504040 + partitioned_distributed_1_1504042 + partitioned_distributed_1_1504044 + partitioned_distributed_1_a_key_1504042 + partitioned_distributed_1_a_key_1504044 + partitioned_distributed_2_1504046 + partitioned_distributed_2_1504048 + partitioned_distributed_2_a_key_1504046 + partitioned_distributed_2_a_key_1504048 + partitioned_distributed_a_key_1504038 + partitioned_distributed_a_key_1504040 +(12 rows) + +\c - - - :master_port +SET search_path TO citus_local_tables_test_schema; +-- error out if converting multi-level partitioned table +CREATE TABLE multi_par (id text, country text) PARTITION BY RANGE (id); +ALTER TABLE multi_par ADD CONSTRAINT unique_constraint UNIQUE (id, country); +CREATE TABLE multi_par_a_to_i PARTITION OF multi_par FOR VALUES FROM ('a') TO ('j'); +-- multi-level partitioning +CREATE TABLE multi_par_j_to_r PARTITION OF multi_par FOR VALUES FROM ('j') TO ('s') PARTITION BY LIST (country); +CREATE TABLE multi_par_j_to_r_japan PARTITION OF multi_par_j_to_r FOR VALUES IN ('japan'); +-- these two should error out +SELECT citus_add_local_table_to_metadata('multi_par'); +ERROR: Citus does not support multi-level partitioned tables +SELECT citus_add_local_table_to_metadata('multi_par',cascade_via_foreign_keys=>true); +ERROR: Citus does not support multi-level partitioned tables +-- should error out when cascading via fkeys as well +CREATE TABLE cas_1 (a text, b text); +ALTER TABLE cas_1 ADD CONSTRAINT unique_constraint_2 UNIQUE (a, b); +ALTER TABLE cas_1 ADD CONSTRAINT fkey_to_multi_parti FOREIGN KEY (a,b) REFERENCES multi_par(id, country); +SELECT citus_add_local_table_to_metadata('cas_1'); +ERROR: relation citus_local_tables_test_schema.cas_1 is involved in a foreign key relationship with another table +SELECT citus_add_local_table_to_metadata('cas_1', cascade_via_foreign_keys=>true); +ERROR: Citus does not support multi-level partitioned tables +SELECT create_reference_table('cas_1'); +ERROR: Citus does not support multi-level partitioned tables +ALTER TABLE cas_1 DROP CONSTRAINT fkey_to_multi_parti; +SELECT create_reference_table('cas_1'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE cas_1 ADD CONSTRAINT fkey_to_multi_parti FOREIGN KEY (a,b) REFERENCES multi_par(id, country); +ERROR: Citus does not support multi-level partitioned tables +-- undistribute tables to avoid unnecessary log messages later +select undistribute_table('citus_local_table_4', cascade_via_foreign_keys=>true); +NOTICE: creating a new table for citus_local_tables_test_schema.citus_local_table_4 +NOTICE: moving the data of citus_local_tables_test_schema.citus_local_table_4 +NOTICE: dropping the old citus_local_tables_test_schema.citus_local_table_4 +NOTICE: renaming the new table to citus_local_tables_test_schema.citus_local_table_4 + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +select undistribute_table('referencing_table', cascade_via_foreign_keys=>true); +NOTICE: creating a new table for citus_local_tables_test_schema.referencing_table +NOTICE: moving the data of citus_local_tables_test_schema.referencing_table +NOTICE: dropping the old citus_local_tables_test_schema.referencing_table +NOTICE: renaming the new table to citus_local_tables_test_schema.referencing_table +NOTICE: creating a new table for citus_local_tables_test_schema.referenced_table +NOTICE: moving the data of citus_local_tables_test_schema.referenced_table +NOTICE: dropping the old citus_local_tables_test_schema.referenced_table +NOTICE: renaming the new table to citus_local_tables_test_schema.referenced_table + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +-- test dropping fkey +CREATE TABLE parent_2_child_1 (a int); +CREATE TABLE parent_1_child_1 (a int); +CREATE TABLE parent_2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE parent_1 (a INT UNIQUE) PARTITION BY RANGE(a); +alter table parent_1 attach partition parent_1_child_1 default ; +alter table parent_2 attach partition parent_2_child_1 default ; +CREATE TABLE ref_table(a int unique); +alter table parent_1 add constraint fkey_test_drop foreign key(a) references ref_table(a); +alter table parent_2 add constraint fkey1 foreign key(a) references ref_table(a); +alter table parent_1 add constraint fkey2 foreign key(a) references parent_2(a); +SELECT create_reference_table('ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +alter table parent_1 drop constraint fkey_test_drop; +select count(*) from pg_constraint where conname = 'fkey_test_drop'; + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- verify we still preserve the child-parent hierarchy after all conversions +-- check the shard partition +select inhrelid::regclass from pg_inherits where (select inhparent::regclass::text) ~ '^parent_1_\d{7}$' order by 1; + inhrelid +--------------------------------------------------------------------- + parent_1_child_1_1190085 +(1 row) + +-- check the shell partition +select inhrelid::regclass from pg_inherits where inhparent='parent_1'::regclass order by 1; + inhrelid +--------------------------------------------------------------------- + parent_1_child_1 +(1 row) + -- cleanup at exit +SET client_min_messages TO ERROR; DROP SCHEMA citus_local_tables_test_schema, "CiTUS!LocalTables", "test_\'index_schema" CASCADE; -NOTICE: drop cascades to 27 other objects diff --git a/src/test/regress/expected/citus_local_tables_mx.out b/src/test/regress/expected/citus_local_tables_mx.out index fc509e6be..b125c7ff0 100644 --- a/src/test/regress/expected/citus_local_tables_mx.out +++ b/src/test/regress/expected/citus_local_tables_mx.out @@ -239,6 +239,412 @@ SELECT stxname FROM pg_statistic_ext ORDER BY stxname; \c - - - :master_port SET search_path TO citus_local_tables_mx; +-- undistribute old tables to prevent unnecessary undistribute logs later +SELECT undistribute_table('citus_local_table', cascade_via_foreign_keys=>true); +NOTICE: creating a new table for citus_local_tables_mx.citus_local_table +NOTICE: moving the data of citus_local_tables_mx.citus_local_table +NOTICE: dropping the old citus_local_tables_mx.citus_local_table +NOTICE: renaming the new table to citus_local_tables_mx.citus_local_table + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +SELECT undistribute_table('citus_local_table_3', cascade_via_foreign_keys=>true); +NOTICE: creating a new table for citus_local_tables_mx.citus_local_table_3 +NOTICE: moving the data of citus_local_tables_mx.citus_local_table_3 +NOTICE: dropping the old citus_local_tables_mx.citus_local_table_3 +NOTICE: renaming the new table to citus_local_tables_mx.citus_local_table_3 +NOTICE: creating a new table for citus_local_tables_mx.citus_local_table_4 +NOTICE: moving the data of citus_local_tables_mx.citus_local_table_4 +NOTICE: dropping the old citus_local_tables_mx.citus_local_table_4 +NOTICE: renaming the new table to citus_local_tables_mx.citus_local_table_4 + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +SELECT undistribute_table('citus_local_table_stats', cascade_via_foreign_keys=>true); +NOTICE: creating a new table for citus_local_tables_mx.citus_local_table_stats +NOTICE: moving the data of citus_local_tables_mx.citus_local_table_stats +NOTICE: dropping the old citus_local_tables_mx.citus_local_table_stats +NOTICE: renaming the new table to citus_local_tables_mx.citus_local_table_stats + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +-- verify that mx nodes have the shell table for partitioned citus local tables +CREATE TABLE local_table_fkey(a INT UNIQUE); +CREATE TABLE local_table_fkey2(a INT UNIQUE); +CREATE TABLE partitioned_mx (a INT UNIQUE) PARTITION BY RANGE(a); +ALTER TABLE partitioned_mx ADD CONSTRAINT fkey_to_local FOREIGN KEY (a) REFERENCES local_table_fkey(a); +CREATE TABLE IF NOT EXISTS partitioned_mx_1 PARTITION OF partitioned_mx FOR VALUES FROM (1) TO (4); +CREATE TABLE partitioned_mx_2 (a INT UNIQUE); +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_2 FOR VALUES FROM (5) TO (8); +SELECT create_reference_table('local_table_fkey'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioned_mx_3 (a INT UNIQUE); +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_3 FOR VALUES FROM (9) TO (12); +-- these should error out since multi-level partitioned citus local tables are not supported +CREATE TABLE IF NOT EXISTS partitioned_mx_4 PARTITION OF partitioned_mx FOR VALUES FROM (13) TO (16) PARTITION BY RANGE (a); +ERROR: distributing multi-level partitioned tables is not supported +BEGIN; + CREATE TABLE partitioned_mx_4(a INT UNIQUE) PARTITION BY RANGE (a); + alter table partitioned_mx attach partition partitioned_mx_4 FOR VALUES FROM (13) TO (16); +ERROR: distributing multi-level partitioned tables is not supported +END; +CREATE TABLE multi_level_p (a INT UNIQUE) PARTITION BY RANGE (a); +CREATE TABLE IF NOT EXISTS multi_level_c PARTITION OF multi_level_p FOR VALUES FROM (13) TO (16) PARTITION BY RANGE (a); +select citus_add_local_table_to_metadata('multi_level_p'); --errors +ERROR: Citus does not support multi-level partitioned tables +select citus_add_local_table_to_metadata('multi_level_p', true); --errors +ERROR: Citus does not support multi-level partitioned tables +-- try attaching a partition with an external foreign key +CREATE TABLE partitioned_mx_4 (a INT UNIQUE); +ALTER TABLE partitioned_mx_4 ADD CONSTRAINT fkey_not_inherited FOREIGN KEY (a) REFERENCES local_table_fkey2(a); +-- these two should error out +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_4 FOR VALUES FROM (13) TO (16); +ERROR: relation citus_local_tables_mx.partitioned_mx_4 is involved in a foreign key relationship with another table +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_4 default; +ERROR: relation citus_local_tables_mx.partitioned_mx_4 is involved in a foreign key relationship with another table +SELECT + nmsp_parent.nspname AS parent_schema, + parent.relname AS parent, + nmsp_child.nspname AS child_schema, + child.relname AS child +FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace +WHERE parent.relname='partitioned_mx' +ORDER BY child; + parent_schema | parent | child_schema | child +--------------------------------------------------------------------- + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_1 + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_2 + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_3 +(3 rows) + +\c - - - :worker_1_port +SELECT relname FROM pg_class WHERE relname LIKE 'partitioned_mx%' ORDER BY relname; + relname +--------------------------------------------------------------------- + partitioned_mx + partitioned_mx_1 + partitioned_mx_1_a_key + partitioned_mx_2 + partitioned_mx_2_a_key + partitioned_mx_3 + partitioned_mx_3_a_key + partitioned_mx_a_key +(8 rows) + +SELECT + nmsp_parent.nspname AS parent_schema, + parent.relname AS parent, + nmsp_child.nspname AS child_schema, + child.relname AS child +FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace +WHERE parent.relname='partitioned_mx' +ORDER BY child; + parent_schema | parent | child_schema | child +--------------------------------------------------------------------- + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_1 + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_2 + citus_local_tables_mx | partitioned_mx | citus_local_tables_mx | partitioned_mx_3 +(3 rows) + +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +SET client_min_messages TO ERROR; +DROP TABLE partitioned_mx; +RESET client_min_messages; +-- test cascading via foreign keys +CREATE TABLE cas_1 (a INT UNIQUE); +CREATE TABLE cas_par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par_1 PARTITION OF cas_par FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par_2 PARTITION OF cas_par FOR VALUES FROM (5) TO (8); +ALTER TABLE cas_par_1 ADD CONSTRAINT fkey_cas_test_1 FOREIGN KEY (a) REFERENCES cas_1(a); +CREATE TABLE cas_par2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par2_1 PARTITION OF cas_par2 FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par2_2 PARTITION OF cas_par2 FOR VALUES FROM (5) TO (8); +ALTER TABLE cas_par2_1 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES cas_1(a); +CREATE TABLE cas_par3 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par3_1 PARTITION OF cas_par3 FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par3_2 PARTITION OF cas_par3 FOR VALUES FROM (5) TO (8); +-- these two should error out as we should call the conversion from the parent +SELECT citus_add_local_table_to_metadata('cas_par2_2'); +ERROR: cannot add local table cas_par2_2 to metadata since it is a partition of cas_par2. Instead, add the parent table cas_par2 to metadata. +SELECT citus_add_local_table_to_metadata('cas_par2_2', cascade_via_foreign_keys=>true); +ERROR: cannot add local table cas_par2_2 to metadata since it is a partition of cas_par2. Instead, add the parent table cas_par2 to metadata. +-- these two should error out as the foreign keys are not inherited from the parent +SELECT citus_add_local_table_to_metadata('cas_par2'); +ERROR: cannot cascade operation via foreign keys as partition table citus_local_tables_mx.cas_par2_1 involved in a foreign key relationship that is not inherited from it's parent table +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); +ERROR: cannot cascade operation via foreign keys as partition table citus_local_tables_mx.cas_par2_1 involved in a foreign key relationship that is not inherited from it's parent table +-- drop the foreign keys and establish them again using the parent table +ALTER TABLE cas_par_1 DROP CONSTRAINT fkey_cas_test_1; +ALTER TABLE cas_par2_1 DROP CONSTRAINT fkey_cas_test_2; +ALTER TABLE cas_par ADD CONSTRAINT fkey_cas_test_1 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_par2 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_par3 ADD CONSTRAINT fkey_cas_test_3 FOREIGN KEY (a) REFERENCES cas_par(a); +-- this should error out as cascade_via_foreign_keys is not set to true +SELECT citus_add_local_table_to_metadata('cas_par2'); +ERROR: relation citus_local_tables_mx.cas_par2 is involved in a foreign key relationship with another table +-- this should work +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- verify the partitioning hierarchy is preserved +select inhrelid::regclass from pg_inherits where inhparent='cas_par'::regclass order by 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +-- verify the fkeys + fkeys with shard ids are created +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; + conname +--------------------------------------------------------------------- + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_1_1330008 + fkey_cas_test_1_1330008 + fkey_cas_test_1_1330008 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_2_1330006 + fkey_cas_test_2_1330006 + fkey_cas_test_2_1330006 + fkey_cas_test_3 + fkey_cas_test_3 + fkey_cas_test_3 + fkey_cas_test_3_1330013 + fkey_cas_test_3_1330013 + fkey_cas_test_3_1330013 +(18 rows) + +-- when all partitions are converted, there should be 40 tables and indexes +-- the individual names are not much relevant, so we only print the count +SELECT count(*) FROM pg_class WHERE relname LIKE 'cas\_%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_mx'); + count +--------------------------------------------------------------------- + 40 +(1 row) + +-- verify on the mx worker +\c - - - :worker_1_port +-- on worker, there should be 20, since the shards are created only on the coordinator +SELECT count(*) FROM pg_class WHERE relname LIKE 'cas\_%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_mx'); + count +--------------------------------------------------------------------- + 20 +(1 row) + +-- verify that the shell foreign keys are created on the worker as well +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; + conname +--------------------------------------------------------------------- + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_3 + fkey_cas_test_3 + fkey_cas_test_3 +(9 rows) + +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +-- undistribute table +-- this one should error out since we don't set the cascade option as true +SELECT undistribute_table('cas_par2'); +ERROR: cannot complete operation because table cas_par2 has a foreign key +-- this one should work +SET client_min_messages TO WARNING; +SELECT undistribute_table('cas_par2', cascade_via_foreign_keys=>true); + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +-- verify the partitioning hierarchy is preserved +select inhrelid::regclass from pg_inherits where inhparent='cas_par'::regclass order by 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +-- verify that the foreign keys with shard ids are gone, due to undistribution +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; + conname +--------------------------------------------------------------------- + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_1 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_2 + fkey_cas_test_3 + fkey_cas_test_3 + fkey_cas_test_3 +(9 rows) + +-- add a non-inherited fkey and verify it fails when trying to convert +ALTER TABLE cas_par2_1 ADD CONSTRAINT fkey_cas_test_3 FOREIGN KEY (a) REFERENCES cas_1(a); +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); +ERROR: cannot cascade operation via foreign keys as partition table citus_local_tables_mx.cas_par2_1 involved in a foreign key relationship that is not inherited from it's parent table +-- verify undistribute_table works proper for the mx worker +\c - - - :worker_1_port +SELECT relname FROM pg_class WHERE relname LIKE 'cas\_%' ORDER BY relname; + relname +--------------------------------------------------------------------- +(0 rows) + +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +CREATE TABLE date_partitioned_citus_local_table_seq( measureid bigserial, eventdate date, measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); +SELECT create_time_partitions('date_partitioned_citus_local_table_seq', INTERVAL '1 month', '2022-01-01', '2021-01-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table_seq'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE date_partitioned_citus_local_table_seq; +-- test sequences +CREATE TABLE par_citus_local_seq(measureid bigserial, val int) PARTITION BY RANGE(val); +CREATE TABLE par_citus_local_seq_1 PARTITION OF par_citus_local_seq FOR VALUES FROM (1) TO (4); +SELECT citus_add_local_table_to_metadata('par_citus_local_seq'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO par_citus_local_seq (val) VALUES (1) RETURNING *; + measureid | val +--------------------------------------------------------------------- + 1 | 1 +(1 row) + +INSERT INTO par_citus_local_seq (val) VALUES (2) RETURNING *; + measureid | val +--------------------------------------------------------------------- + 2 | 2 +(1 row) + +\c - - - :worker_1_port +-- insert on the worker +INSERT INTO citus_local_tables_mx.par_citus_local_seq (val) VALUES (1) RETURNING *; + measureid | val +--------------------------------------------------------------------- + 3940649673949185 | 1 +(1 row) + +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +INSERT INTO par_citus_local_seq (val) VALUES (2) RETURNING *; + measureid | val +--------------------------------------------------------------------- + 3 | 2 +(1 row) + +SELECT undistribute_table('par_citus_local_seq'); +NOTICE: converting the partitions of citus_local_tables_mx.par_citus_local_seq +NOTICE: creating a new table for citus_local_tables_mx.par_citus_local_seq_1 +NOTICE: moving the data of citus_local_tables_mx.par_citus_local_seq_1 +NOTICE: dropping the old citus_local_tables_mx.par_citus_local_seq_1 +NOTICE: renaming the new table to citus_local_tables_mx.par_citus_local_seq_1 +NOTICE: creating a new table for citus_local_tables_mx.par_citus_local_seq +NOTICE: dropping the old citus_local_tables_mx.par_citus_local_seq +NOTICE: renaming the new table to citus_local_tables_mx.par_citus_local_seq + undistribute_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO par_citus_local_seq (val) VALUES (3) RETURNING *; + measureid | val +--------------------------------------------------------------------- + 4 | 3 +(1 row) + +SELECT measureid FROM par_citus_local_seq ORDER BY measureid; + measureid +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 + 3940649673949185 +(5 rows) + +-- test adding invalid foreign key to partition table +CREATE TABLE citus_local_parent_1 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE citus_local_parent_2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE citus_local_plain (a INT UNIQUE); +CREATE TABLE ref (a INT UNIQUE); +SELECT create_reference_table('ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +alter table citus_local_parent_1 add foreign key (a) references citus_local_parent_2(a); +alter table citus_local_plain add foreign key (a) references citus_local_parent_2(a); +CREATE TABLE citus_local_parent_1_child_1 PARTITION OF citus_local_parent_1 FOR VALUES FROM (3) TO (5); +CREATE TABLE citus_local_parent_1_child_2 PARTITION OF citus_local_parent_1 FOR VALUES FROM (30) TO (50); +CREATE TABLE citus_local_parent_2_child_1 PARTITION OF citus_local_parent_2 FOR VALUES FROM (40) TO (60); +-- this one should error out, since we cannot convert it to citus local table, +-- as citus local table partitions cannot have non-inherited foreign keys +alter table citus_local_parent_1_child_1 add foreign key(a) references ref(a); +ERROR: cannot build foreign key between reference table and a partition +-- this should work +alter table citus_local_parent_1 add constraint fkey_to_drop_test foreign key(a) references ref(a); +-- this should undistribute the table, and the entries should be gone from pg_dist_partition +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; + logicalrelid +--------------------------------------------------------------------- + citus_local_parent_1 + citus_local_parent_2 + citus_local_parent_2_child_1 + citus_local_parent_1_child_1 + citus_local_parent_1_child_2 +(5 rows) + +set client_min_messages to error; +alter table citus_local_parent_1 drop constraint fkey_to_drop_test; +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; + logicalrelid +--------------------------------------------------------------------- +(0 rows) + SELECT master_remove_distributed_table_metadata_from_workers('citus_local_table_4'::regclass::oid, 'citus_local_tables_mx', 'citus_local_table_4'); master_remove_distributed_table_metadata_from_workers --------------------------------------------------------------------- @@ -259,4 +665,3 @@ $$); -- cleanup at exit DROP SCHEMA citus_local_tables_mx CASCADE; -NOTICE: drop cascades to 19 other objects diff --git a/src/test/regress/expected/create_citus_local_table_cascade.out b/src/test/regress/expected/create_citus_local_table_cascade.out index 31d4e6f9c..fcf3f4c0a 100644 --- a/src/test/regress/expected/create_citus_local_table_cascade.out +++ b/src/test/regress/expected/create_citus_local_table_cascade.out @@ -116,11 +116,12 @@ BEGIN; ROLLBACK; BEGIN; CREATE TABLE partitioned_table (col_1 INT REFERENCES local_table_1 (col_1)) PARTITION BY RANGE (col_1); - -- now that we introduced a partitioned table into our foreign key subgraph, - -- citus_add_local_table_to_metadata(cascade_via_foreign_keys) would fail for - -- partitioned_table as citus_add_local_table_to_metadata doesn't support partitioned tables SELECT citus_add_local_table_to_metadata('local_table_2', cascade_via_foreign_keys=>true); -ERROR: cannot add local table "partitioned_table" to metadata, only regular tables and foreign tables can be added to citus metadata + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + ROLLBACK; BEGIN; DROP TABLE local_table_2; diff --git a/src/test/regress/expected/fkeys_between_local_ref.out b/src/test/regress/expected/fkeys_between_local_ref.out index 7a620b84e..1e3f7eb20 100644 --- a/src/test/regress/expected/fkeys_between_local_ref.out +++ b/src/test/regress/expected/fkeys_between_local_ref.out @@ -99,10 +99,9 @@ CREATE TABLE partitioned_table_1_100_200 PARTITION OF partitioned_table_1 FOR VA CREATE TABLE partitioned_table_1_200_300 PARTITION OF partitioned_table_1 FOR VALUES FROM (200) TO (300); INSERT INTO partitioned_table_1 SELECT i FROM generate_series(195, 205) i; ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_8 FOREIGN KEY (col_1) REFERENCES local_table_4(col_1); --- now that we attached partitioned table to graph below errors out --- since we cannot create citus local table from partitioned tables +BEGIN; ALTER TABLE reference_table_1 ADD CONSTRAINT fkey_9 FOREIGN KEY (col_1) REFERENCES local_table_1(col_1); -ERROR: cannot add local table "partitioned_table_1" to metadata, only regular tables and foreign tables can be added to citus metadata +ROLLBACK; ALTER TABLE partitioned_table_1 DROP CONSTRAINT fkey_8; BEGIN; -- now that we detached partitioned table from graph, succeeds @@ -329,7 +328,7 @@ BEGIN; -- show that we validate foreign key constraints, errors out INSERT INTO local_table_5 VALUES (300); -ERROR: insert or update on table "local_table_5_1518070" violates foreign key constraint "local_table_5_col_1_fkey1_1518070" +ERROR: insert or update on table "local_table_5_1518073" violates foreign key constraint "local_table_5_col_1_fkey1_1518073" ROLLBACK; BEGIN; CREATE SCHEMA another_schema_fkeys_between_local_ref; @@ -620,10 +619,9 @@ ORDER BY tablename; reference_table_1 | n | t (7 rows) --- this errors out as we don't support creating citus local --- tables from partitioned tables +BEGIN; CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1); -ERROR: cannot add local table "part_local_table" to metadata, only regular tables and foreign tables can be added to citus metadata +ROLLBACK; -- they fail as col_99 does not exist CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_99) REFERENCES reference_table_1(col_1)); ERROR: column "col_99" referenced in foreign key constraint does not exist diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index 9ebb23a60..0b1545db5 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -3763,6 +3763,306 @@ BEGIN; ROLLBACK; DROP TABLE pi_table; +-- 6) test with citus local table +select 1 from citus_add_node('localhost', :master_port, groupid=>0); +NOTICE: Replicating reference table "orders_reference" to the node localhost:xxxxx +NOTICE: Replicating reference table "customer" to the node localhost:xxxxx +NOTICE: Replicating reference table "nation" to the node localhost:xxxxx +NOTICE: Replicating reference table "part" to the node localhost:xxxxx +NOTICE: Replicating reference table "supplier" to the node localhost:xxxxx +NOTICE: Replicating reference table "users_ref_test_table" to the node localhost:xxxxx +NOTICE: Replicating reference table "events_reference_table" to the node localhost:xxxxx +NOTICE: Replicating reference table "users_reference_table" to the node localhost:xxxxx + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +CREATE TABLE date_partitioned_citus_local_table( + measureid integer, + eventdate date, + measure_data jsonb) PARTITION BY RANGE(eventdate); +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- test interval must be multiple days for date partitioned table +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '6 hours', '2022-01-01', '2021-01-01'); +ERROR: partition interval of date partitioned table must be day or multiple days +CONTEXT: PL/pgSQL function get_missing_time_partition_ranges(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at RAISE +PL/pgSQL function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at FOR over SELECT rows +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 week 1 day 1 hour', '2022-01-01', '2021-01-01'); +ERROR: partition interval of date partitioned table must be day or multiple days +CONTEXT: PL/pgSQL function get_missing_time_partition_ranges(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at RAISE +PL/pgSQL function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) line XX at FOR over SELECT rows +-- test with various intervals +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-02-01', '2021-01-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; + parent_table | partition_column | partition | from_value | to_value | access_method +--------------------------------------------------------------------- + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_01 | 01-01-2021 | 01-02-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_02 | 01-02-2021 | 01-03-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_03 | 01-03-2021 | 01-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_04 | 01-04-2021 | 01-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_05 | 01-05-2021 | 01-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_06 | 01-06-2021 | 01-07-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_07 | 01-07-2021 | 01-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_08 | 01-08-2021 | 01-09-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_09 | 01-09-2021 | 01-10-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_10 | 01-10-2021 | 01-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_11 | 01-11-2021 | 01-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_12 | 01-12-2021 | 01-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_13 | 01-13-2021 | 01-14-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_14 | 01-14-2021 | 01-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_15 | 01-15-2021 | 01-16-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_16 | 01-16-2021 | 01-17-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_17 | 01-17-2021 | 01-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_18 | 01-18-2021 | 01-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_19 | 01-19-2021 | 01-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_20 | 01-20-2021 | 01-21-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_21 | 01-21-2021 | 01-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_22 | 01-22-2021 | 01-23-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_23 | 01-23-2021 | 01-24-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_24 | 01-24-2021 | 01-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_25 | 01-25-2021 | 01-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_26 | 01-26-2021 | 01-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_27 | 01-27-2021 | 01-28-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_28 | 01-28-2021 | 01-29-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_29 | 01-29-2021 | 01-30-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_30 | 01-30-2021 | 01-31-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_31 | 01-31-2021 | 02-01-2021 | heap +(31 rows) + +ROLLBACK; +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 week', '2022-01-01', '2021-01-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; + parent_table | partition_column | partition | from_value | to_value | access_method +--------------------------------------------------------------------- + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2020w53 | 12-28-2020 | 01-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w01 | 01-04-2021 | 01-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w02 | 01-11-2021 | 01-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w03 | 01-18-2021 | 01-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w04 | 01-25-2021 | 02-01-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w05 | 02-01-2021 | 02-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w06 | 02-08-2021 | 02-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w07 | 02-15-2021 | 02-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w08 | 02-22-2021 | 03-01-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w09 | 03-01-2021 | 03-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w10 | 03-08-2021 | 03-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w11 | 03-15-2021 | 03-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w12 | 03-22-2021 | 03-29-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w13 | 03-29-2021 | 04-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w14 | 04-05-2021 | 04-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w15 | 04-12-2021 | 04-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w16 | 04-19-2021 | 04-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w17 | 04-26-2021 | 05-03-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w18 | 05-03-2021 | 05-10-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w19 | 05-10-2021 | 05-17-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w20 | 05-17-2021 | 05-24-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w21 | 05-24-2021 | 05-31-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w22 | 05-31-2021 | 06-07-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w23 | 06-07-2021 | 06-14-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w24 | 06-14-2021 | 06-21-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w25 | 06-21-2021 | 06-28-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w26 | 06-28-2021 | 07-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w27 | 07-05-2021 | 07-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w28 | 07-12-2021 | 07-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w29 | 07-19-2021 | 07-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w30 | 07-26-2021 | 08-02-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w31 | 08-02-2021 | 08-09-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w32 | 08-09-2021 | 08-16-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w33 | 08-16-2021 | 08-23-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w34 | 08-23-2021 | 08-30-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w35 | 08-30-2021 | 09-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w36 | 09-06-2021 | 09-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w37 | 09-13-2021 | 09-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w38 | 09-20-2021 | 09-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w39 | 09-27-2021 | 10-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w40 | 10-04-2021 | 10-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w41 | 10-11-2021 | 10-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w42 | 10-18-2021 | 10-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w43 | 10-25-2021 | 11-01-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w44 | 11-01-2021 | 11-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w45 | 11-08-2021 | 11-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w46 | 11-15-2021 | 11-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w47 | 11-22-2021 | 11-29-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w48 | 11-29-2021 | 12-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w49 | 12-06-2021 | 12-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w50 | 12-13-2021 | 12-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w51 | 12-20-2021 | 12-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021w52 | 12-27-2021 | 01-03-2022 | heap +(53 rows) + +ROLLBACK; +set client_min_messages to error; +DROP TABLE date_partitioned_citus_local_table; +-- also test with foreign key +CREATE TABLE date_partitioned_citus_local_table( + measureid integer, + eventdate date, + measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- test interval must be multiple days for date partitioned table +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-02-01', '2021-01-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + +CREATE TABLE date_partitioned_citus_local_table_2( + measureid integer, + eventdate date, + measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE date_partitioned_citus_local_table_2 ADD CONSTRAINT fkey_1 FOREIGN KEY (measureid, eventdate) REFERENCES date_partitioned_citus_local_table(measureid, eventdate); +SELECT create_time_partitions('date_partitioned_citus_local_table_2', INTERVAL '1 day', '2021-02-01', '2021-01-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + +-- after the above work, these should also work for creating new partitions +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-03-01', '2021-02-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; + parent_table | partition_column | partition | from_value | to_value | access_method +--------------------------------------------------------------------- + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_01 | 01-01-2021 | 01-02-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_02 | 01-02-2021 | 01-03-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_03 | 01-03-2021 | 01-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_04 | 01-04-2021 | 01-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_05 | 01-05-2021 | 01-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_06 | 01-06-2021 | 01-07-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_07 | 01-07-2021 | 01-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_08 | 01-08-2021 | 01-09-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_09 | 01-09-2021 | 01-10-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_10 | 01-10-2021 | 01-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_11 | 01-11-2021 | 01-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_12 | 01-12-2021 | 01-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_13 | 01-13-2021 | 01-14-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_14 | 01-14-2021 | 01-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_15 | 01-15-2021 | 01-16-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_16 | 01-16-2021 | 01-17-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_17 | 01-17-2021 | 01-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_18 | 01-18-2021 | 01-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_19 | 01-19-2021 | 01-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_20 | 01-20-2021 | 01-21-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_21 | 01-21-2021 | 01-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_22 | 01-22-2021 | 01-23-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_23 | 01-23-2021 | 01-24-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_24 | 01-24-2021 | 01-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_25 | 01-25-2021 | 01-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_26 | 01-26-2021 | 01-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_27 | 01-27-2021 | 01-28-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_28 | 01-28-2021 | 01-29-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_29 | 01-29-2021 | 01-30-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_30 | 01-30-2021 | 01-31-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_31 | 01-31-2021 | 02-01-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_01 | 02-01-2021 | 02-02-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_02 | 02-02-2021 | 02-03-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_03 | 02-03-2021 | 02-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_04 | 02-04-2021 | 02-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_05 | 02-05-2021 | 02-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_06 | 02-06-2021 | 02-07-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_07 | 02-07-2021 | 02-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_08 | 02-08-2021 | 02-09-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_09 | 02-09-2021 | 02-10-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_10 | 02-10-2021 | 02-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_11 | 02-11-2021 | 02-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_12 | 02-12-2021 | 02-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_13 | 02-13-2021 | 02-14-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_14 | 02-14-2021 | 02-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_15 | 02-15-2021 | 02-16-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_16 | 02-16-2021 | 02-17-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_17 | 02-17-2021 | 02-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_18 | 02-18-2021 | 02-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_19 | 02-19-2021 | 02-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_20 | 02-20-2021 | 02-21-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_21 | 02-21-2021 | 02-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_22 | 02-22-2021 | 02-23-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_23 | 02-23-2021 | 02-24-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_24 | 02-24-2021 | 02-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_25 | 02-25-2021 | 02-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_26 | 02-26-2021 | 02-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_27 | 02-27-2021 | 02-28-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_02_28 | 02-28-2021 | 03-01-2021 | heap +(59 rows) + +ROLLBACK; +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table_2', INTERVAL '1 day', '2021-03-01', '2021-02-01'); + create_time_partitions +--------------------------------------------------------------------- + t +(1 row) + + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; + parent_table | partition_column | partition | from_value | to_value | access_method +--------------------------------------------------------------------- + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_01 | 01-01-2021 | 01-02-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_02 | 01-02-2021 | 01-03-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_03 | 01-03-2021 | 01-04-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_04 | 01-04-2021 | 01-05-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_05 | 01-05-2021 | 01-06-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_06 | 01-06-2021 | 01-07-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_07 | 01-07-2021 | 01-08-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_08 | 01-08-2021 | 01-09-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_09 | 01-09-2021 | 01-10-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_10 | 01-10-2021 | 01-11-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_11 | 01-11-2021 | 01-12-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_12 | 01-12-2021 | 01-13-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_13 | 01-13-2021 | 01-14-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_14 | 01-14-2021 | 01-15-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_15 | 01-15-2021 | 01-16-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_16 | 01-16-2021 | 01-17-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_17 | 01-17-2021 | 01-18-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_18 | 01-18-2021 | 01-19-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_19 | 01-19-2021 | 01-20-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_20 | 01-20-2021 | 01-21-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_21 | 01-21-2021 | 01-22-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_22 | 01-22-2021 | 01-23-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_23 | 01-23-2021 | 01-24-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_24 | 01-24-2021 | 01-25-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_25 | 01-25-2021 | 01-26-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_26 | 01-26-2021 | 01-27-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_27 | 01-27-2021 | 01-28-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_28 | 01-28-2021 | 01-29-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_29 | 01-29-2021 | 01-30-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_30 | 01-30-2021 | 01-31-2021 | heap + date_partitioned_citus_local_table | eventdate | date_partitioned_citus_local_table_p2021_01_31 | 01-31-2021 | 02-01-2021 | heap +(31 rows) + +ROLLBACK; +set client_min_messages to notice; -- c) test drop_old_time_partitions -- 1) test with date partitioned table CREATE TABLE date_partitioned_table_to_exp (event_date date, event int) partition by range (event_date); @@ -3875,6 +4175,46 @@ SELECT partition FROM time_partitions WHERE parent_table = '"test !/ \n _dist_12 \set VERBOSITY default DROP TABLE "test !/ \n _dist_123_table_exp"; +-- 4) test with citus local tables +CREATE TABLE date_partitioned_table_to_exp (event_date date, event int) partition by range (event_date); +SELECT citus_add_local_table_to_metadata('date_partitioned_table_to_exp'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE date_partitioned_table_to_exp_d00 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2000-01-01') TO ('2009-12-31'); +CREATE TABLE date_partitioned_table_to_exp_d10 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2010-01-01') TO ('2019-12-31'); +CREATE TABLE date_partitioned_table_to_exp_d20 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2020-01-01') TO ('2029-12-31'); +\set VERBOSITY terse +-- expire no partitions +CALL drop_old_time_partitions('date_partitioned_table_to_exp', '1999-01-01'); +SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_table_to_exp'::regclass ORDER BY partition::text; + partition +--------------------------------------------------------------------- + date_partitioned_table_to_exp_d00 + date_partitioned_table_to_exp_d10 + date_partitioned_table_to_exp_d20 +(3 rows) + +-- expire 2 old partitions +CALL drop_old_time_partitions('date_partitioned_table_to_exp', '2021-01-01'); +NOTICE: dropping date_partitioned_table_to_exp_d00 with start time 01-01-2000 and end time 12-31-2009 +NOTICE: dropping date_partitioned_table_to_exp_d10 with start time 01-01-2010 and end time 12-31-2019 +SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_table_to_exp'::regclass ORDER BY partition::text; + partition +--------------------------------------------------------------------- + date_partitioned_table_to_exp_d20 +(1 row) + +\set VERBOSITY default +DROP TABLE date_partitioned_table_to_exp; +SELECT citus_remove_node('localhost', :master_port); + citus_remove_node +--------------------------------------------------------------------- + +(1 row) + -- d) invalid tables for helper UDFs CREATE TABLE multiple_partition_column_table( event_id bigserial, @@ -3986,10 +4326,12 @@ NOTICE: dropping metadata on the node (localhost,57637) (1 row) DROP SCHEMA partitioning_schema CASCADE; -NOTICE: drop cascades to 4 other objects +NOTICE: drop cascades to 6 other objects DETAIL: drop cascades to table partitioning_schema."schema-test" drop cascades to table partitioning_schema.another_distributed_table drop cascades to table partitioning_schema.distributed_parent_table +drop cascades to table partitioning_schema.date_partitioned_citus_local_table +drop cascades to table partitioning_schema.date_partitioned_citus_local_table_2 drop cascades to table partitioning_schema.part_table_with_very_long_name RESET search_path; DROP TABLE IF EXISTS diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index d1573522c..25f10703f 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -1289,14 +1289,6 @@ ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_4 FOREIGN KEY (col_1) REFERE ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_5 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); SELECT undistribute_table('partitioned_table_1', cascade_via_foreign_keys=>true); NOTICE: converting the partitions of single_node.partitioned_table_1 -NOTICE: creating a new table for single_node.partitioned_table_1_100_200 -NOTICE: moving the data of single_node.partitioned_table_1_100_200 -NOTICE: dropping the old single_node.partitioned_table_1_100_200 -NOTICE: renaming the new table to single_node.partitioned_table_1_100_200 -NOTICE: creating a new table for single_node.partitioned_table_1_200_300 -NOTICE: moving the data of single_node.partitioned_table_1_200_300 -NOTICE: dropping the old single_node.partitioned_table_1_200_300 -NOTICE: renaming the new table to single_node.partitioned_table_1_200_300 NOTICE: creating a new table for single_node.partitioned_table_1 NOTICE: dropping the old single_node.partitioned_table_1 NOTICE: renaming the new table to single_node.partitioned_table_1 @@ -1312,6 +1304,14 @@ NOTICE: creating a new table for single_node.citus_local_table_1 NOTICE: moving the data of single_node.citus_local_table_1 NOTICE: dropping the old single_node.citus_local_table_1 NOTICE: renaming the new table to single_node.citus_local_table_1 +NOTICE: creating a new table for single_node.partitioned_table_1_100_200 +NOTICE: moving the data of single_node.partitioned_table_1_100_200 +NOTICE: dropping the old single_node.partitioned_table_1_100_200 +NOTICE: renaming the new table to single_node.partitioned_table_1_100_200 +NOTICE: creating a new table for single_node.partitioned_table_1_200_300 +NOTICE: moving the data of single_node.partitioned_table_1_200_300 +NOTICE: dropping the old single_node.partitioned_table_1_200_300 +NOTICE: renaming the new table to single_node.partitioned_table_1_200_300 undistribute_table --------------------------------------------------------------------- diff --git a/src/test/regress/expected/undistribute_table_cascade.out b/src/test/regress/expected/undistribute_table_cascade.out index db98bf2f0..d40f41fe4 100644 --- a/src/test/regress/expected/undistribute_table_cascade.out +++ b/src/test/regress/expected/undistribute_table_cascade.out @@ -382,12 +382,12 @@ BEGIN; ORDER BY 1,2,3; conname | conrelid | confrelid --------------------------------------------------------------------- + fkey_10 | partitioned_table_2 | reference_table_3 fkey_10 | partitioned_table_2_100_200 | reference_table_3 fkey_10 | partitioned_table_2_200_300 | reference_table_3 - fkey_10 | partitioned_table_2 | reference_table_3 + fkey_9 | partitioned_table_1 | reference_table_3 fkey_9 | partitioned_table_1_100_200 | reference_table_3 fkey_9 | partitioned_table_1_200_300 | reference_table_3 - fkey_9 | partitioned_table_1 | reference_table_3 (6 rows) ROLLBACK; diff --git a/src/test/regress/sql/citus_local_table_triggers.sql b/src/test/regress/sql/citus_local_table_triggers.sql index cd262d677..d091c498b 100644 --- a/src/test/regress/sql/citus_local_table_triggers.sql +++ b/src/test/regress/sql/citus_local_table_triggers.sql @@ -301,5 +301,40 @@ BEGIN; SELECT * FROM reference_table; ROLLBACK; +-- test on partitioned citus local tables +CREATE TABLE par_citus_local_table (val int) PARTITION BY RANGE(val); +CREATE TABLE par_citus_local_table_1 PARTITION OF par_citus_local_table FOR VALUES FROM (1) TO (10000); +CREATE TABLE par_another_citus_local_table (val int unique) PARTITION BY RANGE(val); +CREATE TABLE par_another_citus_local_table_1 PARTITION OF par_another_citus_local_table FOR VALUES FROM (1) TO (10000); + +ALTER TABLE par_another_citus_local_table ADD CONSTRAINT fkey_self FOREIGN KEY(val) REFERENCES par_another_citus_local_table(val); +ALTER TABLE par_citus_local_table ADD CONSTRAINT fkey_c_to_c FOREIGN KEY(val) REFERENCES par_another_citus_local_table(val) ON UPDATE CASCADE; + +SELECT citus_add_local_table_to_metadata('par_another_citus_local_table', cascade_via_foreign_keys=>true); + +CREATE TABLE par_reference_table(val int); +SELECT create_reference_table('par_reference_table'); + +CREATE FUNCTION par_insert_100() RETURNS trigger AS $par_insert_100$ +BEGIN + INSERT INTO par_reference_table VALUES (100); + RETURN NEW; +END; +$par_insert_100$ LANGUAGE plpgsql; + +BEGIN; + CREATE TRIGGER par_insert_100_trigger + AFTER TRUNCATE ON par_another_citus_local_table + FOR EACH STATEMENT EXECUTE FUNCTION par_insert_100(); + + CREATE TRIGGER insert_100_trigger + AFTER TRUNCATE ON par_citus_local_table + FOR EACH STATEMENT EXECUTE FUNCTION par_insert_100(); + + TRUNCATE par_another_citus_local_table CASCADE; + -- we should see two rows with "100" + SELECT * FROM par_reference_table; +ROLLBACK; + -- cleanup at exit DROP SCHEMA citus_local_table_triggers, "interesting!schema" CASCADE; diff --git a/src/test/regress/sql/citus_local_tables.sql b/src/test/regress/sql/citus_local_tables.sql index e7c495614..7e7f3f30d 100644 --- a/src/test/regress/sql/citus_local_tables.sql +++ b/src/test/regress/sql/citus_local_tables.sql @@ -93,42 +93,6 @@ SELECT create_distributed_table('distributed_table', 'a'); -- cannot create citus local table from an existing citus table SELECT citus_add_local_table_to_metadata('distributed_table'); --- partitioned table tests -- - -CREATE TABLE partitioned_table(a int, b int) PARTITION BY RANGE (a); -CREATE TABLE partitioned_table_1 PARTITION OF partitioned_table FOR VALUES FROM (0) TO (10); -CREATE TABLE partitioned_table_2 PARTITION OF partitioned_table FOR VALUES FROM (10) TO (20); - --- cannot create partitioned citus local tables -SELECT citus_add_local_table_to_metadata('partitioned_table'); - -BEGIN; - CREATE TABLE citus_local_table PARTITION OF partitioned_table FOR VALUES FROM (20) TO (30); - - -- cannot create citus local table as a partition of a local table - SELECT citus_add_local_table_to_metadata('citus_local_table'); -ROLLBACK; - -BEGIN; - CREATE TABLE citus_local_table (a int, b int); - - SELECT citus_add_local_table_to_metadata('citus_local_table'); - - -- cannot create citus local table as a partition of a local table - -- via ALTER TABLE commands as well - ALTER TABLE partitioned_table ATTACH PARTITION citus_local_table FOR VALUES FROM (20) TO (30); -ROLLBACK; - -BEGIN; - SELECT create_distributed_table('partitioned_table', 'a'); - - CREATE TABLE citus_local_table (a int, b int); - SELECT citus_add_local_table_to_metadata('citus_local_table'); - - -- cannot attach citus local table to a partitioned distributed table - ALTER TABLE partitioned_table ATTACH PARTITION citus_local_table FOR VALUES FROM (20) TO (30); -ROLLBACK; - -- show that we do not support inheritance relationships -- CREATE TABLE parent_table (a int, b text); @@ -539,5 +503,95 @@ TRUNCATE referenced_table CASCADE; RESET client_min_messages; \set VERBOSITY terse +-- test for partitioned tables +SET client_min_messages TO ERROR; +-- verify we can convert partitioned tables into Citus Local Tables +CREATE TABLE partitioned (user_id int, time timestamp with time zone, data jsonb, PRIMARY KEY (user_id, time )) PARTITION BY RANGE ("time"); +CREATE TABLE partition1 PARTITION OF partitioned FOR VALUES FROM ('2018-04-13 00:00:00+00') TO ('2018-04-14 00:00:00+00'); +CREATE TABLE partition2 PARTITION OF partitioned FOR VALUES FROM ('2018-04-14 00:00:00+00') TO ('2018-04-15 00:00:00+00'); +SELECT citus_add_local_table_to_metadata('partitioned'); +-- partitions added after the conversion get converted into CLT as well +CREATE TABLE partition3 PARTITION OF partitioned FOR VALUES FROM ('2018-04-15 00:00:00+00') TO ('2018-04-16 00:00:00+00'); +--verify partitioning hierarchy is preserved after conversion +select inhrelid::regclass from pg_inherits where inhparent='partitioned'::regclass order by 1; +SELECT partition, from_value, to_value, access_method + FROM time_partitions + WHERE partition::text LIKE '%partition%' + ORDER BY partition::text; +-- undistribute succesfully +SELECT undistribute_table('partitioned'); +-- verify the partitioning hierarchy is preserved after undistributing +select inhrelid::regclass from pg_inherits where inhparent='partitioned'::regclass order by 1; + +-- verify table is undistributed +SELECT relname FROM pg_class WHERE relname LIKE 'partition3%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_test_schema') + ORDER BY relname; + +-- drop successfully +DROP TABLE partitioned; + +-- test creating distributed tables from partitioned citus local tables +CREATE TABLE partitioned_distributed (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE partitioned_distributed_1 PARTITION OF partitioned_distributed FOR VALUES FROM (1) TO (4); +CREATE TABLE partitioned_distributed_2 PARTITION OF partitioned_distributed FOR VALUES FROM (5) TO (8); +SELECT citus_add_local_table_to_metadata('partitioned_distributed'); +SELECT create_distributed_table('partitioned_distributed','a'); + +\c - - - :worker_1_port +SELECT relname FROM pg_class + WHERE relname LIKE 'partitioned_distributed%' + AND relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_test_schema') + ORDER BY relname; +\c - - - :master_port +SET search_path TO citus_local_tables_test_schema; + +-- error out if converting multi-level partitioned table +CREATE TABLE multi_par (id text, country text) PARTITION BY RANGE (id); +ALTER TABLE multi_par ADD CONSTRAINT unique_constraint UNIQUE (id, country); +CREATE TABLE multi_par_a_to_i PARTITION OF multi_par FOR VALUES FROM ('a') TO ('j'); +-- multi-level partitioning +CREATE TABLE multi_par_j_to_r PARTITION OF multi_par FOR VALUES FROM ('j') TO ('s') PARTITION BY LIST (country); +CREATE TABLE multi_par_j_to_r_japan PARTITION OF multi_par_j_to_r FOR VALUES IN ('japan'); +-- these two should error out +SELECT citus_add_local_table_to_metadata('multi_par'); +SELECT citus_add_local_table_to_metadata('multi_par',cascade_via_foreign_keys=>true); + +-- should error out when cascading via fkeys as well +CREATE TABLE cas_1 (a text, b text); +ALTER TABLE cas_1 ADD CONSTRAINT unique_constraint_2 UNIQUE (a, b); +ALTER TABLE cas_1 ADD CONSTRAINT fkey_to_multi_parti FOREIGN KEY (a,b) REFERENCES multi_par(id, country); +SELECT citus_add_local_table_to_metadata('cas_1'); +SELECT citus_add_local_table_to_metadata('cas_1', cascade_via_foreign_keys=>true); +SELECT create_reference_table('cas_1'); +ALTER TABLE cas_1 DROP CONSTRAINT fkey_to_multi_parti; +SELECT create_reference_table('cas_1'); +ALTER TABLE cas_1 ADD CONSTRAINT fkey_to_multi_parti FOREIGN KEY (a,b) REFERENCES multi_par(id, country); + +-- undistribute tables to avoid unnecessary log messages later +select undistribute_table('citus_local_table_4', cascade_via_foreign_keys=>true); +select undistribute_table('referencing_table', cascade_via_foreign_keys=>true); + +-- test dropping fkey +CREATE TABLE parent_2_child_1 (a int); +CREATE TABLE parent_1_child_1 (a int); +CREATE TABLE parent_2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE parent_1 (a INT UNIQUE) PARTITION BY RANGE(a); +alter table parent_1 attach partition parent_1_child_1 default ; +alter table parent_2 attach partition parent_2_child_1 default ; +CREATE TABLE ref_table(a int unique); +alter table parent_1 add constraint fkey_test_drop foreign key(a) references ref_table(a); +alter table parent_2 add constraint fkey1 foreign key(a) references ref_table(a); +alter table parent_1 add constraint fkey2 foreign key(a) references parent_2(a); +SELECT create_reference_table('ref_table'); +alter table parent_1 drop constraint fkey_test_drop; +select count(*) from pg_constraint where conname = 'fkey_test_drop'; +-- verify we still preserve the child-parent hierarchy after all conversions +-- check the shard partition +select inhrelid::regclass from pg_inherits where (select inhparent::regclass::text) ~ '^parent_1_\d{7}$' order by 1; +-- check the shell partition +select inhrelid::regclass from pg_inherits where inhparent='parent_1'::regclass order by 1; + -- cleanup at exit +SET client_min_messages TO ERROR; DROP SCHEMA citus_local_tables_test_schema, "CiTUS!LocalTables", "test_\'index_schema" CASCADE; diff --git a/src/test/regress/sql/citus_local_tables_mx.sql b/src/test/regress/sql/citus_local_tables_mx.sql index 391e9f842..547ba80df 100644 --- a/src/test/regress/sql/citus_local_tables_mx.sql +++ b/src/test/regress/sql/citus_local_tables_mx.sql @@ -168,6 +168,180 @@ SELECT stxname FROM pg_statistic_ext ORDER BY stxname; \c - - - :master_port SET search_path TO citus_local_tables_mx; +-- undistribute old tables to prevent unnecessary undistribute logs later +SELECT undistribute_table('citus_local_table', cascade_via_foreign_keys=>true); +SELECT undistribute_table('citus_local_table_3', cascade_via_foreign_keys=>true); +SELECT undistribute_table('citus_local_table_stats', cascade_via_foreign_keys=>true); + +-- verify that mx nodes have the shell table for partitioned citus local tables +CREATE TABLE local_table_fkey(a INT UNIQUE); +CREATE TABLE local_table_fkey2(a INT UNIQUE); +CREATE TABLE partitioned_mx (a INT UNIQUE) PARTITION BY RANGE(a); +ALTER TABLE partitioned_mx ADD CONSTRAINT fkey_to_local FOREIGN KEY (a) REFERENCES local_table_fkey(a); + +CREATE TABLE IF NOT EXISTS partitioned_mx_1 PARTITION OF partitioned_mx FOR VALUES FROM (1) TO (4); +CREATE TABLE partitioned_mx_2 (a INT UNIQUE); +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_2 FOR VALUES FROM (5) TO (8); +SELECT create_reference_table('local_table_fkey'); +CREATE TABLE partitioned_mx_3 (a INT UNIQUE); +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_3 FOR VALUES FROM (9) TO (12); +-- these should error out since multi-level partitioned citus local tables are not supported +CREATE TABLE IF NOT EXISTS partitioned_mx_4 PARTITION OF partitioned_mx FOR VALUES FROM (13) TO (16) PARTITION BY RANGE (a); +BEGIN; + CREATE TABLE partitioned_mx_4(a INT UNIQUE) PARTITION BY RANGE (a); + alter table partitioned_mx attach partition partitioned_mx_4 FOR VALUES FROM (13) TO (16); +END; +CREATE TABLE multi_level_p (a INT UNIQUE) PARTITION BY RANGE (a); +CREATE TABLE IF NOT EXISTS multi_level_c PARTITION OF multi_level_p FOR VALUES FROM (13) TO (16) PARTITION BY RANGE (a); +select citus_add_local_table_to_metadata('multi_level_p'); --errors +select citus_add_local_table_to_metadata('multi_level_p', true); --errors +-- try attaching a partition with an external foreign key +CREATE TABLE partitioned_mx_4 (a INT UNIQUE); +ALTER TABLE partitioned_mx_4 ADD CONSTRAINT fkey_not_inherited FOREIGN KEY (a) REFERENCES local_table_fkey2(a); +-- these two should error out +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_4 FOR VALUES FROM (13) TO (16); +ALTER TABLE partitioned_mx ATTACH PARTITION partitioned_mx_4 default; + +SELECT + nmsp_parent.nspname AS parent_schema, + parent.relname AS parent, + nmsp_child.nspname AS child_schema, + child.relname AS child +FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace +WHERE parent.relname='partitioned_mx' +ORDER BY child; + +\c - - - :worker_1_port +SELECT relname FROM pg_class WHERE relname LIKE 'partitioned_mx%' ORDER BY relname; +SELECT + nmsp_parent.nspname AS parent_schema, + parent.relname AS parent, + nmsp_child.nspname AS child_schema, + child.relname AS child +FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace +WHERE parent.relname='partitioned_mx' +ORDER BY child; + +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +SET client_min_messages TO ERROR; +DROP TABLE partitioned_mx; +RESET client_min_messages; +-- test cascading via foreign keys +CREATE TABLE cas_1 (a INT UNIQUE); +CREATE TABLE cas_par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par_1 PARTITION OF cas_par FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par_2 PARTITION OF cas_par FOR VALUES FROM (5) TO (8); +ALTER TABLE cas_par_1 ADD CONSTRAINT fkey_cas_test_1 FOREIGN KEY (a) REFERENCES cas_1(a); +CREATE TABLE cas_par2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par2_1 PARTITION OF cas_par2 FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par2_2 PARTITION OF cas_par2 FOR VALUES FROM (5) TO (8); +ALTER TABLE cas_par2_1 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES cas_1(a); +CREATE TABLE cas_par3 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par3_1 PARTITION OF cas_par3 FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par3_2 PARTITION OF cas_par3 FOR VALUES FROM (5) TO (8); +-- these two should error out as we should call the conversion from the parent +SELECT citus_add_local_table_to_metadata('cas_par2_2'); +SELECT citus_add_local_table_to_metadata('cas_par2_2', cascade_via_foreign_keys=>true); +-- these two should error out as the foreign keys are not inherited from the parent +SELECT citus_add_local_table_to_metadata('cas_par2'); +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); +-- drop the foreign keys and establish them again using the parent table +ALTER TABLE cas_par_1 DROP CONSTRAINT fkey_cas_test_1; +ALTER TABLE cas_par2_1 DROP CONSTRAINT fkey_cas_test_2; +ALTER TABLE cas_par ADD CONSTRAINT fkey_cas_test_1 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_par2 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_par3 ADD CONSTRAINT fkey_cas_test_3 FOREIGN KEY (a) REFERENCES cas_par(a); +-- this should error out as cascade_via_foreign_keys is not set to true +SELECT citus_add_local_table_to_metadata('cas_par2'); +-- this should work +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); +-- verify the partitioning hierarchy is preserved +select inhrelid::regclass from pg_inherits where inhparent='cas_par'::regclass order by 1; +-- verify the fkeys + fkeys with shard ids are created +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; +-- when all partitions are converted, there should be 40 tables and indexes +-- the individual names are not much relevant, so we only print the count +SELECT count(*) FROM pg_class WHERE relname LIKE 'cas\_%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_mx'); +-- verify on the mx worker +\c - - - :worker_1_port +-- on worker, there should be 20, since the shards are created only on the coordinator +SELECT count(*) FROM pg_class WHERE relname LIKE 'cas\_%' AND relnamespace IN + (SELECT oid FROM pg_namespace WHERE nspname = 'citus_local_tables_mx'); +-- verify that the shell foreign keys are created on the worker as well +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +-- undistribute table +-- this one should error out since we don't set the cascade option as true +SELECT undistribute_table('cas_par2'); +-- this one should work +SET client_min_messages TO WARNING; +SELECT undistribute_table('cas_par2', cascade_via_foreign_keys=>true); +-- verify the partitioning hierarchy is preserved +select inhrelid::regclass from pg_inherits where inhparent='cas_par'::regclass order by 1; +-- verify that the foreign keys with shard ids are gone, due to undistribution +select conname from pg_constraint where conname like 'fkey_cas_test%' order by conname; +-- add a non-inherited fkey and verify it fails when trying to convert +ALTER TABLE cas_par2_1 ADD CONSTRAINT fkey_cas_test_3 FOREIGN KEY (a) REFERENCES cas_1(a); +SELECT citus_add_local_table_to_metadata('cas_par2', cascade_via_foreign_keys=>true); +-- verify undistribute_table works proper for the mx worker +\c - - - :worker_1_port +SELECT relname FROM pg_class WHERE relname LIKE 'cas\_%' ORDER BY relname; +\c - - - :master_port +SET search_path TO citus_local_tables_mx; + +CREATE TABLE date_partitioned_citus_local_table_seq( measureid bigserial, eventdate date, measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); +SELECT create_time_partitions('date_partitioned_citus_local_table_seq', INTERVAL '1 month', '2022-01-01', '2021-01-01'); +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table_seq'); +DROP TABLE date_partitioned_citus_local_table_seq; +-- test sequences +CREATE TABLE par_citus_local_seq(measureid bigserial, val int) PARTITION BY RANGE(val); +CREATE TABLE par_citus_local_seq_1 PARTITION OF par_citus_local_seq FOR VALUES FROM (1) TO (4); +SELECT citus_add_local_table_to_metadata('par_citus_local_seq'); +INSERT INTO par_citus_local_seq (val) VALUES (1) RETURNING *; +INSERT INTO par_citus_local_seq (val) VALUES (2) RETURNING *; +\c - - - :worker_1_port +-- insert on the worker +INSERT INTO citus_local_tables_mx.par_citus_local_seq (val) VALUES (1) RETURNING *; +\c - - - :master_port +SET search_path TO citus_local_tables_mx; +INSERT INTO par_citus_local_seq (val) VALUES (2) RETURNING *; +SELECT undistribute_table('par_citus_local_seq'); +INSERT INTO par_citus_local_seq (val) VALUES (3) RETURNING *; +SELECT measureid FROM par_citus_local_seq ORDER BY measureid; + +-- test adding invalid foreign key to partition table +CREATE TABLE citus_local_parent_1 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE citus_local_parent_2 (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE citus_local_plain (a INT UNIQUE); +CREATE TABLE ref (a INT UNIQUE); +SELECT create_reference_table('ref'); +alter table citus_local_parent_1 add foreign key (a) references citus_local_parent_2(a); +alter table citus_local_plain add foreign key (a) references citus_local_parent_2(a); +CREATE TABLE citus_local_parent_1_child_1 PARTITION OF citus_local_parent_1 FOR VALUES FROM (3) TO (5); +CREATE TABLE citus_local_parent_1_child_2 PARTITION OF citus_local_parent_1 FOR VALUES FROM (30) TO (50); +CREATE TABLE citus_local_parent_2_child_1 PARTITION OF citus_local_parent_2 FOR VALUES FROM (40) TO (60); +-- this one should error out, since we cannot convert it to citus local table, +-- as citus local table partitions cannot have non-inherited foreign keys +alter table citus_local_parent_1_child_1 add foreign key(a) references ref(a); +-- this should work +alter table citus_local_parent_1 add constraint fkey_to_drop_test foreign key(a) references ref(a); +-- this should undistribute the table, and the entries should be gone from pg_dist_partition +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; +set client_min_messages to error; +alter table citus_local_parent_1 drop constraint fkey_to_drop_test; +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; + SELECT master_remove_distributed_table_metadata_from_workers('citus_local_table_4'::regclass::oid, 'citus_local_tables_mx', 'citus_local_table_4'); -- both workers should print 0 as master_remove_distributed_table_metadata_from_workers @@ -176,6 +350,5 @@ SELECT run_command_on_workers( $$ SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename='citus_local_table_4' $$); - -- cleanup at exit DROP SCHEMA citus_local_tables_mx CASCADE; diff --git a/src/test/regress/sql/create_citus_local_table_cascade.sql b/src/test/regress/sql/create_citus_local_table_cascade.sql index d8854f195..738073bb3 100644 --- a/src/test/regress/sql/create_citus_local_table_cascade.sql +++ b/src/test/regress/sql/create_citus_local_table_cascade.sql @@ -76,9 +76,6 @@ ROLLBACK; BEGIN; CREATE TABLE partitioned_table (col_1 INT REFERENCES local_table_1 (col_1)) PARTITION BY RANGE (col_1); - -- now that we introduced a partitioned table into our foreign key subgraph, - -- citus_add_local_table_to_metadata(cascade_via_foreign_keys) would fail for - -- partitioned_table as citus_add_local_table_to_metadata doesn't support partitioned tables SELECT citus_add_local_table_to_metadata('local_table_2', cascade_via_foreign_keys=>true); ROLLBACK; diff --git a/src/test/regress/sql/fkeys_between_local_ref.sql b/src/test/regress/sql/fkeys_between_local_ref.sql index 640a7f66a..0cdeb706a 100644 --- a/src/test/regress/sql/fkeys_between_local_ref.sql +++ b/src/test/regress/sql/fkeys_between_local_ref.sql @@ -82,11 +82,9 @@ CREATE TABLE partitioned_table_1_200_300 PARTITION OF partitioned_table_1 FOR VA INSERT INTO partitioned_table_1 SELECT i FROM generate_series(195, 205) i; ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_8 FOREIGN KEY (col_1) REFERENCES local_table_4(col_1); - --- now that we attached partitioned table to graph below errors out --- since we cannot create citus local table from partitioned tables +BEGIN; ALTER TABLE reference_table_1 ADD CONSTRAINT fkey_9 FOREIGN KEY (col_1) REFERENCES local_table_1(col_1); - +ROLLBACK; ALTER TABLE partitioned_table_1 DROP CONSTRAINT fkey_8; BEGIN; @@ -405,9 +403,9 @@ SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partit WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref') ORDER BY tablename; --- this errors out as we don't support creating citus local --- tables from partitioned tables +BEGIN; CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1); +ROLLBACK; -- they fail as col_99 does not exist CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_99) REFERENCES reference_table_1(col_1)); diff --git a/src/test/regress/sql/multi_partitioning.sql b/src/test/regress/sql/multi_partitioning.sql index d26535b37..67f7ee807 100644 --- a/src/test/regress/sql/multi_partitioning.sql +++ b/src/test/regress/sql/multi_partitioning.sql @@ -1785,6 +1785,61 @@ ROLLBACK; DROP TABLE pi_table; +-- 6) test with citus local table +select 1 from citus_add_node('localhost', :master_port, groupid=>0); +CREATE TABLE date_partitioned_citus_local_table( + measureid integer, + eventdate date, + measure_data jsonb) PARTITION BY RANGE(eventdate); + +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table'); + +-- test interval must be multiple days for date partitioned table +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '6 hours', '2022-01-01', '2021-01-01'); +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 week 1 day 1 hour', '2022-01-01', '2021-01-01'); + +-- test with various intervals +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-02-01', '2021-01-01'); + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; +ROLLBACK; + +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 week', '2022-01-01', '2021-01-01'); + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; +ROLLBACK; + +set client_min_messages to error; +DROP TABLE date_partitioned_citus_local_table; +-- also test with foreign key +CREATE TABLE date_partitioned_citus_local_table( + measureid integer, + eventdate date, + measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); + +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table'); + +-- test interval must be multiple days for date partitioned table +SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-02-01', '2021-01-01'); + +CREATE TABLE date_partitioned_citus_local_table_2( + measureid integer, + eventdate date, + measure_data jsonb, PRIMARY KEY (measureid, eventdate)) PARTITION BY RANGE(eventdate); + +SELECT citus_add_local_table_to_metadata('date_partitioned_citus_local_table_2'); +ALTER TABLE date_partitioned_citus_local_table_2 ADD CONSTRAINT fkey_1 FOREIGN KEY (measureid, eventdate) REFERENCES date_partitioned_citus_local_table(measureid, eventdate); +SELECT create_time_partitions('date_partitioned_citus_local_table_2', INTERVAL '1 day', '2021-02-01', '2021-01-01'); +-- after the above work, these should also work for creating new partitions +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table', INTERVAL '1 day', '2021-03-01', '2021-02-01'); + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; +ROLLBACK; +BEGIN; + SELECT create_time_partitions('date_partitioned_citus_local_table_2', INTERVAL '1 day', '2021-03-01', '2021-02-01'); + SELECT * FROM time_partitions WHERE parent_table = 'date_partitioned_citus_local_table'::regclass ORDER BY 3; +ROLLBACK; +set client_min_messages to notice; -- c) test drop_old_time_partitions -- 1) test with date partitioned table CREATE TABLE date_partitioned_table_to_exp (event_date date, event int) partition by range (event_date); @@ -1858,6 +1913,29 @@ SELECT partition FROM time_partitions WHERE parent_table = '"test !/ \n _dist_12 \set VERBOSITY default DROP TABLE "test !/ \n _dist_123_table_exp"; +-- 4) test with citus local tables +CREATE TABLE date_partitioned_table_to_exp (event_date date, event int) partition by range (event_date); +SELECT citus_add_local_table_to_metadata('date_partitioned_table_to_exp'); + +CREATE TABLE date_partitioned_table_to_exp_d00 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2000-01-01') TO ('2009-12-31'); +CREATE TABLE date_partitioned_table_to_exp_d10 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2010-01-01') TO ('2019-12-31'); +CREATE TABLE date_partitioned_table_to_exp_d20 PARTITION OF date_partitioned_table_to_exp FOR VALUES FROM ('2020-01-01') TO ('2029-12-31'); + +\set VERBOSITY terse + +-- expire no partitions +CALL drop_old_time_partitions('date_partitioned_table_to_exp', '1999-01-01'); +SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_table_to_exp'::regclass ORDER BY partition::text; + +-- expire 2 old partitions +CALL drop_old_time_partitions('date_partitioned_table_to_exp', '2021-01-01'); +SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_table_to_exp'::regclass ORDER BY partition::text; + +\set VERBOSITY default +DROP TABLE date_partitioned_table_to_exp; + +SELECT citus_remove_node('localhost', :master_port); + -- d) invalid tables for helper UDFs CREATE TABLE multiple_partition_column_table( event_id bigserial,