diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index a2898736d..cc2ebd33e 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -1223,7 +1223,10 @@ CreateCitusTableLike(TableConversionState *con) } else if (IsCitusTableType(con->relationId, CITUS_LOCAL_TABLE)) { - CreateCitusLocalTable(con->newRelationId, false); + CitusTableCacheEntry *entry = GetCitusTableCacheEntry(con->relationId); + bool autoConverted = entry->autoConverted; + bool cascade = false; + CreateCitusLocalTable(con->newRelationId, cascade, autoConverted); /* * creating Citus local table adds a shell table on top diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index e048c4d64..fb42348e2 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -87,7 +87,8 @@ CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, { LockRelationsWithLockMode(relationIdList, lockMode); - if (cascadeOperationType == CASCADE_ADD_LOCAL_TABLE_TO_METADATA) + if (cascadeOperationType == CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA || + cascadeOperationType == CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA) { /* * In CreateCitusLocalTable function, this check would never error out, @@ -474,11 +475,25 @@ ExecuteCascadeOperationForRelationIdList(List *relationIdList, break; } - case CASCADE_ADD_LOCAL_TABLE_TO_METADATA: + case CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA: { if (!IsCitusTable(relationId)) { - CreateCitusLocalTable(relationId, cascadeViaForeignKeys); + bool autoConverted = false; + CreateCitusLocalTable(relationId, cascadeViaForeignKeys, + autoConverted); + } + + break; + } + + case CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA: + { + if (!IsCitusTable(relationId)) + { + bool autoConverted = true; + CreateCitusLocalTable(relationId, cascadeViaForeignKeys, + autoConverted); } break; diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index 8218c68da..785501dc3 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -52,6 +52,9 @@ static void ErrorIfAddingPartitionTableToMetadata(Oid relationId); static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation); static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId); static void ErrorIfUnsupportedCitusLocalColumnDefinition(Relation relation); +static void NoticeIfAutoConvertingLocalTables(bool autoConverted); +static void NoticeRelationIsAlreadyAddedToMetadata(Oid relationId); +static CascadeOperationType GetCascadeTypeForCitusLocalTables(bool autoConverted); static List * GetShellTableDDLEventsForCitusLocalTable(Oid relationId); static uint64 ConvertLocalTableToShard(Oid relationId); static void RenameRelationToShardRelation(Oid shellRelationId, uint64 shardId); @@ -79,7 +82,8 @@ static void DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelat static void DropDefaultColumnDefinition(Oid relationId, char *columnName); static void TransferSequenceOwnership(Oid ownedSequenceId, Oid targetRelationId, char *columnName); -static void InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId); +static void InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId, + bool autoConverted); static void FinalizeCitusLocalTableCreation(Oid relationId, List *dependentSequenceList); @@ -114,23 +118,8 @@ citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeig { CheckCitusVersion(ERROR); - if (ShouldEnableLocalReferenceForeignKeys()) - { - /* - * When foreign keys between reference tables and postgres tables are - * enabled, we automatically undistribute citus local tables that are - * not chained with any reference tables back to postgres tables. - * So give a warning to user for that. - */ - ereport(WARNING, (errmsg("local tables that are added to metadata but not " - "chained with reference tables via foreign keys might " - "be automatically converted back to postgres tables"), - errhint("Consider setting " - "citus.enable_local_reference_table_foreign_keys " - "to 'off' to disable this behavior"))); - } - - CreateCitusLocalTable(relationId, cascadeViaForeignKeys); + bool autoConverted = false; + CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted); } @@ -190,7 +179,7 @@ remove_local_tables_from_metadata(PG_FUNCTION_ARGS) * single placement is only allowed to be on the coordinator. */ void -CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) +CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConverted) { /* * These checks should be done before acquiring any locks on relation. @@ -212,6 +201,23 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); + if (!autoConverted && IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) + { + /* + * We allow users to mark local tables already added to metadata + * as "autoConverted = false". + * If the user called citus_add_local_table_to_metadata for a table that is + * already added to metadata, we should mark this one and connected relations + * as auto-converted = false. + */ + UpdateAutoConvertedForConnectedRelations(list_make1_oid(relationId), + autoConverted); + + NoticeRelationIsAlreadyAddedToMetadata(relationId); + + return; + } + /* * Lock target relation with an AccessExclusiveLock as we don't want * multiple backends manipulating this relation. We could actually simply @@ -236,6 +242,8 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) */ relation_close(relation, NoLock); + NoticeIfAutoConvertingLocalTables(autoConverted); + if (TableHasExternalForeignKeys(relationId)) { if (!cascadeViaForeignKeys) @@ -259,12 +267,14 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) qualifiedRelationName, qualifiedRelationName))); } + CascadeOperationType cascadeType = + GetCascadeTypeForCitusLocalTables(autoConverted); + /* * By acquiring AccessExclusiveLock, make sure that no modifications happen * on the relations. */ - CascadeOperationForFkeyConnectedRelations(relationId, lockMode, - CASCADE_ADD_LOCAL_TABLE_TO_METADATA); + CascadeOperationForFkeyConnectedRelations(relationId, lockMode, cascadeType); /* * We converted every foreign key connected table in our subgraph @@ -280,8 +290,11 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) { relationList = lappend_oid(relationList, relationId); + CascadeOperationType cascadeType = + GetCascadeTypeForCitusLocalTables(autoConverted); + CascadeOperationForRelationIdList(relationList, AccessExclusiveLock, - CASCADE_ADD_LOCAL_TABLE_TO_METADATA); + cascadeType); return; } } @@ -335,7 +348,7 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(shardRelationId, shellRelationId); - InsertMetadataForCitusLocalTable(shellRelationId, shardId); + InsertMetadataForCitusLocalTable(shellRelationId, shardId, autoConverted); /* * Ensure that the sequences used in column defaults of the table @@ -403,7 +416,13 @@ CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, * again with the attach command */ DropRelationForeignKeys(relationId, fKeyFlags); - CreateCitusLocalTable(relationId, false); + + /* get the autoconverted field from the parent */ + CitusTableCacheEntry *entry = GetCitusTableCacheEntry(parentRelationId); + + bool cascade = false; + bool autoConverted = entry->autoConverted; + CreateCitusLocalTable(relationId, cascade, autoConverted); ExecuteAndLogUtilityCommand(attachCommand); } @@ -540,6 +559,95 @@ ErrorIfUnsupportedCitusLocalColumnDefinition(Relation relation) } +/* + * NoticeIfAutoConvertingLocalTables logs a NOTICE message to inform the user that we are + * automatically adding local tables to metadata. The user should know that this table + * will be undistributed automatically, if it gets disconnected from reference table(s). + */ +static void +NoticeIfAutoConvertingLocalTables(bool autoConverted) +{ + if (autoConverted && ShouldEnableLocalReferenceForeignKeys()) + { + /* + * When foreign keys between reference tables and postgres tables are + * enabled, we automatically undistribute citus local tables that are + * not chained with any reference tables back to postgres tables. + * So give a warning to user for that. + */ + ereport(NOTICE, (errmsg("local tables that are added to metadata but not " + "chained with reference tables via foreign keys might " + "be automatically converted back to postgres tables"), + errhint("Consider setting " + "citus.enable_local_reference_table_foreign_keys " + "to 'off' to disable this behavior"))); + } +} + + +/* + * NoticeRelationIsAlreadyAddedToMetadata logs a NOTICE message that informs the user + * that the given relation is already added to metadata. + * We set the field autoConverted for these cases to false. + * This function tells the user that the given table will not be removed from metadata, + * as it was a user request. + */ +static void +NoticeRelationIsAlreadyAddedToMetadata(Oid relationId) +{ + char *relname = get_rel_name(relationId); + ereport(NOTICE, (errmsg("relation \"%s\" is already added to metadata", relname), + errdetail("This relation will not be removed from metadata even if " + "it is not connected to a reference table via foreign " + "key(s), since it is added to metadata by the user"))); +} + + +/* + * GetCascadeTypeForCitusLocalTables returns CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA + * if autoConverted is true. Returns CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA otherwise. + */ +static CascadeOperationType +GetCascadeTypeForCitusLocalTables(bool autoConverted) +{ + if (autoConverted) + { + return CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA; + } + + return CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA; +} + + +/* + * UpdateAutoConvertedForConnectedRelations updates the autoConverted field on + * pg_dist_partition for the foreign key connected relations of the given relations. + * Sets it to given autoConverted value for all of the connected relations. + * We don't need to update partition relations separately, since the foreign key + * graph already includes them, as they have the same (inherited) fkeys as their parents. + */ +void +UpdateAutoConvertedForConnectedRelations(List *relationIds, bool autoConverted) +{ + InvalidateForeignKeyGraph(); + + List *relationIdList = NIL; + Oid relid = InvalidOid; + foreach_oid(relid, relationIds) + { + List *connectedRelations = GetForeignKeyConnectedRelationIdList(relid); + relationIdList = list_concat_unique_oid(relationIdList, connectedRelations); + } + + relationIdList = SortList(relationIdList, CompareOids); + + foreach_oid(relid, relationIdList) + { + UpdatePgDistPartitionAutoConverted(relid, false); + } +} + + /* * GetShellTableDDLEventsForCitusLocalTable returns a list of DDL commands * to create the shell table from scratch. @@ -1090,7 +1198,8 @@ TransferSequenceOwnership(Oid sequenceId, Oid targetRelationId, char *targetColu * pg_dist_partition, pg_dist_shard & pg_dist_placement. */ static void -InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId) +InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId, + bool autoConverted) { Assert(OidIsValid(citusLocalTableId)); Assert(shardId != INVALID_SHARD_ID); @@ -1102,7 +1211,7 @@ InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId) Var *distributionColumn = NULL; InsertIntoPgDistPartition(citusLocalTableId, distributionMethod, distributionColumn, colocationId, - replicationModel); + replicationModel, autoConverted); /* set shard storage type according to relation type */ char shardStorageType = ShardStorageType(citusLocalTableId); diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 89d14f2af..2b6544e24 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -483,9 +483,12 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio bool localTableEmpty = TableEmpty(relationId); Oid colocatedTableId = ColocatedTableId(colocationId); + /* setting to false since this flag is only valid for citus local tables */ + bool autoConverted = false; + /* create an entry for distributed table in pg_dist_partition */ InsertIntoPgDistPartition(relationId, distributionMethod, distributionColumn, - colocationId, replicationModel); + colocationId, replicationModel, autoConverted); /* * Ensure that the sequences used in column defaults of the table diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 8ea36fdeb..6011977cc 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -24,11 +24,12 @@ #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/deparse_shard_query.h" #include "distributed/distribution_column.h" +#include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" -#include "distributed/coordinator_protocol.h" #include "distributed/metadata_sync.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" @@ -69,11 +70,15 @@ static void ErrorIfAttachCitusTableToPgLocalTable(Oid parentRelationId, Oid partitionRelationId); static bool AlterTableDefinesFKeyBetweenPostgresAndNonDistTable( AlterTableStmt *alterTableStatement); +static bool ShouldMarkConnectedRelationsNotAutoConverted(Oid leftRelationId, + Oid rightRelationId); static bool RelationIdListContainsCitusTableType(List *relationIdList, CitusTableType citusTableType); static bool RelationIdListContainsPostgresTable(List *relationIdList); static void ConvertPostgresLocalTablesToCitusLocalTables( AlterTableStmt *alterTableStatement); +static bool RangeVarListHasRelationConvertedByUser(List *relationRangeVarList, + AlterTableStmt *alterTableStatement); static int CompareRangeVarsByOid(const void *leftElement, const void *rightElement); static List * GetAlterTableAddFKeyRightRelationIdList( AlterTableStmt *alterTableStatement); @@ -491,7 +496,10 @@ PreprocessAttachPartitionToCitusTable(Oid parentRelationId, Oid partitionRelatio * cannot have non-inherited foreign keys. */ bool cascadeViaForeignKeys = false; - CreateCitusLocalTable(partitionRelationId, cascadeViaForeignKeys); + CitusTableCacheEntry *entry = GetCitusTableCacheEntry(parentRelationId); + bool autoConverted = entry->autoConverted; + CreateCitusLocalTable(partitionRelationId, cascadeViaForeignKeys, + autoConverted); } else if (IsCitusTableType(parentRelationId, DISTRIBUTED_TABLE)) { @@ -858,6 +866,14 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, rightRelationId = RangeVarGetRelid(constraint->pktable, lockmode, alterTableStatement->missing_ok); + if (processUtilityContext != PROCESS_UTILITY_SUBCOMMAND && + ShouldMarkConnectedRelationsNotAutoConverted(leftRelationId, + rightRelationId)) + { + List *relationList = list_make2_oid(leftRelationId, rightRelationId); + UpdateAutoConvertedForConnectedRelations(relationList, false); + } + /* * Foreign constraint validations will be done in workers. If we do not * set this flag, PostgreSQL tries to do additional checking when we drop @@ -1170,6 +1186,31 @@ AlterTableDefinesFKeyBetweenPostgresAndNonDistTable(AlterTableStmt *alterTableSt } +/* + * ShouldMarkConnectedRelationsNotAutoConverted takes two relations. + * If both of them are Citus Local Tables, and one of them is auto-converted while the + * other one is not; then it returns true. False otherwise. + */ +static bool +ShouldMarkConnectedRelationsNotAutoConverted(Oid leftRelationId, Oid rightRelationId) +{ + if (!IsCitusTableType(leftRelationId, CITUS_LOCAL_TABLE)) + { + return false; + } + + if (!IsCitusTableType(rightRelationId, CITUS_LOCAL_TABLE)) + { + return false; + } + + CitusTableCacheEntry *entryLeft = GetCitusTableCacheEntry(leftRelationId); + CitusTableCacheEntry *entryRight = GetCitusTableCacheEntry(rightRelationId); + + return entryLeft->autoConverted != entryRight->autoConverted; +} + + /* * RelationIdListContainsCitusTableType returns true if given relationIdList * contains a citus table with given type. @@ -1229,6 +1270,13 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement */ relationRangeVarList = SortList(relationRangeVarList, CompareRangeVarsByOid); + bool autoConverted = true; + + if (RangeVarListHasRelationConvertedByUser(relationRangeVarList, alterTableStatement)) + { + autoConverted = false; + } + /* * Here we should operate on RangeVar objects since relations oid's would * change in below loop due to CreateCitusLocalTable. @@ -1248,14 +1296,30 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement */ continue; } + else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) + { + CitusTableCacheEntry *entry = GetCitusTableCacheEntry(relationId); + if (!entry->autoConverted || autoConverted) + { + /* + * relationRangeVarList has also reference and citus local tables + * involved in this ADD FOREIGN KEY command. Moreover, even if + * relationId was belonging to a postgres local table initially, + * we might had already converted it to a citus local table by cascading. + * Note that we cannot skip here, if the relation is marked as + * auto-converted, but we are marking the relations in the command + * auto-converted = false. Because in that case, it means that this + * relation is marked as auto-converted because of a connection + * with a reference table, but now it is connected to a Citus + * Local Table. In that case, we need to mark this relation as + * auto-converted = false, so cannot do a "continue" here. + */ + continue; + } + } else if (IsCitusTable(relationId)) { - /* - * relationRangeVarList has also reference and citus local tables - * involved in this ADD FOREIGN KEY command. Moreover, even if - * relationId was belonging to a postgres local table initially, - * we might had already converted it to a citus local table by cascading. - */ + /* we can directly skip for table types other than citus local tables */ continue; } @@ -1289,7 +1353,7 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement } else { - CreateCitusLocalTable(relationId, cascade); + CreateCitusLocalTable(relationId, cascade, autoConverted); } } PG_CATCH(); @@ -1315,6 +1379,43 @@ ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement } +/* + * RangeVarListHasRelationConvertedByUser takes a list of relations and returns true + * if any of these relations is marked as auto-converted = false. Returns true otherwise. + * This function also takes the current alterTableStatement command, to obtain the + * necessary locks. + */ +static bool +RangeVarListHasRelationConvertedByUser(List *relationRangeVarList, + AlterTableStmt *alterTableStatement) +{ + RangeVar *relationRangeVar; + foreach_ptr(relationRangeVar, relationRangeVarList) + { + /* + * Here we iterate the relation list, and if at least one of the relations + * is marked as not-auto-converted, we should mark all of them as + * not-auto-converted. In that case, we return true here. + */ + List *commandList = alterTableStatement->cmds; + LOCKMODE lockMode = AlterTableGetLockLevel(commandList); + bool missingOk = alterTableStatement->missing_ok; + Oid relationId = RangeVarGetRelid(relationRangeVar, lockMode, missingOk); + if (OidIsValid(relationId) && IsCitusTable(relationId) && + IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) + { + CitusTableCacheEntry *entry = GetCitusTableCacheEntry(relationId); + if (!entry->autoConverted) + { + return true; + } + } + } + + return false; +} + + /* * CompareRangeVarsByOid is a comparison function to sort RangeVar object list. */ diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 29bc2df01..f0ce9ae47 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -95,7 +95,7 @@ static void IncrementUtilityHookCountersIfNecessary(Node *parsetree); static void PostStandardProcessUtility(Node *parsetree); static void DecrementUtilityHookCountersIfNecessary(Node *parsetree); static bool IsDropSchemaOrDB(Node *parsetree); -static bool ShouldUndistributeCitusLocalTables(void); +static bool ShouldCheckUndistributeCitusLocalTables(void); /* @@ -271,7 +271,7 @@ multi_ProcessUtility(PlannedStmt *pstmt, * can happen due to various kinds of drop commands, we immediately * undistribute them at the end of the command. */ - if (ShouldUndistributeCitusLocalTables()) + if (ShouldCheckUndistributeCitusLocalTables()) { UndistributeDisconnectedCitusLocalTables(); } @@ -687,7 +687,8 @@ ProcessUtilityInternal(PlannedStmt *pstmt, /* * UndistributeDisconnectedCitusLocalTables undistributes citus local tables that * are not connected to any reference tables via their individual foreign key - * subgraphs. + * subgraphs. Note that this function undistributes only the auto-converted tables, + * i.e the ones that are converted by Citus by cascading through foreign keys. */ void UndistributeDisconnectedCitusLocalTables(void) @@ -717,10 +718,11 @@ UndistributeDisconnectedCitusLocalTables(void) if (PartitionTable(citusLocalTableId)) { /* we skip here, we'll undistribute from the parent if necessary */ + UnlockRelationOid(citusLocalTableId, lockMode); continue; } - if (ConnectedToReferenceTableViaFKey(citusLocalTableId)) + if (!ShouldUndistributeCitusLocalTable(citusLocalTableId)) { /* still connected to a reference table, skip it */ UnlockRelationOid(citusLocalTableId, lockMode); @@ -751,11 +753,11 @@ UndistributeDisconnectedCitusLocalTables(void) /* - * ShouldUndistributeCitusLocalTables returns true if we might need to check - * citus local tables for their connectivity to reference tables. + * ShouldCheckUndistributeCitusLocalTables returns true if we might need to check + * citus local tables for undistributing automatically. */ static bool -ShouldUndistributeCitusLocalTables(void) +ShouldCheckUndistributeCitusLocalTables(void) { if (!ConstraintDropped) { diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index a84009a1a..789601d44 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -1387,6 +1387,20 @@ BuildCitusTableCacheEntry(Oid relationId) cacheEntry->replicationModel = DatumGetChar(replicationModelDatum); } + if (isNullArray[Anum_pg_dist_partition_autoconverted - 1]) + { + /* + * We don't expect this to happen, but set it to false (the default value) + * to not break if anything goes wrong. + */ + cacheEntry->autoConverted = false; + } + else + { + cacheEntry->autoConverted = DatumGetBool( + datumArray[Anum_pg_dist_partition_autoconverted - 1]); + } + heap_freetuple(distPartitionTuple); BuildCachedShardList(cacheEntry); @@ -3653,6 +3667,7 @@ ResetCitusTableCacheEntry(CitusTableCacheEntry *cacheEntry) cacheEntry->hasUninitializedShardInterval = false; cacheEntry->hasUniformHashDistribution = false; cacheEntry->hasOverlappingShardInterval = false; + cacheEntry->autoConverted = false; pfree(cacheEntry); } @@ -3956,23 +3971,19 @@ CitusTableTypeIdList(CitusTableType citusTableType) HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { - bool isNull = false; + bool isNullArray[Natts_pg_dist_partition]; + Datum datumArray[Natts_pg_dist_partition]; + heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); - Datum partMethodDatum = - heap_getattr(heapTuple, Anum_pg_dist_partition_partmethod, - tupleDescriptor, &isNull); - Datum replicationModelDatum = - heap_getattr(heapTuple, Anum_pg_dist_partition_repmodel, - tupleDescriptor, &isNull); + Datum partMethodDatum = datumArray[Anum_pg_dist_partition_partmethod - 1]; + Datum replicationModelDatum = datumArray[Anum_pg_dist_partition_repmodel - 1]; Oid partitionMethod = DatumGetChar(partMethodDatum); Oid replicationModel = DatumGetChar(replicationModelDatum); if (IsCitusTableTypeInternal(partitionMethod, replicationModel, citusTableType)) { - Datum relationIdDatum = heap_getattr(heapTuple, - Anum_pg_dist_partition_logicalrelid, - tupleDescriptor, &isNull); + Datum relationIdDatum = datumArray[Anum_pg_dist_partition_logicalrelid - 1]; Oid relationId = DatumGetObjectId(relationIdDatum); diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 2e9f6489b..6cc2b6ea5 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -2075,6 +2075,9 @@ citus_internal_add_partition_metadata(PG_FUNCTION_ARGS) char *distributionColumnString = NULL; Var *distributionColumnVar = NULL; + /* this flag is only valid for citus local tables, so set it to false */ + bool autoConverted = false; + /* only owner of the table (or superuser) is allowed to add the Citus metadata */ EnsureTableOwner(relationId); @@ -2123,7 +2126,7 @@ citus_internal_add_partition_metadata(PG_FUNCTION_ARGS) } InsertIntoPgDistPartition(relationId, distributionMethod, distributionColumnVar, - colocationId, replicationModel); + colocationId, replicationModel, autoConverted); PG_RETURN_VOID(); } diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index ddd8b49bc..9522656a6 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -1744,7 +1744,7 @@ InsertShardPlacementRow(uint64 shardId, uint64 placementId, void InsertIntoPgDistPartition(Oid relationId, char distributionMethod, Var *distributionColumn, uint32 colocationId, - char replicationModel) + char replicationModel, bool autoConverted) { char *distributionColumnString = NULL; @@ -1764,6 +1764,7 @@ InsertIntoPgDistPartition(Oid relationId, char distributionMethod, CharGetDatum(distributionMethod); newValues[Anum_pg_dist_partition_colocationid - 1] = UInt32GetDatum(colocationId); newValues[Anum_pg_dist_partition_repmodel - 1] = CharGetDatum(replicationModel); + newValues[Anum_pg_dist_partition_autoconverted - 1] = BoolGetDatum(autoConverted); /* set partkey column to NULL for reference tables */ if (distributionMethod != DISTRIBUTE_BY_NONE) @@ -2069,6 +2070,56 @@ UpdatePlacementGroupId(uint64 placementId, int groupId) } +/* + * UpdatePgDistPartitionAutoConverted sets the autoConverted for the partition identified + * by citusTableId. + */ +void +UpdatePgDistPartitionAutoConverted(Oid citusTableId, bool autoConverted) +{ + ScanKeyData scanKey[1]; + int scanKeyCount = 1; + bool indexOK = true; + Datum values[Natts_pg_dist_partition]; + bool isnull[Natts_pg_dist_partition]; + bool replace[Natts_pg_dist_partition]; + + Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); + TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); + ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(citusTableId)); + + SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, + DistPartitionLogicalRelidIndexId(), + indexOK, + NULL, scanKeyCount, scanKey); + + HeapTuple heapTuple = systable_getnext(scanDescriptor); + if (!HeapTupleIsValid(heapTuple)) + { + ereport(ERROR, (errmsg("could not find valid entry for citus table with oid: %u", + citusTableId))); + } + + memset(replace, 0, sizeof(replace)); + + values[Anum_pg_dist_partition_autoconverted - 1] = BoolGetDatum(autoConverted); + isnull[Anum_pg_dist_partition_autoconverted - 1] = false; + replace[Anum_pg_dist_partition_autoconverted - 1] = true; + + heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); + + CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); + + CitusInvalidateRelcacheByRelid(citusTableId); + + CommandCounterIncrement(); + + systable_endscan(scanDescriptor); + table_close(pgDistPartition, NoLock); +} + + /* * Check that the current user has `mode` permissions on relationId, error out * if not. Superusers always have such permissions. diff --git a/src/backend/distributed/sql/citus--10.2-3--11.0-1.sql b/src/backend/distributed/sql/citus--10.2-3--11.0-1.sql index b202061db..9210b7e98 100644 --- a/src/backend/distributed/sql/citus--10.2-3--11.0-1.sql +++ b/src/backend/distributed/sql/citus--10.2-3--11.0-1.sql @@ -8,3 +8,4 @@ DROP FUNCTION IF EXISTS pg_catalog.master_apply_delete_command(text); DROP FUNCTION pg_catalog.master_get_table_metadata(text); +ALTER TABLE pg_catalog.pg_dist_partition ADD COLUMN autoconverted boolean DEFAULT false; diff --git a/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-3.sql b/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-3.sql index 491e42633..dd3c6a03c 100644 --- a/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-3.sql +++ b/src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-3.sql @@ -24,3 +24,4 @@ CREATE FUNCTION pg_catalog.master_get_table_metadata( AS 'MODULE_PATHNAME', $$master_get_table_metadata$$; COMMENT ON FUNCTION master_get_table_metadata(relation_name text) IS 'fetch metadata values for the table'; +ALTER TABLE pg_catalog.pg_dist_partition DROP COLUMN autoconverted; diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index 51d8f9fbd..4edc9e424 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -947,10 +947,11 @@ ColocationGroupTableList(uint32 colocationId, uint32 count) HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { - bool isNull = false; - Oid colocatedTableId = heap_getattr(heapTuple, - Anum_pg_dist_partition_logicalrelid, - tupleDescriptor, &isNull); + bool isNullArray[Natts_pg_dist_partition]; + Datum datumArray[Natts_pg_dist_partition]; + heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); + Oid colocatedTableId = DatumGetObjectId( + datumArray[Anum_pg_dist_partition_logicalrelid - 1]); colocatedTableList = lappend_oid(colocatedTableList, colocatedTableId); heapTuple = systable_getnext(scanDescriptor); @@ -1116,7 +1117,6 @@ ColocatedTableId(Oid colocationId) { Oid colocatedTableId = InvalidOid; bool indexOK = true; - bool isNull = false; ScanKeyData scanKey[1]; int scanKeyCount = 1; @@ -1141,8 +1141,11 @@ ColocatedTableId(Oid colocationId) HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { - colocatedTableId = heap_getattr(heapTuple, Anum_pg_dist_partition_logicalrelid, - tupleDescriptor, &isNull); + bool isNullArray[Natts_pg_dist_partition]; + Datum datumArray[Natts_pg_dist_partition]; + heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); + colocatedTableId = DatumGetObjectId( + datumArray[Anum_pg_dist_partition_logicalrelid - 1]); /* * Make sure the relation isn't dropped for the remainder of diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index 8996feb15..84ff21f8c 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -139,12 +139,23 @@ GetForeignKeyConnectedRelationIdList(Oid relationId) /* - * ConnectedToReferenceTableViaFKey returns true if given relationId is - * connected to a reference table via its foreign key subgraph. + * ShouldUndistributeCitusLocalTable returns true if given relationId needs + * to be undistributed. Here we do not undistribute table if it's converted by the user, + * or connected to a table converted by the user, or a reference table, via foreign keys. */ bool -ConnectedToReferenceTableViaFKey(Oid relationId) +ShouldUndistributeCitusLocalTable(Oid relationId) { + CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); + if (!cacheEntry->autoConverted) + { + /* + * The relation is not added to metadata automatically, + * we shouldn't undistribute it. + */ + return false; + } + /* * As we will operate on foreign key connected relations, here we * invalidate foreign key graph so that we act on fresh graph. @@ -152,7 +163,8 @@ ConnectedToReferenceTableViaFKey(Oid relationId) InvalidateForeignKeyGraph(); List *fkeyConnectedRelations = GetForeignKeyConnectedRelationIdList(relationId); - return RelationIdListHasReferenceTable(fkeyConnectedRelations); + + return !RelationIdListHasReferenceTable(fkeyConnectedRelations); } diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index ba8ee6d31..9c0c86c5b 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -512,8 +512,11 @@ typedef enum CascadeOperationType /* execute UndistributeTable on each relation */ CASCADE_FKEY_UNDISTRIBUTE_TABLE = 1 << 1, - /* execute CreateCitusLocalTable on each relation */ - CASCADE_ADD_LOCAL_TABLE_TO_METADATA = 1 << 2, + /* execute CreateCitusLocalTable on each relation, with autoConverted = false */ + CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA = 1 << 2, + + /* execute CreateCitusLocalTable on each relation, with autoConverted = true */ + CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA = 1 << 3, } CascadeOperationType; extern void CascadeOperationForFkeyConnectedRelations(Oid relationId, @@ -533,16 +536,16 @@ extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, bool skip_validation); /* create_citus_local_table.c */ -extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys); +extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, + bool autoConverted); extern List * GetExplicitIndexOidList(Oid relationId); extern bool ShouldPropagateSetCommand(VariableSetStmt *setStmt); extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *setCommand); -/* create_citus_local_table.c */ - -extern void CreateCitusLocalTable(Oid relationId, bool cascade); extern void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, Oid parentRelationId); +extern void UpdateAutoConvertedForConnectedRelations(List *relationId, bool + autoConverted); #endif /*CITUS_COMMANDS_H */ diff --git a/src/include/distributed/foreign_key_relationship.h b/src/include/distributed/foreign_key_relationship.h index 491142d13..6e476162e 100644 --- a/src/include/distributed/foreign_key_relationship.h +++ b/src/include/distributed/foreign_key_relationship.h @@ -16,7 +16,7 @@ #include "nodes/primnodes.h" extern List * GetForeignKeyConnectedRelationIdList(Oid relationId); -extern bool ConnectedToReferenceTableViaFKey(Oid relationId); +extern bool ShouldUndistributeCitusLocalTable(Oid relationId); extern List * ReferencedRelationIdList(Oid relationId); extern List * ReferencingRelationIdList(Oid relationId); extern void SetForeignConstraintRelationshipGraphInvalid(void); diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index fc76c47e4..c87db457d 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -63,6 +63,7 @@ typedef struct char partitionMethod; uint32 colocationId; char replicationModel; + bool autoConverted; /* table auto-added to metadata, valid for citus local tables */ /* pg_dist_shard metadata (variable-length ShardInterval array) for this table */ int shardIntervalArrayLength; diff --git a/src/include/distributed/metadata_utility.h b/src/include/distributed/metadata_utility.h index 0e4a6e6c8..742aec20a 100644 --- a/src/include/distributed/metadata_utility.h +++ b/src/include/distributed/metadata_utility.h @@ -234,7 +234,8 @@ extern uint64 InsertShardPlacementRow(uint64 shardId, uint64 placementId, int32 groupId); extern void InsertIntoPgDistPartition(Oid relationId, char distributionMethod, Var *distributionColumn, uint32 colocationId, - char replicationModel); + char replicationModel, bool autoConverted); +extern void UpdatePgDistPartitionAutoConverted(Oid citusTableId, bool autoConverted); extern void DeletePartitionRow(Oid distributedRelationId); extern void DeleteShardRow(uint64 shardId); extern void UpdateShardPlacementState(uint64 placementId, char shardState); diff --git a/src/include/distributed/pg_dist_partition.h b/src/include/distributed/pg_dist_partition.h index 36a94da7a..032cc195a 100644 --- a/src/include/distributed/pg_dist_partition.h +++ b/src/include/distributed/pg_dist_partition.h @@ -28,6 +28,7 @@ typedef struct FormData_pg_dist_partition uint32 colocationid; /* id of the co-location group of particular table belongs to */ char repmodel; /* replication model; see codes below */ #endif + bool autoconverted; } FormData_pg_dist_partition; /* ---------------- @@ -41,12 +42,13 @@ typedef FormData_pg_dist_partition *Form_pg_dist_partition; * compiler constants for pg_dist_partitions * ---------------- */ -#define Natts_pg_dist_partition 5 +#define Natts_pg_dist_partition 6 #define Anum_pg_dist_partition_logicalrelid 1 #define Anum_pg_dist_partition_partmethod 2 #define Anum_pg_dist_partition_partkey 3 #define Anum_pg_dist_partition_colocationid 4 #define Anum_pg_dist_partition_repmodel 5 +#define Anum_pg_dist_partition_autoconverted 6 /* valid values for partmethod include append, hash, and range */ #define DISTRIBUTE_BY_APPEND 'a' diff --git a/src/test/regress/after_pg_upgrade_schedule b/src/test/regress/after_pg_upgrade_schedule index 8e3a87fbe..524ff6e72 100644 --- a/src/test/regress/after_pg_upgrade_schedule +++ b/src/test/regress/after_pg_upgrade_schedule @@ -1 +1 @@ -test: upgrade_basic_after upgrade_columnar_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects +test: upgrade_basic_after upgrade_columnar_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects upgrade_autoconverted_after diff --git a/src/test/regress/before_pg_upgrade_schedule b/src/test/regress/before_pg_upgrade_schedule index beb14b0af..be691bed1 100644 --- a/src/test/regress/before_pg_upgrade_schedule +++ b/src/test/regress/before_pg_upgrade_schedule @@ -7,3 +7,4 @@ test: upgrade_columnar_before test: upgrade_ref2ref_before test: upgrade_type_before test: upgrade_distributed_function_before upgrade_rebalance_strategy_before +test: upgrade_autoconverted_before diff --git a/src/test/regress/citus_tests/config.py b/src/test/regress/citus_tests/config.py index b4bdb9991..41aa2bfd0 100644 --- a/src/test/regress/citus_tests/config.py +++ b/src/test/regress/citus_tests/config.py @@ -343,3 +343,4 @@ class PGUpgradeConfig(CitusBaseClusterConfig): self.old_datadir = self.temp_dir + "/oldData" self.new_datadir = self.temp_dir + "/newData" self.user = SUPER_USER_NAME + self.add_coordinator_to_metadata = True diff --git a/src/test/regress/expected/auto_undist_citus_local.out b/src/test/regress/expected/auto_undist_citus_local.out index 260264f9c..71c614c79 100644 --- a/src/test/regress/expected/auto_undist_citus_local.out +++ b/src/test/regress/expected/auto_undist_citus_local.out @@ -13,12 +13,6 @@ SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); -- show that DROP CONSTRAINT cascades to undistributing citus_local_table CREATE TABLE citus_local_table(l1 int); -SELECT citus_add_local_table_to_metadata('citus_local_table'); - citus_add_local_table_to_metadata ---------------------------------------------------------------------- - -(1 row) - CREATE TABLE reference_table(r1 int primary key); SELECT create_reference_table('reference_table'); create_reference_table @@ -30,8 +24,8 @@ ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) R SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid; logicalrelid | partmethod | repmodel --------------------------------------------------------------------- - citus_local_table | n | s reference_table | n | t + citus_local_table | n | s (2 rows) CREATE OR REPLACE FUNCTION drop_constraint_cascade_via_perform_deletion(IN table_name regclass, IN constraint_name text) @@ -50,8 +44,8 @@ BEGIN; SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid; logicalrelid | partmethod | repmodel --------------------------------------------------------------------- - citus_local_table | n | s reference_table | n | t + citus_local_table | n | s (2 rows) ROLLBACK; @@ -454,6 +448,737 @@ SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalre reference_table_1 | n | t (1 row) +-- verify that citus local tables converted by the user will not be auto-undistributed +DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3; +CREATE TABLE citus_local_table_1(a INT UNIQUE); +CREATE TABLE citus_local_table_2(a INT UNIQUE); +CREATE TABLE citus_local_table_3(a INT UNIQUE); +ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_cas_test FOREIGN KEY (a) REFERENCES citus_local_table_2 (a); +SELECT citus_add_local_table_to_metadata('citus_local_table_1', cascade_via_foreign_keys=>true); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT citus_add_local_table_to_metadata('citus_local_table_3'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE citus_local_table_3 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES citus_local_table_2 (a); +ALTER TABLE citus_local_table_3 DROP CONSTRAINT fkey_cas_test_2; +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_table_1'::regclass, + 'citus_local_table_2'::regclass, + 'citus_local_table_3'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_table_1 | f + citus_local_table_2 | f + citus_local_table_3 | f +(3 rows) + +ALTER TABLE citus_local_table_1 DROP CONSTRAINT fkey_cas_test; +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_table_1'::regclass, + 'citus_local_table_2'::regclass, + 'citus_local_table_3'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_table_1 | f + citus_local_table_2 | f + citus_local_table_3 | f +(3 rows) + +-- verify that tables that are connected to reference tables are marked as autoConverted = true +CREATE TABLE ref_test(a int UNIQUE); +SELECT create_reference_table('ref_test'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE auto_local_table_1(a int UNIQUE); +CREATE TABLE auto_local_table_2(a int UNIQUE REFERENCES auto_local_table_1(a)); +ALTER TABLE auto_local_table_1 ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (a) REFERENCES ref_test(a); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('auto_local_table_1'::regclass, + 'auto_local_table_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + auto_local_table_1 | t + auto_local_table_2 | t +(2 rows) + +-- verify that we can mark both of them with autoConverted = false, by converting one of them manually +SELECT citus_add_local_table_to_metadata('auto_local_table_1'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('auto_local_table_1'::regclass, + 'auto_local_table_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + auto_local_table_1 | f + auto_local_table_2 | f +(2 rows) + +-- test with partitioned tables +CREATE TABLE partitioned_table_1 (a int unique) partition by range(a); +CREATE TABLE partitioned_table_1_1 partition of partitioned_table_1 FOR VALUES FROM (0) TO (10); +CREATE TABLE partitioned_table_2 (a int unique) partition by range(a); +CREATE TABLE partitioned_table_2_1 partition of partitioned_table_2 FOR VALUES FROM (0) TO (10); +CREATE TABLE ref_fkey_to_partitioned (a int unique references partitioned_table_1(a), b int unique references partitioned_table_2(a)); +SELECT create_reference_table('ref_fkey_to_partitioned'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- verify that partitioned tables and partitions are converted automatically +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + partitioned_table_1_1 | t + partitioned_table_1 | t + partitioned_table_2_1 | t + partitioned_table_2 | t +(4 rows) + +BEGIN; + SELECT citus_add_local_table_to_metadata('partitioned_table_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + -- verify that they are now marked as auto-converted = false + SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + partitioned_table_1_1 | f + partitioned_table_1 | f + partitioned_table_2_1 | f + partitioned_table_2 | f +(4 rows) + +ROLLBACK; +-- now they should be undistributed +DROP TABLE ref_fkey_to_partitioned; +-- verify that they are undistributed +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- +(0 rows) + +-- verify creating fkeys update auto-converted to false +CREATE TABLE table_ref(a int unique); +CREATE TABLE table_auto_conv(a int unique references table_ref(a)) partition by range(a); +CREATE TABLE table_auto_conv_child partition of table_auto_conv FOR VALUES FROM (1) TO (4); +CREATE TABLE table_auto_conv_2(a int unique references table_auto_conv(a)); +CREATE TABLE table_not_auto_conv(a int unique); +select create_reference_table('table_ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- table_not_auto_conv should not be here, as it's not converted yet +-- other tables should be marked as auto-converted +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + table_auto_conv | t + table_auto_conv_2 | t + table_auto_conv_child | t +(3 rows) + +select citus_add_local_table_to_metadata('table_not_auto_conv'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +alter table table_not_auto_conv add constraint fkey_to_mark_not_autoconverted foreign key (a) references table_auto_conv_2(a); +-- all of them should be marked as auto-converted = false +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + table_auto_conv | f + table_auto_conv_2 | f + table_auto_conv_child | f + table_not_auto_conv | f +(4 rows) + +-- create&attach new partition, it should be marked as auto-converted = false, too +CREATE TABLE table_auto_conv_child_2 partition of table_auto_conv FOR VALUES FROM (5) TO (8); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass, + 'table_auto_conv_child_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + table_auto_conv | f + table_auto_conv_2 | f + table_auto_conv_child | f + table_not_auto_conv | f + table_auto_conv_child_2 | f +(5 rows) + +-- get the autoconverted field from the parent in case of +-- CREATE TABLE .. PARTITION OF .. +create table citus_local_parent_t(a int, b int REFERENCES table_ref(a)) PARTITION BY RANGE (b); +create table citus_child_t PARTITION OF citus_local_parent_t FOR VALUES FROM (1) TO (10); +-- should be set to true +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_parent_t'::regclass, + 'citus_child_t'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_parent_t | t + citus_child_t | t +(2 rows) + +-- test CREATE TABLE REFERENCES +CREATE TABLE citus_local_user_created(a int unique); +SELECT citus_add_local_table_to_metadata('citus_local_user_created'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE citus_local_references(a int unique references citus_local_user_created(a)); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_user_created'::regclass, + 'citus_local_references'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_user_created | f + citus_local_references | f +(2 rows) + +SET citus.shard_replication_factor to 1; +-- test with a graph that includes distributed table +CREATE TABLE distr_table (a INT UNIQUE); +SELECT create_distributed_table('distr_table','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test converting in create_reference_table time +CREATE TABLE refr_table (a INT UNIQUE, b INT UNIQUE); +CREATE TABLE citus_loc_1 (a INT UNIQUE REFERENCES refr_table(a), b INT UNIQUE); +CREATE TABLE citus_loc_2 (a INT UNIQUE REFERENCES citus_loc_1(a), b INT UNIQUE REFERENCES citus_loc_1(b), c INT UNIQUE REFERENCES refr_table(a)); +CREATE TABLE citus_loc_3 (a INT UNIQUE REFERENCES citus_loc_3(a)); +CREATE TABLE citus_loc_4 (a INT UNIQUE REFERENCES citus_loc_2(b), b INT UNIQUE REFERENCES citus_loc_2(a), c INT UNIQUE REFERENCES citus_loc_3(a)); +ALTER TABLE refr_table ADD CONSTRAINT fkey_ref_to_loc FOREIGN KEY (b) REFERENCES citus_loc_2(a); +SELECT create_reference_table('refr_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE distr_table ADD CONSTRAINT fkey_dist_to_ref FOREIGN KEY (a) REFERENCES refr_table(a); +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | t | n + citus_loc_1 | t | n + citus_loc_4 | t | n + citus_loc_3 | t | n +(6 rows) + +BEGIN; + SELECT citus_add_local_table_to_metadata('citus_loc_3'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n +(6 rows) + +ROLLBACK; +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | t | n + citus_loc_1 | t | n + citus_loc_4 | t | n + citus_loc_3 | t | n +(6 rows) + +BEGIN; + SELECT citus_add_local_table_to_metadata('citus_loc_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n +(6 rows) + +ROLLBACK; +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | t | n + citus_loc_1 | t | n + citus_loc_4 | t | n + citus_loc_3 | t | n +(6 rows) + +BEGIN; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + select citus_add_local_table_to_metadata('part_citus_loc_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_partitioned_rels FOREIGN KEY (a) references part_citus_loc_2(a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('part_citus_loc_1'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + part_citus_loc_2 | f | n + part_citus_loc_1 | f | n +(2 rows) + +ROLLBACK; +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + select citus_add_local_table_to_metadata('part_citus_loc_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_partitioned_test FOREIGN KEY (a) references refr_table(a); + CREATE TABLE part_citus_loc_2_2 PARTITION OF part_citus_loc_2 FOR VALUES FROM (4) TO (5); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + part_citus_loc_2_1 | f | n + part_citus_loc_2 | f | n + part_citus_loc_2_2 | f | n +(3 rows) + +ROLLBACK; +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + select citus_add_local_table_to_metadata('part_citus_loc_2'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_partitioned_test FOREIGN KEY (a) references citus_loc_4(a); + -- reference to citus local, use alter table attach partition + CREATE TABLE part_citus_loc_2_2 (a INT UNIQUE); + ALTER TABLE part_citus_loc_2 ATTACH PARTITION part_citus_loc_2_2 FOR VALUES FROM (3) TO (5); + CREATE TABLE part_citus_loc_2_3 PARTITION OF part_citus_loc_2 FOR VALUES FROM (7) TO (8); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass, + 'part_citus_loc_2_3'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_2_1 | f | n + part_citus_loc_2 | f | n + part_citus_loc_2_2 | f | n + part_citus_loc_2_3 | f | n +(8 rows) + +ROLLBACK; +-- +-- now mark whole graph as autoConverted = false +-- +select citus_add_local_table_to_metadata('citus_loc_1'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n +(6 rows) + +BEGIN; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE REFERENCES citus_loc_1(a)) PARTITION BY RANGE (a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1 | f | n +(7 rows) + +ROLLBACK; +begin; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_testt FOREIGN KEY (a) references citus_loc_3(a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1 | f | n +(7 rows) + +rollback; +CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); +CREATE TABLE part_citus_loc_1_1 PARTITION OF part_citus_loc_1 FOR VALUES FROM (0) TO (2); +CREATE TABLE part_citus_loc_1_2 PARTITION OF part_citus_loc_1 FOR VALUES FROM (3) TO (5); +ALTER TABLE citus_loc_4 ADD CONSTRAINT fkey_to_part_table FOREIGN KEY (a) REFERENCES part_citus_loc_1(a); +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n +(9 rows) + +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE REFERENCES part_citus_loc_1(a)) PARTITION BY RANGE (a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n + part_citus_loc_2 | f | n +(10 rows) + +ROLLBACK; +-- use alter table +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_from_to_partitioned FOREIGN KEY (a) references part_citus_loc_1(a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n + part_citus_loc_2 | f | n +(10 rows) + +ROLLBACK; +-- alter table foreign key reverse order +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_from_to_partitioned FOREIGN KEY (a) references part_citus_loc_2(a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n + part_citus_loc_2 | f | n +(10 rows) + +ROLLBACK; +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + -- reference to ref + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_to_ref_test FOREIGN KEY (a) REFERENCES refr_table(a); + CREATE TABLE part_citus_loc_2_2 PARTITION OF part_citus_loc_2 FOR VALUES FROM (3) TO (5); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n + part_citus_loc_2_1 | t | n + part_citus_loc_2 | t | n + part_citus_loc_2_2 | t | n +(12 rows) + +ROLLBACK; +-- the same, but fkey to citus local, not reference table +-- also with attach partition +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_to_ref_test FOREIGN KEY (a) REFERENCES citus_loc_4(a); + -- reference to citus local, use create table partition of + CREATE TABLE part_citus_loc_2_2(a INT UNIQUE); + ALTER TABLE part_citus_loc_2 ATTACH PARTITION part_citus_loc_2_2 FOR VALUES FROM (3) TO (5); + CREATE TABLE part_citus_loc_2_3 PARTITION OF part_citus_loc_2 FOR VALUES FROM (7) TO (9); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass, + 'part_citus_loc_2_3'::regclass) + ORDER BY logicalrelid; + logicalrelid | autoconverted | partmethod +--------------------------------------------------------------------- + distr_table | f | h + refr_table | f | n + citus_loc_2 | f | n + citus_loc_1 | f | n + citus_loc_4 | f | n + citus_loc_3 | f | n + part_citus_loc_1_1 | f | n + part_citus_loc_1_2 | f | n + part_citus_loc_1 | f | n + part_citus_loc_2_1 | f | n + part_citus_loc_2 | f | n + part_citus_loc_2_2 | f | n + part_citus_loc_2_3 | f | n +(13 rows) + +ROLLBACK; -- a single drop table cascades into multiple undistributes DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3, citus_local_table_2, reference_table_1; CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int); @@ -491,9 +1216,9 @@ ALTER TABLE reference_table_1 OWNER TO another_user; SELECT run_command_on_placements('reference_table_1', 'ALTER TABLE %s OWNER TO another_user'); run_command_on_placements --------------------------------------------------------------------- - (localhost,57636,1810039,t,"ALTER TABLE") - (localhost,57637,1810039,t,"ALTER TABLE") - (localhost,57638,1810039,t,"ALTER TABLE") + (localhost,57636,1810093,t,"ALTER TABLE") + (localhost,57637,1810093,t,"ALTER TABLE") + (localhost,57638,1810093,t,"ALTER TABLE") (3 rows) SET citus.enable_ddl_propagation to ON; diff --git a/src/test/regress/expected/citus_local_tables.out b/src/test/regress/expected/citus_local_tables.out index d734ae32f..871425744 100644 --- a/src/test/regress/expected/citus_local_tables.out +++ b/src/test/regress/expected/citus_local_tables.out @@ -424,15 +424,19 @@ BEGIN; (1 row) - -- should not see any citus local tables + -- should see only citus local tables that are not converted automatically SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables WHERE tablename=logicalrelid::regclass::text AND schemaname='citus_local_tables_test_schema' AND partmethod = 'n' AND repmodel = 's' ORDER BY 1; - logicalrelid + logicalrelid --------------------------------------------------------------------- -(0 rows) + citus_local_table_1 + citus_local_table_2 + local_table_3 + unlogged_table +(4 rows) ROLLBACK; -- define foreign keys between dummy_reference_table and citus local tables diff --git a/src/test/regress/expected/citus_local_tables_mx.out b/src/test/regress/expected/citus_local_tables_mx.out index 3ed66c36d..c0cc244a1 100644 --- a/src/test/regress/expected/citus_local_tables_mx.out +++ b/src/test/regress/expected/citus_local_tables_mx.out @@ -628,7 +628,7 @@ ERROR: cannot build foreign key between reference table and a partition -- this should work alter table citus_local_parent_1 add constraint fkey_to_drop_test foreign key(a) references ref(a); -- this should undistribute the table, and the entries should be gone from pg_dist_partition -select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%' order by logicalrelid; logicalrelid --------------------------------------------------------------------- citus_local_parent_1 diff --git a/src/test/regress/expected/columnar_citus_integration.out b/src/test/regress/expected/columnar_citus_integration.out index aa7180cad..582945bae 100644 --- a/src/test/regress/expected/columnar_citus_integration.out +++ b/src/test/regress/expected/columnar_citus_integration.out @@ -259,7 +259,7 @@ NOTICE: renaming the new table to columnar_citus_integration.table_option (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid = 'table_option'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) @@ -578,7 +578,7 @@ NOTICE: renaming the new table to columnar_citus_integration.table_option (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid = 'table_option'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) @@ -817,7 +817,7 @@ NOTICE: renaming the new table to columnar_citus_integration.table_option_refer (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid = 'table_option_reference'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) @@ -1051,7 +1051,7 @@ NOTICE: renaming the new table to columnar_citus_integration.table_option_citus (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid = 'table_option_citus_local'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/failure_add_disable_node.out b/src/test/regress/expected/failure_add_disable_node.out index 051ebb7b0..8780575e7 100644 --- a/src/test/regress/expected/failure_add_disable_node.out +++ b/src/test/regress/expected/failure_add_disable_node.out @@ -23,7 +23,7 @@ ORDER BY 1, 2; -- verify there are no tables that could prevent add/remove node operations SELECT * FROM pg_dist_partition; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/fkeys_between_local_ref.out b/src/test/regress/expected/fkeys_between_local_ref.out index 1e3f7eb20..49fc9d100 100644 --- a/src/test/regress/expected/fkeys_between_local_ref.out +++ b/src/test/regress/expected/fkeys_between_local_ref.out @@ -359,17 +359,18 @@ BEGIN; DROP TABLE local_table_3 CASCADE; DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE; - -- now we shouldn't see local_table_5 since now it is not connected to any reference tables - SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition + -- now we shouldn't see local_table_5 since now it is not connected to any reference tables/citus local tables + -- and it's converted automatically + SELECT logicalrelid::text AS tablename, partmethod, repmodel, autoconverted FROM pg_dist_partition WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref') ORDER BY tablename; - tablename | partmethod | repmodel + tablename | partmethod | repmodel | autoconverted --------------------------------------------------------------------- - distributed_table | h | s - local_table_1 | n | s - local_table_2 | n | s - local_table_4 | n | s - reference_table_1 | n | t + distributed_table | h | s | f + local_table_1 | n | s | t + local_table_2 | n | s | t + local_table_4 | n | s | t + reference_table_1 | n | t | f (5 rows) ROLLBACK; diff --git a/src/test/regress/expected/multi_metadata_attributes.out b/src/test/regress/expected/multi_metadata_attributes.out index 85ececdac..67389f51a 100644 --- a/src/test/regress/expected/multi_metadata_attributes.out +++ b/src/test/regress/expected/multi_metadata_attributes.out @@ -6,7 +6,9 @@ -- part of the query so new changes to it won't affect this test. SELECT attrelid::regclass, attname, atthasmissing, attmissingval FROM pg_attribute -WHERE atthasmissing AND attrelid NOT IN ('pg_dist_node'::regclass, 'pg_dist_rebalance_strategy'::regclass) +WHERE atthasmissing AND attrelid NOT IN ('pg_dist_node'::regclass, + 'pg_dist_rebalance_strategy'::regclass, + 'pg_dist_partition'::regclass) ORDER BY attrelid, attname; attrelid | attname | atthasmissing | attmissingval --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out index 7a5b75e11..2e7d42a50 100644 --- a/src/test/regress/expected/multi_metadata_sync.out +++ b/src/test/regress/expected/multi_metadata_sync.out @@ -19,7 +19,7 @@ COMMENT ON FUNCTION master_metadata_snapshot() IS 'commands to create the metadata snapshot'; -- Show that none of the existing tables are qualified to be MX tables SELECT * FROM pg_dist_partition WHERE partmethod='h' AND repmodel='s'; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) @@ -273,9 +273,9 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; (4 rows) SELECT * FROM pg_dist_partition ORDER BY logicalrelid; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 0 | s + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 0 | s | f (1 row) SELECT * FROM pg_dist_shard ORDER BY shardid; @@ -410,9 +410,9 @@ SELECT * FROM pg_dist_node ORDER BY nodeid; (4 rows) SELECT * FROM pg_dist_partition ORDER BY logicalrelid; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 0 | s + mx_testing_schema.mx_test_table | h | {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 0 | s | f (1 row) SELECT * FROM pg_dist_shard ORDER BY shardid; @@ -731,7 +731,7 @@ ORDER BY \d mx_test_schema_1.mx_table_1 \d mx_test_schema_2.mx_table_2 SELECT * FROM pg_dist_partition; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index fed9039a2..482d41589 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -4208,7 +4208,11 @@ SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_tab (1 row) \set VERBOSITY default +set client_min_messages to error; DROP TABLE date_partitioned_table_to_exp; +DROP TABLE date_partitioned_citus_local_table CASCADE; +DROP TABLE date_partitioned_citus_local_table_2; +set client_min_messages to notice; SELECT citus_remove_node('localhost', :master_port); citus_remove_node --------------------------------------------------------------------- @@ -4326,12 +4330,10 @@ NOTICE: dropping metadata on the node (localhost,57637) (1 row) DROP SCHEMA partitioning_schema CASCADE; -NOTICE: drop cascades to 6 other objects +NOTICE: drop cascades to 4 other objects DETAIL: drop cascades to table partitioning_schema."schema-test" drop cascades to table partitioning_schema.another_distributed_table drop cascades to table partitioning_schema.distributed_parent_table -drop cascades to table partitioning_schema.date_partitioned_citus_local_table -drop cascades to table partitioning_schema.date_partitioned_citus_local_table_2 drop cascades to table partitioning_schema.part_table_with_very_long_name RESET search_path; DROP TABLE IF EXISTS diff --git a/src/test/regress/expected/multi_table_ddl.out b/src/test/regress/expected/multi_table_ddl.out index f1d23a06d..4c991dfcb 100644 --- a/src/test/regress/expected/multi_table_ddl.out +++ b/src/test/regress/expected/multi_table_ddl.out @@ -60,7 +60,7 @@ DROP TABLE testtableddl; RESET citus.shard_replication_factor; -- ensure no metadata of distributed tables are remaining SELECT * FROM pg_dist_partition; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/ref_citus_local_fkeys.out b/src/test/regress/expected/ref_citus_local_fkeys.out index a94208a13..daf4aeca6 100644 --- a/src/test/regress/expected/ref_citus_local_fkeys.out +++ b/src/test/regress/expected/ref_citus_local_fkeys.out @@ -52,22 +52,19 @@ NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.cit -- show that we support drop constraint ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref; NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;') -NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506000 citus_local_table -NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE -- we support ON UPDATE CASCADE behaviour in "ALTER TABLE ADD fkey citus_local_table (to reference table)" commands ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506002, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE;') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE;') -- show that on update cascade works INSERT INTO reference_table VALUES (12); NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (12) INSERT INTO citus_local_table VALUES (12); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506002 (l1) VALUES (12) +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (12) UPDATE reference_table SET r1=13 WHERE r1=12; NOTICE: executing the command locally: UPDATE ref_citus_local_fkeys.reference_table_1506001 reference_table SET r1 = 13 WHERE (r1 OPERATOR(pg_catalog.=) 12) -- should print a row with 13 SELECT * FROM citus_local_table ORDER BY l1; -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506002 citus_local_table ORDER BY l1 +NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506000 citus_local_table ORDER BY l1 l1 --------------------------------------------------------------------- 13 @@ -75,37 +72,32 @@ NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.cit -- drop constraint for next commands ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506002, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;') -NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506002 citus_local_table -NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;') INSERT INTO citus_local_table VALUES (2); +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (2) -- show that we are checking for foreign key constraint while defining, below should fail ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') -ERROR: insert or update on table "citus_local_table_1506003" violates foreign key constraint "fkey_local_to_ref_1506003" +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') +ERROR: insert or update on table "citus_local_table_1506000" violates foreign key constraint "fkey_local_to_ref_1506000" INSERT INTO reference_table VALUES (2); NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (2) -- this should work ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506004, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') -- show that we are checking for foreign key constraint after defining, this should fail INSERT INTO citus_local_table VALUES (1); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506004 (l1) VALUES (1) -ERROR: insert or update on table "citus_local_table_1506004" violates foreign key constraint "fkey_local_to_ref_1506004" +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (1) +ERROR: insert or update on table "citus_local_table_1506000" violates foreign key constraint "fkey_local_to_ref_1506000" INSERT INTO reference_table VALUES (1); NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (1) -- this should work INSERT INTO citus_local_table VALUES (1); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506004 (l1) VALUES (1) +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (1) -- drop and add constraint for next commands ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506004, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;') -NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506004 citus_local_table -NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;') ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506005, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);') -- show that drop table without CASCADE errors out DROP TABLE reference_table; ERROR: cannot drop table reference_table because other objects depend on it @@ -114,15 +106,12 @@ BEGIN; DROP TABLE reference_table CASCADE; NOTICE: drop cascades to constraint fkey_local_to_ref on table citus_local_table NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.reference_table_xxxxx CASCADE -NOTICE: drop cascades to constraint fkey_local_to_ref_1506005 on table ref_citus_local_fkeys.citus_local_table_1506005 -NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506005 citus_local_table -NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE +NOTICE: drop cascades to constraint fkey_local_to_ref_1506000 on table ref_citus_local_fkeys.citus_local_table_1506000 ROLLBACK; -- drop tables finally DROP TABLE citus_local_table, reference_table; NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.reference_table_xxxxx CASCADE -NOTICE: drop cascades to constraint fkey_local_to_ref_1506005 on table ref_citus_local_fkeys.citus_local_table_1506005 +NOTICE: drop cascades to constraint fkey_local_to_ref_1506000 on table ref_citus_local_fkeys.citus_local_table_1506000 NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE --------------------------------------------------------------------- -- foreign key from reference table to citus local table -- @@ -152,11 +141,11 @@ SELECT create_reference_table('reference_table'); (1 row) INSERT INTO reference_table VALUES (3); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506007 (r1) VALUES (3) +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506003 (r1) VALUES (3) -- show that we are checking for foreign key constraint while defining, this should fail ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1);') -ERROR: insert or update on table "reference_table_1506007" violates foreign key constraint "fkey_ref_to_local_1506007" +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1);') +ERROR: insert or update on table "reference_table_1506003" violates foreign key constraint "fkey_ref_to_local_1506003" -- we do not support CASCADE / SET NULL / SET DEFAULT behavior in "ALTER TABLE ADD fkey reference_table (to citus_local_table)" commands ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE CASCADE; ERROR: cannot define foreign key constraint, foreign keys from reference tables to local tables can only be defined with NO ACTION or RESTRICT behaviors @@ -171,29 +160,38 @@ ERROR: cannot define foreign key constraint, foreign keys from reference tables ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON UPDATE SET DEFAULT; ERROR: cannot define foreign key constraint, foreign keys from reference tables to local tables can only be defined with NO ACTION or RESTRICT behaviors INSERT INTO citus_local_table VALUES (3); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506006 (l1) VALUES (3) +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506002 (l1) VALUES (3) -- .. but we allow such foreign keys with RESTRICT behavior BEGIN; ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT;') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT;') ROLLBACK; -- .. and we allow such foreign keys with NO ACTION behavior ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;') -- show that adding/dropping foreign keys from reference to citus local -- tables works fine with remote execution too SET citus.enable_local_execution TO OFF; ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local; -NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys +SELECT undistribute_table('citus_local_table'); +NOTICE: creating a new table for ref_citus_local_fkeys.citus_local_table +NOTICE: moving the data of ref_citus_local_fkeys.citus_local_table +NOTICE: dropping the old ref_citus_local_fkeys.citus_local_table +NOTICE: renaming the new table to ref_citus_local_fkeys.citus_local_table + undistribute_table +--------------------------------------------------------------------- + +(1 row) + ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION; ERROR: cannot execute command because a local execution has accessed a placement in the transaction SET citus.enable_local_execution TO ON; ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506009, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506005, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;') -- show that we are checking for foreign key constraint after defining, this should fail INSERT INTO reference_table VALUES (4); -NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506007 (r1) VALUES (4) -ERROR: insert or update on table "reference_table_1506007" violates foreign key constraint "fkey_ref_to_local_1506007" +NOTICE: executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506003 (r1) VALUES (4) +ERROR: insert or update on table "reference_table_1506003" violates foreign key constraint "fkey_ref_to_local_1506003" -- enable the worker_2 to show that we don't try to set up the foreign keys -- between reference tables and citus local tables in worker_2 placements of -- the reference tables @@ -207,9 +205,9 @@ NOTICE: Replicating reference table "reference_table" to the node localhost:xxx -- show that we support drop constraint BEGIN; ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local; -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506009, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506005, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;') NOTICE: removing table ref_citus_local_fkeys.citus_local_table from metadata as it is not connected to any reference tables via foreign keys -NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506009 citus_local_table +NOTICE: executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506005 citus_local_table NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE ROLLBACK; -- show that drop table errors as expected @@ -219,7 +217,7 @@ ERROR: cannot drop table citus_local_table because other objects depend on it DROP TABLE citus_local_table CASCADE; NOTICE: drop cascades to constraint fkey_ref_to_local on table reference_table NOTICE: executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE -NOTICE: drop cascades to constraint fkey_ref_to_local_1506007 on table ref_citus_local_fkeys.reference_table_1506007 +NOTICE: drop cascades to constraint fkey_ref_to_local_1506003 on table ref_citus_local_fkeys.reference_table_1506003 BEGIN; CREATE TABLE citus_local_table_1(a int, b int, unique (a,b)); CREATE TABLE citus_local_table_2(a int, b int, unique (a,b)); @@ -237,7 +235,7 @@ BEGIN; -- show that we properly handle multi column foreign keys ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b); -NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506010, 'ref_citus_local_fkeys', 1506011, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b);') +NOTICE: executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506006, 'ref_citus_local_fkeys', 1506007, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b);') COMMIT; -- when local execution is disabled, citus local table cannot be created BEGIN; diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index 2d709ac86..4e0e9f98a 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -1248,7 +1248,7 @@ NOTICE: renaming the new table to single_node.test_2 (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid = 'test_2'::regclass; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/start_stop_metadata_sync.out b/src/test/regress/expected/start_stop_metadata_sync.out index 841ed9812..a71edacf3 100644 --- a/src/test/regress/expected/start_stop_metadata_sync.out +++ b/src/test/regress/expected/start_stop_metadata_sync.out @@ -120,11 +120,11 @@ SELECT * FROM test_matview; (1 row) SELECT * FROM pg_dist_partition WHERE logicalrelid::text LIKE 'events%' ORDER BY logicalrelid::text; - logicalrelid | partmethod | partkey | colocationid | repmodel + logicalrelid | partmethod | partkey | colocationid | repmodel | autoconverted --------------------------------------------------------------------- - events | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s - events_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s - events_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s + events | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s | f + events_2021_feb | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s | f + events_2021_jan | h | {VAR :varno 1 :varattno 1 :vartype 1184 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} | 1390012 | s | f (3 rows) SELECT count(*) > 0 FROM pg_dist_node; diff --git a/src/test/regress/expected/turn_mx_off_1.out b/src/test/regress/expected/turn_mx_off_1.out new file mode 100644 index 000000000..7497f24c6 --- /dev/null +++ b/src/test/regress/expected/turn_mx_off_1.out @@ -0,0 +1,16 @@ +ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SET client_min_messages TO ERROR; +SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary'; + stop_metadata_sync_to_node +--------------------------------------------------------------------- + + + +(3 rows) + diff --git a/src/test/regress/expected/upgrade_autoconverted_after.out b/src/test/regress/expected/upgrade_autoconverted_after.out new file mode 100644 index 000000000..06c93c173 --- /dev/null +++ b/src/test/regress/expected/upgrade_autoconverted_after.out @@ -0,0 +1,9 @@ +select logicalrelid, autoconverted from pg_dist_partition + where logicalrelid IN ('citus_local_autoconverted'::regclass, + 'citus_local_not_autoconverted'::regclass); + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_autoconverted | t + citus_local_not_autoconverted | f +(2 rows) + diff --git a/src/test/regress/expected/upgrade_autoconverted_before.out b/src/test/regress/expected/upgrade_autoconverted_before.out new file mode 100644 index 000000000..6d2f7a000 --- /dev/null +++ b/src/test/regress/expected/upgrade_autoconverted_before.out @@ -0,0 +1,24 @@ +CREATE TABLE ref_not_autoconverted(a int unique); +CREATE TABLE citus_local_autoconverted(a int unique references ref_not_autoconverted(a)); +CREATE TABLE citus_local_not_autoconverted(a int unique); +select create_reference_table('ref_not_autoconverted'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +select citus_add_local_table_to_metadata('citus_local_not_autoconverted'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +select logicalrelid, autoconverted from pg_dist_partition + where logicalrelid IN ('citus_local_autoconverted'::regclass, + 'citus_local_not_autoconverted'::regclass); + logicalrelid | autoconverted +--------------------------------------------------------------------- + citus_local_autoconverted | t + citus_local_not_autoconverted | f +(2 rows) + diff --git a/src/test/regress/expected/upgrade_basic_after.out b/src/test/regress/expected/upgrade_basic_after.out index 15ef5bff5..29c195d34 100644 --- a/src/test/regress/expected/upgrade_basic_after.out +++ b/src/test/regress/expected/upgrade_basic_after.out @@ -1,6 +1,7 @@ SET search_path TO upgrade_basic, public, pg_catalog; BEGIN; -SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' ORDER BY tablename; +-- We have the tablename filter to avoid adding an alternative output for when the coordinator is in metadata vs when not +SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' and tablename NOT LIKE 'r_%' ORDER BY tablename; schemaname | tablename | indexname | tablespace | indexdef --------------------------------------------------------------------- upgrade_basic | r | r_pkey | | CREATE UNIQUE INDEX r_pkey ON upgrade_basic.r USING btree (a) diff --git a/src/test/regress/sql/auto_undist_citus_local.sql b/src/test/regress/sql/auto_undist_citus_local.sql index 935145a8e..66ce13ea3 100644 --- a/src/test/regress/sql/auto_undist_citus_local.sql +++ b/src/test/regress/sql/auto_undist_citus_local.sql @@ -10,7 +10,6 @@ SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0); -- show that DROP CONSTRAINT cascades to undistributing citus_local_table CREATE TABLE citus_local_table(l1 int); -SELECT citus_add_local_table_to_metadata('citus_local_table'); CREATE TABLE reference_table(r1 int primary key); SELECT create_reference_table('reference_table'); ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON DELETE CASCADE; @@ -209,6 +208,430 @@ SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalre ALTER TABLE reference_table_1 DROP COLUMN r1 CASCADE; SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid; +-- verify that citus local tables converted by the user will not be auto-undistributed +DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3; +CREATE TABLE citus_local_table_1(a INT UNIQUE); +CREATE TABLE citus_local_table_2(a INT UNIQUE); +CREATE TABLE citus_local_table_3(a INT UNIQUE); +ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_cas_test FOREIGN KEY (a) REFERENCES citus_local_table_2 (a); +SELECT citus_add_local_table_to_metadata('citus_local_table_1', cascade_via_foreign_keys=>true); +SELECT citus_add_local_table_to_metadata('citus_local_table_3'); +ALTER TABLE citus_local_table_3 ADD CONSTRAINT fkey_cas_test_2 FOREIGN KEY (a) REFERENCES citus_local_table_2 (a); +ALTER TABLE citus_local_table_3 DROP CONSTRAINT fkey_cas_test_2; +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_table_1'::regclass, + 'citus_local_table_2'::regclass, + 'citus_local_table_3'::regclass) + ORDER BY logicalrelid; +ALTER TABLE citus_local_table_1 DROP CONSTRAINT fkey_cas_test; +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_table_1'::regclass, + 'citus_local_table_2'::regclass, + 'citus_local_table_3'::regclass) + ORDER BY logicalrelid; + +-- verify that tables that are connected to reference tables are marked as autoConverted = true +CREATE TABLE ref_test(a int UNIQUE); +SELECT create_reference_table('ref_test'); +CREATE TABLE auto_local_table_1(a int UNIQUE); +CREATE TABLE auto_local_table_2(a int UNIQUE REFERENCES auto_local_table_1(a)); +ALTER TABLE auto_local_table_1 ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (a) REFERENCES ref_test(a); + +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('auto_local_table_1'::regclass, + 'auto_local_table_2'::regclass) + ORDER BY logicalrelid; + +-- verify that we can mark both of them with autoConverted = false, by converting one of them manually +SELECT citus_add_local_table_to_metadata('auto_local_table_1'); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('auto_local_table_1'::regclass, + 'auto_local_table_2'::regclass) + ORDER BY logicalrelid; + +-- test with partitioned tables +CREATE TABLE partitioned_table_1 (a int unique) partition by range(a); +CREATE TABLE partitioned_table_1_1 partition of partitioned_table_1 FOR VALUES FROM (0) TO (10); +CREATE TABLE partitioned_table_2 (a int unique) partition by range(a); +CREATE TABLE partitioned_table_2_1 partition of partitioned_table_2 FOR VALUES FROM (0) TO (10); +CREATE TABLE ref_fkey_to_partitioned (a int unique references partitioned_table_1(a), b int unique references partitioned_table_2(a)); +SELECT create_reference_table('ref_fkey_to_partitioned'); + +-- verify that partitioned tables and partitions are converted automatically +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; + +BEGIN; + SELECT citus_add_local_table_to_metadata('partitioned_table_2'); + + -- verify that they are now marked as auto-converted = false + SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +-- now they should be undistributed +DROP TABLE ref_fkey_to_partitioned; +-- verify that they are undistributed +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('partitioned_table_1'::regclass, + 'partitioned_table_1_1'::regclass, + 'partitioned_table_2'::regclass, + 'partitioned_table_2_1'::regclass) + ORDER BY logicalrelid; + +-- verify creating fkeys update auto-converted to false +CREATE TABLE table_ref(a int unique); +CREATE TABLE table_auto_conv(a int unique references table_ref(a)) partition by range(a); +CREATE TABLE table_auto_conv_child partition of table_auto_conv FOR VALUES FROM (1) TO (4); +CREATE TABLE table_auto_conv_2(a int unique references table_auto_conv(a)); +CREATE TABLE table_not_auto_conv(a int unique); +select create_reference_table('table_ref'); + +-- table_not_auto_conv should not be here, as it's not converted yet +-- other tables should be marked as auto-converted +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass) + ORDER BY logicalrelid; + +select citus_add_local_table_to_metadata('table_not_auto_conv'); +alter table table_not_auto_conv add constraint fkey_to_mark_not_autoconverted foreign key (a) references table_auto_conv_2(a); + +-- all of them should be marked as auto-converted = false +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass) + ORDER BY logicalrelid; + +-- create&attach new partition, it should be marked as auto-converted = false, too +CREATE TABLE table_auto_conv_child_2 partition of table_auto_conv FOR VALUES FROM (5) TO (8); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('table_auto_conv'::regclass, + 'table_auto_conv_child'::regclass, + 'table_auto_conv_2'::regclass, + 'table_not_auto_conv'::regclass, + 'table_auto_conv_child_2'::regclass) + ORDER BY logicalrelid; + +-- get the autoconverted field from the parent in case of +-- CREATE TABLE .. PARTITION OF .. +create table citus_local_parent_t(a int, b int REFERENCES table_ref(a)) PARTITION BY RANGE (b); +create table citus_child_t PARTITION OF citus_local_parent_t FOR VALUES FROM (1) TO (10); +-- should be set to true +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_parent_t'::regclass, + 'citus_child_t'::regclass) + ORDER BY logicalrelid; + +-- test CREATE TABLE REFERENCES +CREATE TABLE citus_local_user_created(a int unique); +SELECT citus_add_local_table_to_metadata('citus_local_user_created'); +CREATE TABLE citus_local_references(a int unique references citus_local_user_created(a)); +SELECT logicalrelid, autoconverted FROM pg_dist_partition + WHERE logicalrelid IN ('citus_local_user_created'::regclass, + 'citus_local_references'::regclass) + ORDER BY logicalrelid; + +SET citus.shard_replication_factor to 1; +-- test with a graph that includes distributed table +CREATE TABLE distr_table (a INT UNIQUE); +SELECT create_distributed_table('distr_table','a'); + +-- test converting in create_reference_table time +CREATE TABLE refr_table (a INT UNIQUE, b INT UNIQUE); +CREATE TABLE citus_loc_1 (a INT UNIQUE REFERENCES refr_table(a), b INT UNIQUE); +CREATE TABLE citus_loc_2 (a INT UNIQUE REFERENCES citus_loc_1(a), b INT UNIQUE REFERENCES citus_loc_1(b), c INT UNIQUE REFERENCES refr_table(a)); +CREATE TABLE citus_loc_3 (a INT UNIQUE REFERENCES citus_loc_3(a)); +CREATE TABLE citus_loc_4 (a INT UNIQUE REFERENCES citus_loc_2(b), b INT UNIQUE REFERENCES citus_loc_2(a), c INT UNIQUE REFERENCES citus_loc_3(a)); +ALTER TABLE refr_table ADD CONSTRAINT fkey_ref_to_loc FOREIGN KEY (b) REFERENCES citus_loc_2(a); +SELECT create_reference_table('refr_table'); +ALTER TABLE distr_table ADD CONSTRAINT fkey_dist_to_ref FOREIGN KEY (a) REFERENCES refr_table(a); + +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + +BEGIN; + SELECT citus_add_local_table_to_metadata('citus_loc_3'); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + +BEGIN; + SELECT citus_add_local_table_to_metadata('citus_loc_2'); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + +BEGIN; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + + select citus_add_local_table_to_metadata('part_citus_loc_2'); + + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_partitioned_rels FOREIGN KEY (a) references part_citus_loc_2(a); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('part_citus_loc_1'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + + select citus_add_local_table_to_metadata('part_citus_loc_2'); + + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_partitioned_test FOREIGN KEY (a) references refr_table(a); + + CREATE TABLE part_citus_loc_2_2 PARTITION OF part_citus_loc_2 FOR VALUES FROM (4) TO (5); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + + select citus_add_local_table_to_metadata('part_citus_loc_2'); + + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_partitioned_test FOREIGN KEY (a) references citus_loc_4(a); + + -- reference to citus local, use alter table attach partition + CREATE TABLE part_citus_loc_2_2 (a INT UNIQUE); + ALTER TABLE part_citus_loc_2 ATTACH PARTITION part_citus_loc_2_2 FOR VALUES FROM (3) TO (5); + CREATE TABLE part_citus_loc_2_3 PARTITION OF part_citus_loc_2 FOR VALUES FROM (7) TO (8); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass, + 'part_citus_loc_2_3'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +-- +-- now mark whole graph as autoConverted = false +-- +select citus_add_local_table_to_metadata('citus_loc_1'); +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass) + ORDER BY logicalrelid; + +BEGIN; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE REFERENCES citus_loc_1(a)) PARTITION BY RANGE (a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +begin; + CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_testt FOREIGN KEY (a) references citus_loc_3(a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass) + ORDER BY logicalrelid; +rollback; + +CREATE TABLE part_citus_loc_1 (a INT UNIQUE) PARTITION BY RANGE (a); +CREATE TABLE part_citus_loc_1_1 PARTITION OF part_citus_loc_1 FOR VALUES FROM (0) TO (2); +CREATE TABLE part_citus_loc_1_2 PARTITION OF part_citus_loc_1 FOR VALUES FROM (3) TO (5); +ALTER TABLE citus_loc_4 ADD CONSTRAINT fkey_to_part_table FOREIGN KEY (a) REFERENCES part_citus_loc_1(a); + +SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass) + ORDER BY logicalrelid; + +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE REFERENCES part_citus_loc_1(a)) PARTITION BY RANGE (a); + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +-- use alter table +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_from_to_partitioned FOREIGN KEY (a) references part_citus_loc_1(a); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +-- alter table foreign key reverse order +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + ALTER TABLE part_citus_loc_1 ADD CONSTRAINT fkey_from_to_partitioned FOREIGN KEY (a) references part_citus_loc_2(a); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + + -- reference to ref + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_to_ref_test FOREIGN KEY (a) REFERENCES refr_table(a); + + CREATE TABLE part_citus_loc_2_2 PARTITION OF part_citus_loc_2 FOR VALUES FROM (3) TO (5); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + +-- the same, but fkey to citus local, not reference table +-- also with attach partition +BEGIN; + CREATE TABLE part_citus_loc_2 (a INT UNIQUE) PARTITION BY RANGE (a); + CREATE TABLE part_citus_loc_2_1 PARTITION OF part_citus_loc_2 FOR VALUES FROM (0) TO (2); + + ALTER TABLE part_citus_loc_2 ADD CONSTRAINT fkey_to_ref_test FOREIGN KEY (a) REFERENCES citus_loc_4(a); + + -- reference to citus local, use create table partition of + CREATE TABLE part_citus_loc_2_2(a INT UNIQUE); + ALTER TABLE part_citus_loc_2 ATTACH PARTITION part_citus_loc_2_2 FOR VALUES FROM (3) TO (5); + + CREATE TABLE part_citus_loc_2_3 PARTITION OF part_citus_loc_2 FOR VALUES FROM (7) TO (9); + + SELECT logicalrelid, autoconverted, partmethod FROM pg_dist_partition + WHERE logicalrelid IN ('distr_table'::regclass, + 'citus_loc_1'::regclass, + 'citus_loc_2'::regclass, + 'citus_loc_3'::regclass, + 'citus_loc_4'::regclass, + 'refr_table'::regclass, + 'part_citus_loc_1'::regclass, + 'part_citus_loc_1_1'::regclass, + 'part_citus_loc_1_2'::regclass, + 'part_citus_loc_2'::regclass, + 'part_citus_loc_2_1'::regclass, + 'part_citus_loc_2_2'::regclass, + 'part_citus_loc_2_3'::regclass) + ORDER BY logicalrelid; +ROLLBACK; + -- a single drop table cascades into multiple undistributes DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3, citus_local_table_2, reference_table_1; CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int); diff --git a/src/test/regress/sql/citus_local_tables.sql b/src/test/regress/sql/citus_local_tables.sql index 730fc0d10..577393554 100644 --- a/src/test/regress/sql/citus_local_tables.sql +++ b/src/test/regress/sql/citus_local_tables.sql @@ -317,7 +317,7 @@ BEGIN; SET client_min_messages TO ERROR; SELECT remove_local_tables_from_metadata(); - -- should not see any citus local tables + -- should see only citus local tables that are not converted automatically SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables WHERE tablename=logicalrelid::regclass::text AND schemaname='citus_local_tables_test_schema' AND diff --git a/src/test/regress/sql/citus_local_tables_mx.sql b/src/test/regress/sql/citus_local_tables_mx.sql index d74e18da6..cb94ed086 100644 --- a/src/test/regress/sql/citus_local_tables_mx.sql +++ b/src/test/regress/sql/citus_local_tables_mx.sql @@ -337,7 +337,7 @@ alter table citus_local_parent_1_child_1 add foreign key(a) references ref(a); -- this should work alter table citus_local_parent_1 add constraint fkey_to_drop_test foreign key(a) references ref(a); -- this should undistribute the table, and the entries should be gone from pg_dist_partition -select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; +select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%' order by logicalrelid; set client_min_messages to error; alter table citus_local_parent_1 drop constraint fkey_to_drop_test; select logicalrelid from pg_dist_partition where logicalrelid::text like 'citus_local_parent%'; diff --git a/src/test/regress/sql/fkeys_between_local_ref.sql b/src/test/regress/sql/fkeys_between_local_ref.sql index 0cdeb706a..630cde968 100644 --- a/src/test/regress/sql/fkeys_between_local_ref.sql +++ b/src/test/regress/sql/fkeys_between_local_ref.sql @@ -267,8 +267,9 @@ BEGIN; DROP TABLE local_table_3 CASCADE; DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE; - -- now we shouldn't see local_table_5 since now it is not connected to any reference tables - SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition + -- now we shouldn't see local_table_5 since now it is not connected to any reference tables/citus local tables + -- and it's converted automatically + SELECT logicalrelid::text AS tablename, partmethod, repmodel, autoconverted FROM pg_dist_partition WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref') ORDER BY tablename; ROLLBACK; diff --git a/src/test/regress/sql/multi_metadata_attributes.sql b/src/test/regress/sql/multi_metadata_attributes.sql index 174c92331..e285db192 100644 --- a/src/test/regress/sql/multi_metadata_attributes.sql +++ b/src/test/regress/sql/multi_metadata_attributes.sql @@ -7,5 +7,7 @@ -- part of the query so new changes to it won't affect this test. SELECT attrelid::regclass, attname, atthasmissing, attmissingval FROM pg_attribute -WHERE atthasmissing AND attrelid NOT IN ('pg_dist_node'::regclass, 'pg_dist_rebalance_strategy'::regclass) +WHERE atthasmissing AND attrelid NOT IN ('pg_dist_node'::regclass, + 'pg_dist_rebalance_strategy'::regclass, + 'pg_dist_partition'::regclass) ORDER BY attrelid, attname; diff --git a/src/test/regress/sql/multi_partitioning.sql b/src/test/regress/sql/multi_partitioning.sql index d5c513b7c..702ffbe23 100644 --- a/src/test/regress/sql/multi_partitioning.sql +++ b/src/test/regress/sql/multi_partitioning.sql @@ -1932,7 +1932,11 @@ CALL drop_old_time_partitions('date_partitioned_table_to_exp', '2021-01-01'); SELECT partition FROM time_partitions WHERE parent_table = 'date_partitioned_table_to_exp'::regclass ORDER BY partition::text; \set VERBOSITY default +set client_min_messages to error; DROP TABLE date_partitioned_table_to_exp; +DROP TABLE date_partitioned_citus_local_table CASCADE; +DROP TABLE date_partitioned_citus_local_table_2; +set client_min_messages to notice; SELECT citus_remove_node('localhost', :master_port); diff --git a/src/test/regress/sql/ref_citus_local_fkeys.sql b/src/test/regress/sql/ref_citus_local_fkeys.sql index 143f3ce2f..76028bc1a 100644 --- a/src/test/regress/sql/ref_citus_local_fkeys.sql +++ b/src/test/regress/sql/ref_citus_local_fkeys.sql @@ -117,6 +117,7 @@ ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REF -- tables works fine with remote execution too SET citus.enable_local_execution TO OFF; ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local; +SELECT undistribute_table('citus_local_table'); ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION; SET citus.enable_local_execution TO ON; diff --git a/src/test/regress/sql/upgrade_autoconverted_after.sql b/src/test/regress/sql/upgrade_autoconverted_after.sql new file mode 100644 index 000000000..8c5d31cb4 --- /dev/null +++ b/src/test/regress/sql/upgrade_autoconverted_after.sql @@ -0,0 +1,3 @@ +select logicalrelid, autoconverted from pg_dist_partition + where logicalrelid IN ('citus_local_autoconverted'::regclass, + 'citus_local_not_autoconverted'::regclass); diff --git a/src/test/regress/sql/upgrade_autoconverted_before.sql b/src/test/regress/sql/upgrade_autoconverted_before.sql new file mode 100644 index 000000000..e09f87d20 --- /dev/null +++ b/src/test/regress/sql/upgrade_autoconverted_before.sql @@ -0,0 +1,8 @@ +CREATE TABLE ref_not_autoconverted(a int unique); +CREATE TABLE citus_local_autoconverted(a int unique references ref_not_autoconverted(a)); +CREATE TABLE citus_local_not_autoconverted(a int unique); +select create_reference_table('ref_not_autoconverted'); +select citus_add_local_table_to_metadata('citus_local_not_autoconverted'); +select logicalrelid, autoconverted from pg_dist_partition + where logicalrelid IN ('citus_local_autoconverted'::regclass, + 'citus_local_not_autoconverted'::regclass); diff --git a/src/test/regress/sql/upgrade_basic_after.sql b/src/test/regress/sql/upgrade_basic_after.sql index a3681e08b..b525b8964 100644 --- a/src/test/regress/sql/upgrade_basic_after.sql +++ b/src/test/regress/sql/upgrade_basic_after.sql @@ -1,7 +1,7 @@ SET search_path TO upgrade_basic, public, pg_catalog; BEGIN; - -SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' ORDER BY tablename; +-- We have the tablename filter to avoid adding an alternative output for when the coordinator is in metadata vs when not +SELECT * FROM pg_indexes WHERE schemaname = 'upgrade_basic' and tablename NOT LIKE 'r_%' ORDER BY tablename; SELECT nextval('pg_dist_shardid_seq') = MAX(shardid)+1 FROM pg_dist_shard; SELECT nextval('pg_dist_placement_placementid_seq') = MAX(placementid)+1 FROM pg_dist_placement;