From 2be14cce2eeda2f42d799f0207cbcea3fe59ca1c Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 7 Dec 2020 16:03:16 +0300 Subject: [PATCH] Adds alter_distributed_table and alter_table_set_access_method UDFs --- src/backend/columnar/cstore_tableam.c | 14 +- .../distributed/commands/alter_table.c | 1403 +++++++++++++++++ ..._table_operation_for_connected_relations.c | 59 +- .../commands/create_citus_local_table.c | 2 +- .../commands/create_distributed_table.c | 360 +---- .../distributed/commands/foreign_constraint.c | 42 + src/backend/distributed/commands/table.c | 8 +- .../distributed/deparser/citus_ruleutils.c | 9 +- .../distributed/operations/node_protocol.c | 22 +- .../distributed/operations/repair_shards.c | 5 +- .../distributed/sql/citus--9.5-1--10.0-1.sql | 3 + .../sql/downgrades/citus--10.0-1--9.5-1.sql | 3 + .../udfs/alter_distributed_table/10.0-1.sql | 9 + .../udfs/alter_distributed_table/latest.sql | 9 + .../alter_table_set_access_method/10.0-1.sql | 9 + .../alter_table_set_access_method/latest.sql | 9 + .../10.0-1.sql | 13 + .../latest.sql | 13 + src/include/columnar/cstore.h | 12 + src/include/columnar/cstore_tableam.h | 1 + src/include/distributed/citus_ruleutils.h | 3 +- src/include/distributed/commands.h | 7 +- .../distributed/coordinator_protocol.h | 5 +- src/include/distributed/metadata_utility.h | 75 +- .../expected/alter_distributed_table.out | 812 ++++++++++ .../expected/alter_distributed_table_0.out | 791 ++++++++++ .../alter_table_set_access_method.out | 394 +++++ .../alter_table_set_access_method_0.out | 6 + .../regress/expected/citus_local_tables.out | 2 +- .../expected/coordinator_shouldhaveshards.out | 162 +- .../expected/isolation_undistribute_table.out | 2 +- src/test/regress/expected/multi_extension.out | 5 +- .../regress/expected/multi_extension_0.out | 5 +- .../multi_mx_alter_distributed_table.out | 163 ++ src/test/regress/expected/pg13_with_ties.out | 2 +- src/test/regress/expected/single_node.out | 167 +- .../regress/expected/undistribute_table.out | 29 +- .../expected/undistribute_table_cascade.out | 19 +- .../expected/upgrade_list_citus_objects.out | 5 +- .../expected/upgrade_list_citus_objects_0.out | 5 +- src/test/regress/multi_mx_schedule | 1 + src/test/regress/multi_schedule | 2 + .../regress/sql/alter_distributed_table.sql | 276 ++++ .../sql/alter_table_set_access_method.sql | 115 ++ .../sql/coordinator_shouldhaveshards.sql | 51 + .../sql/multi_mx_alter_distributed_table.sql | 52 + src/test/regress/sql/single_node.sql | 44 + .../sql/undistribute_table_cascade.sql | 9 + 48 files changed, 4765 insertions(+), 449 deletions(-) create mode 100644 src/backend/distributed/commands/alter_table.c create mode 100644 src/backend/distributed/sql/udfs/alter_distributed_table/10.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/alter_distributed_table/latest.sql create mode 100644 src/backend/distributed/sql/udfs/alter_table_set_access_method/10.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/alter_table_set_access_method/latest.sql create mode 100644 src/backend/distributed/sql/udfs/worker_change_sequence_dependency/10.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/worker_change_sequence_dependency/latest.sql create mode 100644 src/test/regress/expected/alter_distributed_table.out create mode 100644 src/test/regress/expected/alter_distributed_table_0.out create mode 100644 src/test/regress/expected/alter_table_set_access_method.out create mode 100644 src/test/regress/expected/alter_table_set_access_method_0.out create mode 100644 src/test/regress/expected/multi_mx_alter_distributed_table.out create mode 100644 src/test/regress/sql/alter_distributed_table.sql create mode 100644 src/test/regress/sql/alter_table_set_access_method.sql create mode 100644 src/test/regress/sql/multi_mx_alter_distributed_table.sql diff --git a/src/backend/columnar/cstore_tableam.c b/src/backend/columnar/cstore_tableam.c index 6a24736ba..6c52660ef 100644 --- a/src/backend/columnar/cstore_tableam.c +++ b/src/backend/columnar/cstore_tableam.c @@ -1440,18 +1440,6 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, } -/* - * ColumnarTableDDLContext holds the instance variable for the TableDDLCommandFunction - * instance described below. - */ -typedef struct ColumnarTableDDLContext -{ - char *schemaName; - char *relationName; - ColumnarOptions options; -} ColumnarTableDDLContext; - - /* * GetTableDDLCommandColumnar is an internal function used to turn a * ColumnarTableDDLContext stored on the context of a TableDDLCommandFunction into a sql @@ -1477,7 +1465,7 @@ GetTableDDLCommandColumnar(void *context) * command that will be executed against a shard. The resulting command will set the * options of the shard to the same options as the relation the shard is based on. */ -static char * +char * GetShardedTableDDLCommandColumnar(uint64 shardId, void *context) { ColumnarTableDDLContext *tableDDLContext = (ColumnarTableDDLContext *) context; diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c new file mode 100644 index 000000000..32f6fcd13 --- /dev/null +++ b/src/backend/distributed/commands/alter_table.c @@ -0,0 +1,1403 @@ +/*------------------------------------------------------------------------- + * + * alter_table.c + * Routines related to the altering of tables. + * + * There are three UDFs defined in this file: + * undistribute_table: + * Turns a distributed table to a local table + * alter_distributed_table: + * Alters distribution_column, shard_count or colocate_with + * properties of a distributed table + * alter_table_set_access_method: + * Changes the access method of a table + * + * All three methods work in similar steps: + * - Create a new table the required way (with a different + * shard count, distribution column, colocate with value, + * access method or local) + * - Move everything to the new table + * - Drop the old one + * + * Copyright (c) Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "fmgr.h" + +#include "access/hash.h" +#include "catalog/dependency.h" +#include "catalog/pg_am.h" +#include "columnar/cstore.h" +#include "columnar/cstore_tableam.h" +#include "distributed/colocation_utils.h" +#include "distributed/commands.h" +#include "distributed/commands/utility_hook.h" +#include "distributed/coordinator_protocol.h" +#include "distributed/deparser.h" +#include "distributed/distribution_column.h" +#include "distributed/listutils.h" +#include "distributed/metadata/dependency.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_sync.h" +#include "distributed/multi_executor.h" +#include "distributed/multi_logical_planner.h" +#include "distributed/multi_partitioning_utils.h" +#include "distributed/reference_table_utils.h" +#include "distributed/worker_protocol.h" +#include "distributed/worker_transaction.h" +#include "executor/spi.h" +#include "nodes/pg_list.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + + +/* Table Conversion Types */ +#define UNDISTRIBUTE_TABLE 'u' +#define ALTER_DISTRIBUTED_TABLE 'a' +#define ALTER_TABLE_SET_ACCESS_METHOD 'm' + +#define UNDISTRIBUTE_TABLE_CASCADE_HINT \ + "Use cascade option to undistribute all the relations involved in " \ + "a foreign key relationship with %s by executing SELECT " \ + "undistribute_table($$%s$$, cascade_via_foreign_keys=>true)" + +typedef struct TableConversionReturn +{ + /* + * commands to create foreign keys for the table + * + * When the table conversion is cascaded we can recreate + * some of the foreign keys of the cascaded tables. So with this + * list we can return it to the initial conversion operation so + * foreign keys can be created after every colocated table is + * converted. + */ + List *foreignKeyCommands; +}TableConversionReturn; + + +typedef TableConversionReturn *(*TableConversionFunction)(struct + TableConversionParameters *); + + +/* + * TableConversionState objects are used for table conversion functions: + * UndistributeTable, AlterDistributedTable, AlterTableSetAccessMethod. + * + * They can be created using TableConversionParameters objects with + * CreateTableConversion function. + * + * TableConversionState objects include everything TableConversionParameters + * objects do and some extra to be used in the conversion process. + */ +typedef struct TableConversionState +{ + /* + * Determines type of conversion: UNDISTRIBUTE_TABLE, + * ALTER_DISTRIBUTED_TABLE, ALTER_TABLE_SET_ACCESS_METHOD. + */ + char conversionType; + + /* Oid of the table to do conversion on */ + Oid relationId; + + /* + * Options to do conversions on the table + * distributionColumn is the name of the new distribution column, + * shardCountIsNull is if the shardCount variable is not given + * shardCount is the new shard count, + * colocateWith is the name of the table to colocate with, 'none', or + * 'default' + * accessMethod is the name of the new accessMethod for the table + */ + char *distributionColumn; + bool shardCountIsNull; + int shardCount; + char *colocateWith; + char *accessMethod; + + /* + * cascadeToColocated determines whether the shardCount and + * colocateWith will be cascaded to the currently colocated tables + */ + CascadeToColocatedOption cascadeToColocated; + + /* + * cascadeViaForeignKeys determines if the conversion operation + * will be cascaded to the graph connected with foreign keys + * to the table + */ + bool cascadeViaForeignKeys; + + + /* schema of the table */ + char *schemaName; + Oid schemaId; + + /* name of the table */ + char *relationName; + + /* new relation oid after the conversion */ + Oid newRelationId; + + /* temporary name for intermediate table */ + char *tempName; + + /*hash that is appended to the name to create tempName */ + uint32 hashOfName; + + /* shard count of the table before conversion */ + int originalShardCount; + + /* list of the table oids of tables colocated with the table before conversion */ + List *colocatedTableList; + + /* new distribution key, if distributionColumn variable is given */ + Var *distributionKey; + + /* distribution key of the table before conversion */ + Var *originalDistributionKey; + + /* access method name of the table before conversion */ + char *originalAccessMethod; + + /* + * The function that will be used for the conversion + * Must comply with conversionType + * UNDISTRIBUTE_TABLE -> UndistributeTable + * ALTER_DISTRIBUTED_TABLE -> AlterDistributedTable + * ALTER_TABLE_SET_ACCESS_METHOD -> AlterTableSetAccessMethod + */ + TableConversionFunction function; +} TableConversionState; + + +static TableConversionReturn * AlterDistributedTable(TableConversionParameters *params); +static TableConversionReturn * AlterTableSetAccessMethod( + TableConversionParameters *params); +static TableConversionReturn * ConvertTable(TableConversionState *con); +static void EnsureTableNotReferencing(Oid relationId, char conversionType); +static void EnsureTableNotReferenced(Oid relationId, char conversionType); +static void EnsureTableNotForeign(Oid relationId); +static void EnsureTableNotPartition(Oid relationId); +static TableConversionState * CreateTableConversion(TableConversionParameters *params); +static void CreateDistributedTableLike(TableConversionState *con); +static void CreateCitusTableLike(TableConversionState *con); +static List * GetViewCreationCommandsOfTable(Oid relationId); +static void ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands); +static void CheckAlterDistributedTableConversionParameters(TableConversionState *con); +static char * CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, + char *sequenceName, + char *sourceSchemaName, + char *sourceName, + char *targetSchemaName, + char *targetName); +static bool WillRecreateForeignKeyToReferenceTable(Oid relationId, + CascadeToColocatedOption cascadeOption); +static void WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId); +static void ExecuteQueryViaSPI(char *query, int SPIOK); + +PG_FUNCTION_INFO_V1(undistribute_table); +PG_FUNCTION_INFO_V1(alter_distributed_table); +PG_FUNCTION_INFO_V1(alter_table_set_access_method); +PG_FUNCTION_INFO_V1(worker_change_sequence_dependency); + + +/* + * undistribute_table gets a distributed table name and + * udistributes it. + */ +Datum +undistribute_table(PG_FUNCTION_ARGS) +{ + Oid relationId = PG_GETARG_OID(0); + bool cascadeViaForeignKeys = PG_GETARG_BOOL(1); + + CheckCitusVersion(ERROR); + + TableConversionParameters params = { + .relationId = relationId, + .cascadeViaForeignKeys = cascadeViaForeignKeys + }; + + UndistributeTable(¶ms); + + PG_RETURN_VOID(); +} + + +/* + * alter_distributed_table gets a distributed table and some other + * parameters and alters some properties of the table according to + * the parameters. + */ +Datum +alter_distributed_table(PG_FUNCTION_ARGS) +{ + Oid relationId = PG_GETARG_OID(0); + + char *distributionColumn = NULL; + if (!PG_ARGISNULL(1)) + { + text *distributionColumnText = PG_GETARG_TEXT_P(1); + distributionColumn = text_to_cstring(distributionColumnText); + } + + int shardCount = 0; + bool shardCountIsNull = true; + if (!PG_ARGISNULL(2)) + { + shardCount = PG_GETARG_INT32(2); + shardCountIsNull = false; + } + + char *colocateWith = NULL; + if (!PG_ARGISNULL(3)) + { + text *colocateWithText = PG_GETARG_TEXT_P(3); + colocateWith = text_to_cstring(colocateWithText); + } + + CascadeToColocatedOption cascadeToColocated = CASCADE_TO_COLOCATED_UNSPECIFIED; + if (!PG_ARGISNULL(4)) + { + if (PG_GETARG_BOOL(4) == true) + { + cascadeToColocated = CASCADE_TO_COLOCATED_YES; + } + else + { + cascadeToColocated = CASCADE_TO_COLOCATED_NO; + } + } + + CheckCitusVersion(ERROR); + + + TableConversionParameters params = { + .relationId = relationId, + .distributionColumn = distributionColumn, + .shardCountIsNull = shardCountIsNull, + .shardCount = shardCount, + .colocateWith = colocateWith, + .cascadeToColocated = cascadeToColocated + }; + + AlterDistributedTable(¶ms); + + PG_RETURN_VOID(); +} + + +/* + * alter_table_set_access_method gets a distributed table and an access + * method and changes table's access method into that. + */ +Datum +alter_table_set_access_method(PG_FUNCTION_ARGS) +{ + Oid relationId = PG_GETARG_OID(0); + + text *accessMethodText = PG_GETARG_TEXT_P(1); + char *accessMethod = text_to_cstring(accessMethodText); + + CheckCitusVersion(ERROR); + + TableConversionParameters params = { + .relationId = relationId, + .accessMethod = accessMethod + }; + + AlterTableSetAccessMethod(¶ms); + + PG_RETURN_VOID(); +} + + +/* + * worker_change_sequence_dependency is a wrapper UDF for + * changeDependencyFor function + */ +Datum +worker_change_sequence_dependency(PG_FUNCTION_ARGS) +{ + Oid sequenceOid = PG_GETARG_OID(0); + Oid sourceRelationOid = PG_GETARG_OID(1); + Oid targetRelationOid = PG_GETARG_OID(2); + + changeDependencyFor(RelationRelationId, sequenceOid, + RelationRelationId, sourceRelationOid, targetRelationOid); + PG_RETURN_VOID(); +} + + +/* + * UndistributeTable undistributes the given table. It uses ConvertTable function to + * create a new local table and move everything to that table. + * + * The local tables, tables with references, partition tables and foreign tables are + * not supported. The function gives errors in these cases. + */ +TableConversionReturn * +UndistributeTable(TableConversionParameters *params) +{ + EnsureCoordinator(); + EnsureRelationExists(params->relationId); + EnsureTableOwner(params->relationId); + + if (!IsCitusTable(params->relationId)) + { + ereport(ERROR, (errmsg("cannot undistribute table " + "because the table is not distributed"))); + } + + if (!params->cascadeViaForeignKeys) + { + EnsureTableNotReferencing(params->relationId, UNDISTRIBUTE_TABLE); + EnsureTableNotReferenced(params->relationId, UNDISTRIBUTE_TABLE); + } + EnsureTableNotForeign(params->relationId); + EnsureTableNotPartition(params->relationId); + + if (PartitionedTable(params->relationId)) + { + List *partitionList = PartitionList(params->relationId); + + /* + * This is a less common pattern where foreing key is directly from/to + * the partition relation as we already handled inherited foreign keys + * on partitions either by erroring out or cascading via foreign keys. + * It seems an acceptable limitation for now to ask users to drop such + * foreign keys manually. + */ + ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(partitionList); + } + + params->conversionType = UNDISTRIBUTE_TABLE; + params->shardCountIsNull = true; + TableConversionState *con = CreateTableConversion(params); + return ConvertTable(con); +} + + +/* + * AlterDistributedTable changes some properties of the given table. It uses + * ConvertTable function to create a new local table and move everything to that table. + * + * The local and reference tables, tables with references, partition tables and foreign + * tables are not supported. The function gives errors in these cases. + */ +TableConversionReturn * +AlterDistributedTable(TableConversionParameters *params) +{ + EnsureCoordinator(); + EnsureRelationExists(params->relationId); + EnsureTableOwner(params->relationId); + + if (!IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) + { + ereport(ERROR, (errmsg("cannot alter table because the table " + "is not distributed"))); + } + + EnsureTableNotForeign(params->relationId); + EnsureTableNotPartition(params->relationId); + EnsureHashDistributedTable(params->relationId); + + params->conversionType = ALTER_DISTRIBUTED_TABLE; + TableConversionState *con = CreateTableConversion(params); + CheckAlterDistributedTableConversionParameters(con); + + if (WillRecreateForeignKeyToReferenceTable(con->relationId, con->cascadeToColocated)) + { + ereport(DEBUG1, (errmsg("setting multi shard modify mode to sequential"))); + SetLocalMultiShardModifyModeToSequential(); + } + return ConvertTable(con); +} + + +/* + * AlterTableSetAccessMethod changes the access method of the given table. It uses + * ConvertTable function to create a new table with the access method and move everything + * to that table. + * + * The local and references tables, tables with references, partition tables and foreign + * tables are not supported. The function gives errors in these cases. + */ +TableConversionReturn * +AlterTableSetAccessMethod(TableConversionParameters *params) +{ + EnsureRelationExists(params->relationId); + EnsureTableOwner(params->relationId); + + if (IsCitusTable(params->relationId)) + { + EnsureCoordinator(); + } + + EnsureTableNotReferencing(params->relationId, ALTER_TABLE_SET_ACCESS_METHOD); + EnsureTableNotReferenced(params->relationId, ALTER_TABLE_SET_ACCESS_METHOD); + EnsureTableNotForeign(params->relationId); + if (IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) + { + EnsureHashDistributedTable(params->relationId); + } + + if (PartitionedTable(params->relationId)) + { + ereport(ERROR, (errmsg("you cannot alter access method of a partitioned table"))); + } + + if (PartitionTable(params->relationId) && + IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) + { + Oid parentRelationId = PartitionParentOid(params->relationId); + if (HasForeignKeyToReferenceTable(parentRelationId)) + { + ereport(DEBUG1, (errmsg("setting multi shard modify mode to sequential"))); + SetLocalMultiShardModifyModeToSequential(); + } + } + + params->conversionType = ALTER_TABLE_SET_ACCESS_METHOD; + params->shardCountIsNull = true; + TableConversionState *con = CreateTableConversion(params); + return ConvertTable(con); +} + + +/* + * ConvertTable is used for converting a table into a new table with different properties. + * The conversion is done by creating a new table, moving everything to the new table and + * dropping the old one. So the oid of the table is not preserved. + * + * The new table will have the same name, columns and rows. It will also have partitions, + * views, sequences of the old table. Finally it will have everything created by + * GetPostLoadTableCreationCommands function, which include indexes. These will be + * re-created during conversion, so their oids are not preserved either (except for + * sequences). However, their names are preserved. + * + * The dropping of old table is done with CASCADE. Anything not mentioned here will + * be dropped. + * + * The function returns a TableConversionReturn object that can stores variables that + * can be used at the caller operations. + */ +TableConversionReturn * +ConvertTable(TableConversionState *con) +{ + if (con->conversionType == UNDISTRIBUTE_TABLE && con->cascadeViaForeignKeys) + { + CascadeOperationForConnectedRelations(con->relationId, ExclusiveLock, + CASCADE_FKEY_UNDISTRIBUTE_TABLE); + + /* + * Undistributed every foreign key connected relation in our foreign key + * subgraph including itself, so return here. + */ + return NULL; + } + char *newAccessMethod = con->accessMethod ? con->accessMethod : + con->originalAccessMethod; + List *preLoadCommands = GetPreLoadTableCreationCommands(con->relationId, true, + newAccessMethod); + + bool includeIndexes = true; + if (con->accessMethod && strcmp(con->accessMethod, "columnar") == 0) + { + ereport(NOTICE, (errmsg("any index will be dropped, because " + "columnar tables cannot have indexes"))); + includeIndexes = false; + } + List *postLoadCommands = GetPostLoadTableCreationCommands(con->relationId, + includeIndexes); + List *justBeforeDropCommands = NIL; + List *attachPartitionCommands = NIL; + + postLoadCommands = list_concat(postLoadCommands, + GetViewCreationCommandsOfTable(con->relationId)); + + List *foreignKeyCommands = NIL; + if (con->conversionType == ALTER_DISTRIBUTED_TABLE) + { + foreignKeyCommands = GetForeignConstraintToReferenceTablesCommands( + con->relationId); + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES || + con->cascadeToColocated == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) + { + List *foreignKeyToDistributedTableCommands = + GetForeignConstraintToDistributedTablesCommands(con->relationId); + foreignKeyCommands = list_concat(foreignKeyCommands, + foreignKeyToDistributedTableCommands); + + List *foreignKeyFromDistributedTableCommands = + GetForeignConstraintFromDistributedTablesCommands(con->relationId); + foreignKeyCommands = list_concat(foreignKeyCommands, + foreignKeyFromDistributedTableCommands); + } + else + { + WarningsForDroppingForeignKeysWithDistributedTables(con->relationId); + } + } + + bool isPartitionTable = false; + char *attachToParentCommand = NULL; + if (PartitionTable(con->relationId)) + { + isPartitionTable = true; + char *detachFromParentCommand = GenerateDetachPartitionCommand(con->relationId); + attachToParentCommand = GenerateAlterTableAttachPartitionCommand(con->relationId); + + justBeforeDropCommands = lappend(justBeforeDropCommands, detachFromParentCommand); + } + + if (PartitionedTable(con->relationId)) + { + ereport(NOTICE, (errmsg("converting the partitions of %s", + quote_qualified_identifier(con->schemaName, + con->relationName)))); + + List *partitionList = PartitionList(con->relationId); + + Oid partitionRelationId = InvalidOid; + foreach_oid(partitionRelationId, partitionList) + { + char *detachPartitionCommand = GenerateDetachPartitionCommand( + partitionRelationId); + char *attachPartitionCommand = GenerateAlterTableAttachPartitionCommand( + partitionRelationId); + + /* + * We first detach the partitions to be able to convert them separately. + * After this they are no longer partitions, so they will not be caught by + * the checks. + */ + ExecuteQueryViaSPI(detachPartitionCommand, SPI_OK_UTILITY); + attachPartitionCommands = lappend(attachPartitionCommands, + attachPartitionCommand); + + CascadeToColocatedOption cascadeOption = CASCADE_TO_COLOCATED_NO; + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES || + con->cascadeToColocated == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) + { + cascadeOption = CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED; + } + + TableConversionParameters partitionParam = { + .relationId = partitionRelationId, + .distributionColumn = con->distributionColumn, + .shardCountIsNull = con->shardCountIsNull, + .shardCount = con->shardCount, + .cascadeToColocated = cascadeOption, + .colocateWith = con->colocateWith, + + /* + * Even if we called UndistributeTable with cascade option, we + * shouldn't cascade via foreign keys on partitions. Otherwise, + * we might try to undistribute partitions of other tables in + * our foreign key subgraph more than once. + */ + .cascadeViaForeignKeys = false + }; + + TableConversionReturn *partitionReturn = con->function(&partitionParam); + if (cascadeOption == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) + { + foreignKeyCommands = list_concat(foreignKeyCommands, + partitionReturn->foreignKeyCommands); + } + } + } + + ereport(NOTICE, (errmsg("creating a new table for %s", + quote_qualified_identifier(con->schemaName, + con->relationName)))); + + TableDDLCommand *tableCreationCommand = NULL; + foreach_ptr(tableCreationCommand, preLoadCommands) + { + Assert(CitusIsA(tableCreationCommand, TableDDLCommand)); + + char *tableCreationSql = GetTableDDLCommand(tableCreationCommand); + Node *parseTree = ParseTreeNode(tableCreationSql); + + RelayEventExtendNames(parseTree, con->schemaName, con->hashOfName); + CitusProcessUtility(parseTree, tableCreationSql, PROCESS_UTILITY_TOPLEVEL, + NULL, None_Receiver, NULL); + } + + /* set columnar options */ +#if HAS_TABLEAM + if (con->accessMethod == NULL && con->originalAccessMethod && + strcmp(con->originalAccessMethod, "columnar") == 0) + { + ColumnarOptions options = { 0 }; + ReadColumnarOptions(con->relationId, &options); + + ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0( + sizeof(ColumnarTableDDLContext)); + + /* build the context */ + context->schemaName = con->schemaName; + context->relationName = con->relationName; + context->options = options; + + char *columnarOptionsSql = GetShardedTableDDLCommandColumnar(con->hashOfName, + context); + + ExecuteQueryViaSPI(columnarOptionsSql, SPI_OK_SELECT); + } +#endif + + con->newRelationId = get_relname_relid(con->tempName, con->schemaId); + + if (con->conversionType == ALTER_DISTRIBUTED_TABLE) + { + CreateDistributedTableLike(con); + } + else if (con->conversionType == ALTER_TABLE_SET_ACCESS_METHOD) + { + CreateCitusTableLike(con); + } + + ReplaceTable(con->relationId, con->newRelationId, justBeforeDropCommands); + + TableDDLCommand *tableConstructionCommand = NULL; + foreach_ptr(tableConstructionCommand, postLoadCommands) + { + Assert(CitusIsA(tableConstructionCommand, TableDDLCommand)); + char *tableConstructionSQL = GetTableDDLCommand(tableConstructionCommand); + ExecuteQueryViaSPI(tableConstructionSQL, SPI_OK_UTILITY); + } + + char *attachPartitionCommand = NULL; + foreach_ptr(attachPartitionCommand, attachPartitionCommands) + { + Node *parseTree = ParseTreeNode(attachPartitionCommand); + + CitusProcessUtility(parseTree, attachPartitionCommand, PROCESS_UTILITY_TOPLEVEL, + NULL, None_Receiver, NULL); + } + + if (isPartitionTable) + { + ExecuteQueryViaSPI(attachToParentCommand, SPI_OK_UTILITY); + } + + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES) + { + Oid colocatedTableId = InvalidOid; + + /* For now we only support cascade to colocation for alter_distributed_table UDF */ + Assert(con->conversionType == ALTER_DISTRIBUTED_TABLE); + foreach_oid(colocatedTableId, con->colocatedTableList) + { + if (colocatedTableId == con->relationId) + { + continue; + } + char *qualifiedRelationName = quote_qualified_identifier(con->schemaName, + con->relationName); + + TableConversionParameters cascadeParam = { + .relationId = colocatedTableId, + .shardCountIsNull = con->shardCountIsNull, + .shardCount = con->shardCount, + .colocateWith = qualifiedRelationName, + .cascadeToColocated = CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED + }; + TableConversionReturn *colocatedReturn = con->function(&cascadeParam); + foreignKeyCommands = list_concat(foreignKeyCommands, + colocatedReturn->foreignKeyCommands); + } + } + + /* recreate foreign keys */ + TableConversionReturn *ret = NULL; + if (con->conversionType == ALTER_DISTRIBUTED_TABLE) + { + if (con->cascadeToColocated != CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) + { + char *foreignKeyCommand = NULL; + foreach_ptr(foreignKeyCommand, foreignKeyCommands) + { + ExecuteQueryViaSPI(foreignKeyCommand, SPI_OK_UTILITY); + } + } + else + { + ret = palloc0(sizeof(TableConversionReturn)); + ret->foreignKeyCommands = foreignKeyCommands; + } + } + + return ret; +} + + +/* + * EnsureTableNotReferencing checks if the table has a reference to another + * table and errors if it is. + */ +void +EnsureTableNotReferencing(Oid relationId, char conversionType) +{ + if (TableReferencing(relationId)) + { + if (conversionType == UNDISTRIBUTE_TABLE) + { + char *qualifiedRelationName = generate_qualified_relation_name(relationId); + ereport(ERROR, (errmsg("cannot complete operation " + "because table %s has a foreign key", + get_rel_name(relationId)), + errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, + qualifiedRelationName, + qualifiedRelationName))); + } + else + { + ereport(ERROR, (errmsg("cannot complete operation " + "because table %s has a foreign key", + get_rel_name(relationId)))); + } + } +} + + +/* + * EnsureTableNotReferenced checks if the table is referenced by another + * table and errors if it is. + */ +void +EnsureTableNotReferenced(Oid relationId, char conversionType) +{ + if (TableReferenced(relationId)) + { + if (conversionType == UNDISTRIBUTE_TABLE) + { + char *qualifiedRelationName = generate_qualified_relation_name(relationId); + ereport(ERROR, (errmsg("cannot complete operation " + "because table %s is referenced by a foreign key", + get_rel_name(relationId)), + errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, + qualifiedRelationName, + qualifiedRelationName))); + } + else + { + ereport(ERROR, (errmsg("cannot complete operation " + "because table %s is referenced by a foreign key", + get_rel_name(relationId)))); + } + } +} + + +/* + * EnsureTableNotForeign checks if the table is a foreign table and errors + * if it is. + */ +void +EnsureTableNotForeign(Oid relationId) +{ + char relationKind = get_rel_relkind(relationId); + if (relationKind == RELKIND_FOREIGN_TABLE) + { + ereport(ERROR, (errmsg("cannot complete operation " + "because it is a foreign table"))); + } +} + + +/* + * EnsureTableNotPartition checks if the table is a partition of another + * table and errors if it is. + */ +void +EnsureTableNotPartition(Oid relationId) +{ + if (PartitionTable(relationId)) + { + Oid parentRelationId = PartitionParentOid(relationId); + char *parentRelationName = get_rel_name(parentRelationId); + ereport(ERROR, (errmsg("cannot complete operation " + "because table is a partition"), + errhint("the parent table is \"%s\"", + parentRelationName))); + } +} + + +TableConversionState * +CreateTableConversion(TableConversionParameters *params) +{ + TableConversionState *con = palloc0(sizeof(TableConversionState)); + + con->conversionType = params->conversionType; + con->relationId = params->relationId; + con->distributionColumn = params->distributionColumn; + con->shardCountIsNull = params->shardCountIsNull; + con->shardCount = params->shardCount; + con->colocateWith = params->colocateWith; + con->accessMethod = params->accessMethod; + con->cascadeToColocated = params->cascadeToColocated; + con->cascadeViaForeignKeys = params->cascadeViaForeignKeys; + + Relation relation = try_relation_open(con->relationId, ExclusiveLock); + if (relation == NULL) + { + ereport(ERROR, (errmsg("cannot complete operation " + "because no such table exists"))); + } + relation_close(relation, NoLock); + con->distributionKey = + BuildDistributionKeyFromColumnName(relation, con->distributionColumn); + + con->originalAccessMethod = NULL; +#if PG_VERSION_NUM >= PG_VERSION_12 + if (!PartitionedTable(con->relationId)) + { + HeapTuple amTuple = SearchSysCache1(AMOID, ObjectIdGetDatum( + relation->rd_rel->relam)); + if (!HeapTupleIsValid(amTuple)) + { + ereport(ERROR, (errmsg("cache lookup failed for access method %d", + relation->rd_rel->relam))); + } + Form_pg_am amForm = (Form_pg_am) GETSTRUCT(amTuple); + con->originalAccessMethod = NameStr(amForm->amname); + ReleaseSysCache(amTuple); + } +#endif + + + con->colocatedTableList = NIL; + if (IsCitusTableType(con->relationId, DISTRIBUTED_TABLE)) + { + con->originalDistributionKey = DistPartitionKey(con->relationId); + + CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(con->relationId); + con->originalShardCount = cacheEntry->shardIntervalArrayLength; + + List *colocatedTableList = ColocatedTableList(con->relationId); + + /* + * we will not add partition tables to the colocatedTableList + * since they will be handled separately. + */ + Oid colocatedTableId = InvalidOid; + foreach_oid(colocatedTableId, colocatedTableList) + { + if (PartitionTable(colocatedTableId)) + { + continue; + } + con->colocatedTableList = lappend_oid(con->colocatedTableList, + colocatedTableId); + } + + /* sort the oids to avoid deadlock */ + con->colocatedTableList = SortList(con->colocatedTableList, CompareOids); + } + + /* find relation and schema names */ + con->relationName = get_rel_name(con->relationId); + con->schemaId = get_rel_namespace(con->relationId); + con->schemaName = get_namespace_name(con->schemaId); + + /* calculate a temp name for the new table */ + con->tempName = pstrdup(con->relationName); + con->hashOfName = hash_any((unsigned char *) con->tempName, strlen(con->tempName)); + AppendShardIdToName(&con->tempName, con->hashOfName); + + if (con->conversionType == UNDISTRIBUTE_TABLE) + { + con->function = &UndistributeTable; + } + else if (con->conversionType == ALTER_DISTRIBUTED_TABLE) + { + con->function = &AlterDistributedTable; + } + else if (con->conversionType == ALTER_TABLE_SET_ACCESS_METHOD) + { + con->function = &AlterTableSetAccessMethod; + } + + return con; +} + + +/* + * CreateDistributedTableLike distributes the new table in con parameter + * like the old one. It checks the distribution column, colocation and + * shard count and if they are not changed sets them to the old table's values. + */ +void +CreateDistributedTableLike(TableConversionState *con) +{ + Var *newDistributionKey = + con->distributionColumn ? con->distributionKey : con->originalDistributionKey; + + char *newColocateWith = con->colocateWith; + if (con->colocateWith == NULL) + { + /* + * If the new distribution column and the old one have the same data type + * and the shard_count parameter is null (which means shard count will not + * change) we can create the new table in the same colocation as the old one. + * In this case we set the new table's colocate_with value as the old table + * so we don't even change the colocation id of the table during conversion. + */ + if (con->originalDistributionKey->vartype == newDistributionKey->vartype && + con->shardCountIsNull) + { + newColocateWith = + quote_qualified_identifier(con->schemaName, con->relationName); + } + else + { + newColocateWith = "default"; + } + } + int newShardCount = 0; + if (con->shardCountIsNull) + { + newShardCount = con->originalShardCount; + } + else + { + newShardCount = con->shardCount; + } + char partitionMethod = PartitionMethod(con->relationId); + CreateDistributedTable(con->newRelationId, newDistributionKey, partitionMethod, + newShardCount, newColocateWith, false); +} + + +/* + * CreateCitusTableLike converts the new table to the Citus table type + * of the old table. + */ +void +CreateCitusTableLike(TableConversionState *con) +{ + if (IsCitusTableType(con->relationId, DISTRIBUTED_TABLE)) + { + CreateDistributedTableLike(con); + } + else if (IsCitusTableType(con->relationId, REFERENCE_TABLE)) + { + CreateDistributedTable(con->newRelationId, NULL, DISTRIBUTE_BY_NONE, 0, + NULL, false); + } + else if (IsCitusTableType(con->relationId, CITUS_LOCAL_TABLE)) + { + CreateCitusLocalTable(con->newRelationId, false); + + /* + * creating Citus local table adds a shell table on top + * so we need its oid now + */ + con->newRelationId = get_relname_relid(con->tempName, con->schemaId); + } +} + + +/* + * GetViewCreationCommandsOfTable takes a table oid generates the CREATE VIEW + * commands for views that depend to the given table. This includes the views + * that recursively depend on the table too. + */ +List * +GetViewCreationCommandsOfTable(Oid relationId) +{ + List *views = GetDependingViews(relationId); + List *commands = NIL; + + Oid viewOid = InvalidOid; + foreach_oid(viewOid, views) + { + Datum viewDefinitionDatum = DirectFunctionCall1(pg_get_viewdef, + ObjectIdGetDatum(viewOid)); + char *viewDefinition = TextDatumGetCString(viewDefinitionDatum); + StringInfo query = makeStringInfo(); + char *viewName = get_rel_name(viewOid); + char *schemaName = get_namespace_name(get_rel_namespace(viewOid)); + char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName); + appendStringInfo(query, + "CREATE VIEW %s AS %s", + qualifiedViewName, + viewDefinition); + commands = lappend(commands, makeTableDDLCommandString(query->data)); + } + return commands; +} + + +/* + * ReplaceTable replaces the source table with the target table. + * It moves all the rows of the source table to target table with INSERT SELECT. + * Changes the dependencies of the sequences owned by source table to target table. + * Then drops the source table and renames the target table to source tables name. + * + * Source and target tables need to be in the same schema and have the same columns. + */ +void +ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands) +{ + char *sourceName = get_rel_name(sourceId); + char *targetName = get_rel_name(targetId); + Oid schemaId = get_rel_namespace(sourceId); + char *schemaName = get_namespace_name(schemaId); + + StringInfo query = makeStringInfo(); + + if (!PartitionedTable(sourceId)) + { + ereport(NOTICE, (errmsg("Moving the data of %s", + quote_qualified_identifier(schemaName, sourceName)))); + + appendStringInfo(query, "INSERT INTO %s SELECT * FROM %s", + quote_qualified_identifier(schemaName, targetName), + quote_qualified_identifier(schemaName, sourceName)); + ExecuteQueryViaSPI(query->data, SPI_OK_INSERT); + } + +#if PG_VERSION_NUM >= PG_VERSION_13 + List *ownedSequences = getOwnedSequences(sourceId); +#else + List *ownedSequences = getOwnedSequences(sourceId, InvalidAttrNumber); +#endif + Oid sequenceOid = InvalidOid; + foreach_oid(sequenceOid, ownedSequences) + { + changeDependencyFor(RelationRelationId, sequenceOid, + RelationRelationId, sourceId, targetId); + if (ShouldSyncTableMetadata(sourceId)) + { + Oid sequenceSchemaOid = get_rel_namespace(sequenceOid); + char *sequenceSchemaName = get_namespace_name(sequenceSchemaOid); + char *sequenceName = get_rel_name(sequenceOid); + char *workerChangeSequenceDependencyCommand = + CreateWorkerChangeSequenceDependencyCommand(sequenceSchemaName, + sequenceName, + schemaName, sourceName, + schemaName, targetName); + SendCommandToWorkersWithMetadata(workerChangeSequenceDependencyCommand); + } + } + + char *justBeforeDropCommand = NULL; + foreach_ptr(justBeforeDropCommand, justBeforeDropCommands) + { + ExecuteQueryViaSPI(justBeforeDropCommand, SPI_OK_UTILITY); + } + + ereport(NOTICE, (errmsg("Dropping the old %s", + quote_qualified_identifier(schemaName, sourceName)))); + + resetStringInfo(query); + appendStringInfo(query, "DROP TABLE %s CASCADE", + quote_qualified_identifier(schemaName, sourceName)); + ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); + + ereport(NOTICE, (errmsg("Renaming the new table to %s", + quote_qualified_identifier(schemaName, sourceName)))); + + resetStringInfo(query); + appendStringInfo(query, "ALTER TABLE %s RENAME TO %s", + quote_qualified_identifier(schemaName, targetName), + quote_identifier(sourceName)); + ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); +} + + +/* + * CheckAlterDistributedTableConversionParameters errors for the cases where + * alter_distributed_table UDF wouldn't work. + */ +void +CheckAlterDistributedTableConversionParameters(TableConversionState *con) +{ + /* Changing nothing is not allowed */ + if (con->distributionColumn == NULL && con->shardCountIsNull && + con->colocateWith == NULL && con->cascadeToColocated != CASCADE_TO_COLOCATED_YES) + { + ereport(ERROR, (errmsg("you have to specify at least one of the " + "distribution_column, shard_count or " + "colocate_with parameters"))); + } + + /* check if the parameters in this conversion are given and same with table's properties */ + bool sameDistColumn = false; + if (con->distributionColumn != NULL && + equal(con->distributionKey, con->originalDistributionKey)) + { + sameDistColumn = true; + } + + bool sameShardCount = false; + if (!con->shardCountIsNull && con->originalShardCount == con->shardCount) + { + sameShardCount = true; + } + + bool sameColocateWith = false; + if (con->colocateWith != NULL && strcmp(con->colocateWith, "default") != 0 && + strcmp(con->colocateWith, "none") != 0) + { + /* check if already colocated with colocate_with */ + Oid colocatedTableOid = InvalidOid; + text *colocateWithText = cstring_to_text(con->colocateWith); + Oid colocateWithTableOid = ResolveRelationId(colocateWithText, false); + foreach_oid(colocatedTableOid, con->colocatedTableList) + { + if (colocateWithTableOid == colocatedTableOid) + { + sameColocateWith = true; + break; + } + } + + /* + * already found colocateWithTableOid so let's check if + * colocate_with table is a distributed table + */ + if (!IsCitusTableType(colocateWithTableOid, DISTRIBUTED_TABLE)) + { + ereport(ERROR, (errmsg("cannot colocate with %s because " + "it is not a distributed table", + con->colocateWith))); + } + } + + /* shard_count:=0 is not allowed */ + if (!con->shardCountIsNull && con->shardCount == 0) + { + ereport(ERROR, (errmsg("shard_count cannot be 0"), + errhint("if you no longer want this to be a " + "distributed table you can try " + "undistribute_table() function"))); + } + + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && + con->distributionColumn != NULL) + { + ereport(ERROR, (errmsg("distribution_column cannot be " + "cascaded to colocated tables"))); + } + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && con->shardCountIsNull && + con->colocateWith == NULL) + { + ereport(ERROR, (errmsg("shard_count or colocate_with is necessary " + "for cascading to colocated tables"))); + } + + /* + * if every parameter is either not given or already the + * same then give error + */ + if ((con->distributionColumn == NULL || sameDistColumn) && + (con->shardCountIsNull || sameShardCount) && + (con->colocateWith == NULL || sameColocateWith)) + { + ereport(ERROR, (errmsg("this call doesn't change any properties of the table"), + errhint("check citus_tables view to see current " + "properties of the table"))); + } + if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && + con->colocateWith != NULL && + strcmp(con->colocateWith, "none") == 0) + { + ereport(ERROR, (errmsg("colocate_with := 'none' cannot be " + "cascaded to colocated tables"))); + } + + int colocatedTableCount = list_length(con->colocatedTableList) - 1; + if (colocatedTableCount > 0 && !con->shardCountIsNull && !sameShardCount && + con->cascadeToColocated == CASCADE_TO_COLOCATED_UNSPECIFIED) + { + ereport(ERROR, (errmsg("cascade_to_colocated parameter is necessary"), + errdetail("this table is colocated with some other tables"), + errhint("cascade_to_colocated := false will break the " + "current colocation, cascade_to_colocated := true " + "will change the shard count of colocated tables " + "too."))); + } + + if (con->colocateWith != NULL && strcmp(con->colocateWith, "default") != 0 && + strcmp(con->colocateWith, "none") != 0) + { + text *colocateWithText = cstring_to_text(con->colocateWith); + Oid colocateWithTableOid = ResolveRelationId(colocateWithText, false); + CitusTableCacheEntry *colocateWithTableCacheEntry = + GetCitusTableCacheEntry(colocateWithTableOid); + int colocateWithTableShardCount = + colocateWithTableCacheEntry->shardIntervalArrayLength; + + if (!con->shardCountIsNull && con->shardCount != colocateWithTableShardCount) + { + ereport(ERROR, (errmsg("shard_count cannot be different than the shard " + "count of the table in colocate_with"), + errhint("if no shard_count is specified shard count " + "will be same with colocate_with table's"))); + } + + if (colocateWithTableShardCount != con->originalShardCount) + { + /* + * shardCount is either 0 or already same with colocateWith table's + * It's ok to set shardCountIsNull to false because we assume giving a table + * to colocate with and no shard count is the same with giving colocate_with + * table's shard count if it is different than the original. + * So it is almost like the shard_count parameter was given by the user. + */ + con->shardCount = colocateWithTableShardCount; + con->shardCountIsNull = false; + } + + Var *colocateWithPartKey = DistPartitionKey(colocateWithTableOid); + + if (con->distributionColumn && + colocateWithPartKey->vartype != con->distributionKey->vartype) + { + ereport(ERROR, (errmsg("cannot colocate with %s and change distribution " + "column to %s because data type of column %s is " + "different then the distribution column of the %s", + con->colocateWith, con->distributionColumn, + con->distributionColumn, con->colocateWith))); + } + else if (!con->distributionColumn && + colocateWithPartKey->vartype != con->originalDistributionKey->vartype) + { + ereport(ERROR, (errmsg("cannot colocate with %s because data type of its " + "distribution column is different than %s", + con->colocateWith, con->relationName))); + } + } + + /* Notices for no operation UDF calls */ + if (sameDistColumn) + { + ereport(NOTICE, (errmsg("table is already distributed by %s", + con->distributionColumn))); + } + + if (sameShardCount) + { + ereport(NOTICE, (errmsg("shard count of the table is already %d", + con->shardCount))); + } + + if (sameColocateWith) + { + ereport(NOTICE, (errmsg("table is already colocated with %s", + con->colocateWith))); + } +} + + +/* + * CreateWorkerChangeSequenceDependencyCommand creates and returns a + * worker_change_sequence_dependency query with the parameters. + */ +static char * +CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, char *sequenceName, + char *sourceSchemaName, char *sourceName, + char *targetSchemaName, char *targetName) +{ + StringInfo query = makeStringInfo(); + appendStringInfo(query, "SELECT worker_change_sequence_dependency('%s', '%s', '%s')", + quote_qualified_identifier(sequenceSchemaName, sequenceName), + quote_qualified_identifier(sourceSchemaName, sourceName), + quote_qualified_identifier(targetSchemaName, targetName)); + return query->data; +} + + +/* + * WillRecreateForeignKeyToReferenceTable checks if the table of relationId has any foreign + * key to a reference table, if conversion will be cascaded to colocated table this function + * also checks if any of the colocated tables have a foreign key to a reference table too + */ +bool +WillRecreateForeignKeyToReferenceTable(Oid relationId, + CascadeToColocatedOption cascadeOption) +{ + if (cascadeOption == CASCADE_TO_COLOCATED_NO || + cascadeOption == CASCADE_TO_COLOCATED_UNSPECIFIED) + { + return HasForeignKeyToReferenceTable(relationId); + } + else if (cascadeOption == CASCADE_TO_COLOCATED_YES) + { + List *colocatedTableList = ColocatedTableList(relationId); + Oid colocatedTableOid = InvalidOid; + foreach_oid(colocatedTableOid, colocatedTableList) + { + if (HasForeignKeyToReferenceTable(colocatedTableOid)) + { + return true; + } + } + } + return false; +} + + +/* + * WarningsForDroppingForeignKeysWithDistributedTables gives warnings for the + * foreign keys that will be dropped because formerly colocated distributed tables + * are not colocated. + */ +void +WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId) +{ + int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; + List *referencingForeingKeys = GetForeignKeyOids(relationId, flags); + flags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; + List *referencedForeignKeys = GetForeignKeyOids(relationId, flags); + + List *foreignKeys = list_concat(referencingForeingKeys, referencedForeignKeys); + + Oid foreignKeyOid = InvalidOid; + foreach_oid(foreignKeyOid, foreignKeys) + { + ereport(WARNING, (errmsg("foreign key %s will be dropped", + get_constraint_name(foreignKeyOid)))); + } +} + + +/* + * ExecuteQueryViaSPI connects to SPI, executes the query and checks if it + * returned the OK value and finishes the SPI connection + */ +void +ExecuteQueryViaSPI(char *query, int SPIOK) +{ + int spiResult = SPI_connect(); + if (spiResult != SPI_OK_CONNECT) + { + ereport(ERROR, (errmsg("could not connect to SPI manager"))); + } + + spiResult = SPI_execute(query, false, 0); + if (spiResult != SPIOK) + { + ereport(ERROR, (errmsg("could not run SPI query"))); + } + + spiResult = SPI_finish(); + if (spiResult != SPI_OK_FINISH) + { + ereport(ERROR, (errmsg("could not finish SPI connection"))); + } +} diff --git a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c index 4615fdb19..d4de6e277 100644 --- a/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c +++ b/src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c @@ -27,9 +27,6 @@ #include "utils/lsyscache.h" -typedef void (*CascadeOperationFunction)(Oid, bool); - - static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList); static bool RelationIdListHasReferenceTable(List *relationIdList); static void LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode); @@ -42,8 +39,6 @@ static char * GetDropFkeyCascadeCommand(Oid relationId, Oid foreignKeyId); static void ExecuteCascadeOperationForRelationIdList(List *relationIdList, CascadeOperationType cascadeOperationType); -static CascadeOperationFunction GetCascadeOperationFunction(CascadeOperationType - cascadeOperationType); /* @@ -342,45 +337,37 @@ ExecuteCascadeOperationForRelationIdList(List *relationIdList, Oid relationId = InvalidOid; foreach_oid(relationId, relationIdList) { - CascadeOperationFunction cascadeOperationFunction = - GetCascadeOperationFunction(cascadeOperationType); - /* * Caller already passed the relations that we should operate on, * so we should not cascade here. */ bool cascadeViaForeignKeys = false; - cascadeOperationFunction(relationId, cascadeViaForeignKeys); - } -} - - -/* - * GetCascadeOperationFunction returns c api for citus table operation according - * to given CascadeOperationType. - */ -static CascadeOperationFunction -GetCascadeOperationFunction(CascadeOperationType cascadeOperationType) -{ - switch (cascadeOperationType) - { - case UNDISTRIBUTE_TABLE: + switch (cascadeOperationType) { - return UndistributeTable; - } + case CASCADE_FKEY_UNDISTRIBUTE_TABLE: + { + TableConversionParameters params = { + .relationId = relationId, + .cascadeViaForeignKeys = cascadeViaForeignKeys + }; + UndistributeTable(¶ms); + break; + } - case CREATE_CITUS_LOCAL_TABLE: - { - return CreateCitusLocalTable; - } + case CASCADE_FKEY_CREATE_CITUS_LOCAL_TABLE: + { + CreateCitusLocalTable(relationId, cascadeViaForeignKeys); + break; + } - default: - { - /* - * This is not expected as other create table functions don't have - * cascade option yet. To be on the safe side, error out here. - */ - ereport(ERROR, (errmsg("citus table function could not be found"))); + default: + { + /* + * This is not expected as other create table functions don't have + * cascade option yet. To be on the safe side, error out here. + */ + ereport(ERROR, (errmsg("citus table function could not be found"))); + } } } } diff --git a/src/backend/distributed/commands/create_citus_local_table.c b/src/backend/distributed/commands/create_citus_local_table.c index c6afa0fd9..1700cec34 100644 --- a/src/backend/distributed/commands/create_citus_local_table.c +++ b/src/backend/distributed/commands/create_citus_local_table.c @@ -139,7 +139,7 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys) if (tableHasExternalForeignKeys && cascadeViaForeignKeys) { CascadeOperationForConnectedRelations(relationId, lockMode, - CREATE_CITUS_LOCAL_TABLE); + CASCADE_FKEY_CREATE_CITUS_LOCAL_TABLE); /* * We converted every foreign key connected table in our subgraph diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 9494737a4..35e0a10da 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -86,10 +86,6 @@ */ #define LOG_PER_TUPLE_AMOUNT 1000000 -#define UNDISTRIBUTE_TABLE_CASCADE_HINT \ - "Use cascade option to undistribute all the relations involved in " \ - "a foreign key relationship with %s by executing SELECT " \ - "undistribute_table($$%s$$, cascade_via_foreign_keys=>true)" /* Replication model to use when creating distributed tables */ int ReplicationModel = REPLICATION_MODEL_COORDINATOR; @@ -97,11 +93,12 @@ int ReplicationModel = REPLICATION_MODEL_COORDINATOR; /* local function forward declarations */ static char DecideReplicationModel(char distributionMethod, bool viaDeprecatedAPI); -static void CreateHashDistributedTableShards(Oid relationId, Oid colocatedTableId, - bool localTableEmpty); +static void CreateHashDistributedTableShards(Oid relationId, int shardCount, + Oid colocatedTableId, bool localTableEmpty); static uint32 ColocationIdForNewTable(Oid relationId, Var *distributionColumn, char distributionMethod, char replicationModel, - char *colocateWithTableName, bool viaDeprecatedAPI); + int shardCount, char *colocateWithTableName, + bool viaDeprecatedAPI); static void EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, char distributionMethod, uint32 colocationId, char replicationModel, bool viaDeprecatedAPI); @@ -117,7 +114,6 @@ static void EnsureLocalTableEmptyIfNecessary(Oid relationId, char distributionMe static bool ShouldLocalTableBeEmpty(Oid relationId, char distributionMethod, bool viaDeprecatedAPI); static void EnsureCitusTableCanBeCreated(Oid relationOid); -static void EnsureRelationExists(Oid relationId); static bool LocalTableEmpty(Oid tableId); static void CopyLocalDataIntoShards(Oid relationId); static List * TupleDescColumnNameList(TupleDesc tupleDescriptor); @@ -129,14 +125,11 @@ static void DoCopyFromLocalTableIntoShards(Relation distributedRelation, DestReceiver *copyDest, TupleTableSlot *slot, EState *estate); -static List * GetViewCreationCommandsOfTable(Oid relationId); -static void ReplaceTable(Oid sourceId, Oid targetId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_create_distributed_table); PG_FUNCTION_INFO_V1(create_distributed_table); PG_FUNCTION_INFO_V1(create_reference_table); -PG_FUNCTION_INFO_V1(undistribute_table); /* @@ -180,7 +173,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) char distributionMethod = LookupDistributionMethod(distributionMethodOid); CreateDistributedTable(relationId, distributionColumn, distributionMethod, - colocateWithTableName, viaDeprecatedAPI); + ShardCount, colocateWithTableName, viaDeprecatedAPI); relation_close(relation, NoLock); @@ -232,7 +225,7 @@ create_distributed_table(PG_FUNCTION_ARGS) char *colocateWithTableName = text_to_cstring(colocateWithTableNameText); CreateDistributedTable(relationId, distributionColumn, distributionMethod, - colocateWithTableName, viaDeprecatedAPI); + ShardCount, colocateWithTableName, viaDeprecatedAPI); relation_close(relation, NoLock); @@ -283,7 +276,7 @@ create_reference_table(PG_FUNCTION_ARGS) } CreateDistributedTable(relationId, distributionColumn, DISTRIBUTE_BY_NONE, - colocateWithTableName, viaDeprecatedAPI); + ShardCount, colocateWithTableName, viaDeprecatedAPI); relation_close(relation, NoLock); @@ -291,24 +284,6 @@ create_reference_table(PG_FUNCTION_ARGS) } -/* - * undistribute_table gets a distributed table name and - * udistributes it. - */ -Datum -undistribute_table(PG_FUNCTION_ARGS) -{ - Oid relationId = PG_GETARG_OID(0); - bool cascadeViaForeignKeys = PG_GETARG_BOOL(1); - - CheckCitusVersion(ERROR); - - UndistributeTable(relationId, cascadeViaForeignKeys); - - PG_RETURN_VOID(); -} - - /* * EnsureCitusTableCanBeCreated checks if * - we are on the coordinator @@ -335,7 +310,7 @@ EnsureCitusTableCanBeCreated(Oid relationOid) * EnsureRelationExists does a basic check on whether the OID belongs to * an existing relation. */ -static void +void EnsureRelationExists(Oid relationId) { if (!RelationExists(relationId)) @@ -361,7 +336,7 @@ EnsureRelationExists(Oid relationId) */ void CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributionMethod, - char *colocateWithTableName, bool viaDeprecatedAPI) + int shardCount, char *colocateWithTableName, bool viaDeprecatedAPI) { /* * distributed tables might have dependencies on different objects, since we create @@ -382,7 +357,7 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio */ uint32 colocationId = ColocationIdForNewTable(relationId, distributionColumn, distributionMethod, replicationModel, - colocateWithTableName, + shardCount, colocateWithTableName, viaDeprecatedAPI); EnsureRelationCanBeDistributed(relationId, distributionColumn, distributionMethod, @@ -421,7 +396,8 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio /* create shards for hash distributed and reference tables */ if (distributionMethod == DISTRIBUTE_BY_HASH) { - CreateHashDistributedTableShards(relationId, colocatedTableId, localTableEmpty); + CreateHashDistributedTableShards(relationId, shardCount, colocatedTableId, + localTableEmpty); } else if (distributionMethod == DISTRIBUTE_BY_NONE) { @@ -450,8 +426,8 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio foreach_oid(partitionRelationId, partitionList) { CreateDistributedTable(partitionRelationId, distributionColumn, - distributionMethod, colocateWithTableName, - viaDeprecatedAPI); + distributionMethod, shardCount, + colocateWithTableName, viaDeprecatedAPI); } } @@ -519,8 +495,8 @@ DecideReplicationModel(char distributionMethod, bool viaDeprecatedAPI) * CreateHashDistributedTableShards creates shards of given hash distributed table. */ static void -CreateHashDistributedTableShards(Oid relationId, Oid colocatedTableId, - bool localTableEmpty) +CreateHashDistributedTableShards(Oid relationId, int shardCount, + Oid colocatedTableId, bool localTableEmpty) { bool useExclusiveConnection = false; @@ -545,10 +521,9 @@ CreateHashDistributedTableShards(Oid relationId, Oid colocatedTableId, /* * This path is only reached by create_distributed_table for the distributed * tables which will not be part of an existing colocation group. Therefore, - * we can directly use ShardCount and ShardReplicationFactor global variables - * here. + * we can directly use ShardReplicationFactor global variable here. */ - CreateShardsWithRoundRobinPolicy(relationId, ShardCount, ShardReplicationFactor, + CreateShardsWithRoundRobinPolicy(relationId, shardCount, ShardReplicationFactor, useExclusiveConnection); } } @@ -568,7 +543,8 @@ CreateHashDistributedTableShards(Oid relationId, Oid colocatedTableId, static uint32 ColocationIdForNewTable(Oid relationId, Var *distributionColumn, char distributionMethod, char replicationModel, - char *colocateWithTableName, bool viaDeprecatedAPI) + int shardCount, char *colocateWithTableName, + bool viaDeprecatedAPI) { uint32 colocationId = INVALID_COLOCATION_ID; @@ -611,13 +587,13 @@ ColocationIdForNewTable(Oid relationId, Var *distributionColumn, if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) == 0) { /* check for default colocation group */ - colocationId = ColocationId(ShardCount, ShardReplicationFactor, + colocationId = ColocationId(shardCount, ShardReplicationFactor, distributionColumnType, distributionColumnCollation); if (colocationId == INVALID_COLOCATION_ID) { - colocationId = CreateColocationGroup(ShardCount, ShardReplicationFactor, + colocationId = CreateColocationGroup(shardCount, ShardReplicationFactor, distributionColumnType, distributionColumnCollation); createdColocationGroup = true; @@ -1543,295 +1519,3 @@ DistributionColumnUsesGeneratedStoredColumn(TupleDesc relationDesc, return false; } - - -/* - * UndistributeTable undistributes the given table. The undistribution is done by - * creating a new table, moving everything to the new table and dropping the old one. - * So the oid of the table is not preserved. - * - * The undistributed table will have the same name, columns and rows. It will also have - * partitions, views, sequences of the old table. Finally it will have everything created - * by GetPostLoadTableCreationCommands function, which include indexes. These will be - * re-created during undistribution, so their oids are not preserved either (except for - * sequences). However, their names are preserved. - * - * The tables with references are not supported. The function gives an error if there are - * any references to or from the table. - * - * The dropping of old table is done with CASCADE. Anything not mentioned here will - * be dropped. - */ -void -UndistributeTable(Oid relationId, bool cascadeViaForeignKeys) -{ - EnsureCoordinator(); - EnsureRelationExists(relationId); - EnsureTableOwner(relationId); - - LOCKMODE lockMode = ExclusiveLock; - Relation relation = try_relation_open(relationId, lockMode); - if (relation == NULL) - { - ereport(ERROR, (errmsg("cannot undistribute table"), - errdetail("because no such distributed table exists"))); - } - - relation_close(relation, NoLock); - - if (!IsCitusTable(relationId)) - { - ereport(ERROR, (errmsg("cannot undistribute table "), - errdetail("because the table is not distributed"))); - } - - bool tableReferencing = TableReferencing(relationId); - bool tableReferenced = TableReferenced(relationId); - if (cascadeViaForeignKeys && (tableReferencing || tableReferenced)) - { - CascadeOperationForConnectedRelations(relationId, lockMode, UNDISTRIBUTE_TABLE); - - /* - * Undistributed every foreign key connected relation in our foreign key - * subgraph including itself, so return here. - */ - return; - } - - if (tableReferencing) - { - char *qualifiedRelationName = generate_qualified_relation_name(relationId); - ereport(ERROR, (errmsg("cannot undistribute table " - "because it has a foreign key"), - errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, - qualifiedRelationName, - qualifiedRelationName))); - } - - if (tableReferenced) - { - char *qualifiedRelationName = generate_qualified_relation_name(relationId); - ereport(ERROR, (errmsg("cannot undistribute table " - "because a foreign key references to it"), - errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, - qualifiedRelationName, - qualifiedRelationName))); - } - - char relationKind = get_rel_relkind(relationId); - if (relationKind == RELKIND_FOREIGN_TABLE) - { - ereport(ERROR, (errmsg("cannot undistribute table " - "because it is a foreign table"))); - } - - if (PartitionTable(relationId)) - { - Oid parentRelationId = PartitionParentOid(relationId); - char *parentRelationName = get_rel_name(parentRelationId); - ereport(ERROR, (errmsg("cannot undistribute table " - "because it is a partition"), - errhint("undistribute the partitioned table \"%s\" instead", - parentRelationName))); - } - - List *preLoadCommands = GetPreLoadTableCreationCommands(relationId, true); - List *postLoadCommands = GetPostLoadTableCreationCommands(relationId); - - postLoadCommands = list_concat(postLoadCommands, - GetViewCreationCommandsOfTable(relationId)); - - int spiResult = SPI_connect(); - if (spiResult != SPI_OK_CONNECT) - { - ereport(ERROR, (errmsg("could not connect to SPI manager"))); - } - - char *relationName = get_rel_name(relationId); - Oid schemaId = get_rel_namespace(relationId); - char *schemaName = get_namespace_name(schemaId); - - if (PartitionedTable(relationId)) - { - ereport(NOTICE, (errmsg("undistributing the partitions of %s", - quote_qualified_identifier(schemaName, relationName)))); - List *partitionList = PartitionList(relationId); - - /* - * This is a less common pattern where foreing key is directly from/to - * the partition relation as we already handled inherited foreign keys - * on partitions either by erroring out or cascading via foreign keys. - * It seems an acceptable limitation for now to ask users to drop such - * foreign keys manually. - */ - ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(partitionList); - - Oid partitionRelationId = InvalidOid; - foreach_oid(partitionRelationId, partitionList) - { - char *detachPartitionCommand = GenerateDetachPartitionCommand( - partitionRelationId); - char *attachPartitionCommand = GenerateAlterTableAttachPartitionCommand( - partitionRelationId); - - /* - * We first detach the partitions to be able to undistribute them separately. - */ - spiResult = SPI_execute(detachPartitionCommand, false, 0); - if (spiResult != SPI_OK_UTILITY) - { - ereport(ERROR, (errmsg("could not run SPI query"))); - } - preLoadCommands = lappend(preLoadCommands, - makeTableDDLCommandString(attachPartitionCommand)); - - /* - * Even if we called UndistributeTable with cascade option, we - * shouldn't cascade via foreign keys on partitions. Otherwise, - * we might try to undistribute partitions of other tables in - * our foreign key subgraph more than once. - */ - bool cascadeOnPartitionsViaForeignKeys = false; - UndistributeTable(partitionRelationId, cascadeOnPartitionsViaForeignKeys); - } - } - - char *tempName = pstrdup(relationName); - uint32 hashOfName = hash_any((unsigned char *) tempName, strlen(tempName)); - AppendShardIdToName(&tempName, hashOfName); - - - ereport(NOTICE, (errmsg("creating a new local table for %s", - quote_qualified_identifier(schemaName, relationName)))); - - TableDDLCommand *tableCreationCommand = NULL; - foreach_ptr(tableCreationCommand, preLoadCommands) - { - Assert(CitusIsA(tableCreationCommand, TableDDLCommand)); - - char *tableCreationSql = GetTableDDLCommand(tableCreationCommand); - Node *parseTree = ParseTreeNode(tableCreationSql); - - RelayEventExtendNames(parseTree, schemaName, hashOfName); - CitusProcessUtility(parseTree, tableCreationSql, PROCESS_UTILITY_TOPLEVEL, - NULL, None_Receiver, NULL); - } - - ReplaceTable(relationId, get_relname_relid(tempName, schemaId)); - - TableDDLCommand *tableConstructionCommand = NULL; - foreach_ptr(tableConstructionCommand, postLoadCommands) - { - Assert(CitusIsA(tableConstructionCommand, TableDDLCommand)); - char *tableConstructionSQL = GetTableDDLCommand(tableConstructionCommand); - spiResult = SPI_execute(tableConstructionSQL, false, 0); - if (spiResult != SPI_OK_UTILITY) - { - ereport(ERROR, (errmsg("could not run SPI query"))); - } - } - - spiResult = SPI_finish(); - if (spiResult != SPI_OK_FINISH) - { - ereport(ERROR, (errmsg("could not finish SPI connection"))); - } -} - - -/* - * GetViewCreationCommandsOfTable takes a table oid generates the CREATE VIEW - * commands for views that depend to the given table. This includes the views - * that recursively depend on the table too. - */ -List * -GetViewCreationCommandsOfTable(Oid relationId) -{ - List *views = GetDependingViews(relationId); - List *commands = NIL; - - Oid viewOid = InvalidOid; - foreach_oid(viewOid, views) - { - Datum viewDefinitionDatum = DirectFunctionCall1(pg_get_viewdef, - ObjectIdGetDatum(viewOid)); - char *viewDefinition = TextDatumGetCString(viewDefinitionDatum); - StringInfo query = makeStringInfo(); - char *viewName = get_rel_name(viewOid); - char *schemaName = get_namespace_name(get_rel_namespace(viewOid)); - char *qualifiedViewName = quote_qualified_identifier(schemaName, viewName); - appendStringInfo(query, - "CREATE VIEW %s AS %s", - qualifiedViewName, - viewDefinition); - commands = lappend(commands, makeTableDDLCommandString(query->data)); - } - return commands; -} - - -/* - * ReplaceTable replaces the source table with the target table. - * It moves all the rows of the source table to target table with INSERT SELECT. - * Changes the dependencies of the sequences owned by source table to target table. - * Then drops the source table and renames the target table to source tables name. - * - * Source and target tables need to be in the same schema and have the same columns. - */ -void -ReplaceTable(Oid sourceId, Oid targetId) -{ - char *sourceName = get_rel_name(sourceId); - char *targetName = get_rel_name(targetId); - Oid schemaId = get_rel_namespace(sourceId); - char *schemaName = get_namespace_name(schemaId); - - StringInfo query = makeStringInfo(); - - ereport(NOTICE, (errmsg("Moving the data of %s", - quote_qualified_identifier(schemaName, sourceName)))); - - appendStringInfo(query, "INSERT INTO %s SELECT * FROM %s", - quote_qualified_identifier(schemaName, targetName), - quote_qualified_identifier(schemaName, sourceName)); - int spiResult = SPI_execute(query->data, false, 0); - if (spiResult != SPI_OK_INSERT) - { - ereport(ERROR, (errmsg("could not run SPI query"))); - } - -#if PG_VERSION_NUM >= PG_VERSION_13 - List *ownedSequences = getOwnedSequences(sourceId); -#else - List *ownedSequences = getOwnedSequences(sourceId, InvalidAttrNumber); -#endif - Oid sequenceOid = InvalidOid; - foreach_oid(sequenceOid, ownedSequences) - { - changeDependencyFor(RelationRelationId, sequenceOid, - RelationRelationId, sourceId, targetId); - } - - ereport(NOTICE, (errmsg("Dropping the old %s", - quote_qualified_identifier(schemaName, sourceName)))); - - resetStringInfo(query); - appendStringInfo(query, "DROP TABLE %s CASCADE", - quote_qualified_identifier(schemaName, sourceName)); - spiResult = SPI_execute(query->data, false, 0); - if (spiResult != SPI_OK_UTILITY) - { - ereport(ERROR, (errmsg("could not run SPI query"))); - } - - ereport(NOTICE, (errmsg("Renaming the new table to %s", - quote_qualified_identifier(schemaName, sourceName)))); - -#if PG_VERSION_NUM >= PG_VERSION_12 - RenameRelationInternal(targetId, - sourceName, false, false); -#else - RenameRelationInternal(targetId, - sourceName, false); -#endif -} diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 8d03ff747..2ae1be6bd 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -616,6 +616,48 @@ GetReferencingForeignConstaintCommands(Oid relationId) } +/* + * GetForeignConstraintToReferenceTablesCommands takes in a relationId, and + * returns the list of foreign constraint commands needed to reconstruct + * foreign key constraints that the table is involved in as the "referencing" + * one and the "referenced" table is a reference table. + */ +List * +GetForeignConstraintToReferenceTablesCommands(Oid relationId) +{ + int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_REFERENCE_TABLES; + return GetForeignConstraintCommandsInternal(relationId, flags); +} + + +/* + * GetForeignConstraintToDistributedTablesCommands takes in a relationId, and + * returns the list of foreign constraint commands needed to reconstruct + * foreign key constraints that the table is involved in as the "referencing" + * one and the "referenced" table is a distributed table. + */ +List * +GetForeignConstraintToDistributedTablesCommands(Oid relationId) +{ + int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; + return GetForeignConstraintCommandsInternal(relationId, flags); +} + + +/* + * GetForeignConstraintFromDistributedTablesCommands takes in a relationId, and + * returns the list of foreign constraint commands needed to reconstruct + * foreign key constraints that the table is involved in as the "referenced" + * one and the "referencing" table is a distributed table. + */ +List * +GetForeignConstraintFromDistributedTablesCommands(Oid relationId) +{ + int flags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; + return GetForeignConstraintCommandsInternal(relationId, flags); +} + + /* * GetForeignConstraintCommandsInternal is a wrapper function to get the * DDL commands to recreate the foreign key constraints returned by diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 6cc142f19..77baade6d 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -225,8 +225,8 @@ PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const bool viaDeprecatedAPI = false; CreateDistributedTable(relationId, parentDistributionColumn, - parentDistributionMethod, parentRelationName, - viaDeprecatedAPI); + parentDistributionMethod, ShardCount, + parentRelationName, viaDeprecatedAPI); } } @@ -299,8 +299,8 @@ PostprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, bool viaDeprecatedAPI = false; CreateDistributedTable(partitionRelationId, distributionColumn, - distributionMethod, parentRelationName, - viaDeprecatedAPI); + distributionMethod, ShardCount, + parentRelationName, viaDeprecatedAPI); } } } diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 63d320f16..c2f924b12 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -248,7 +248,8 @@ pg_get_sequencedef(Oid sequenceRelationId) * DEFAULT clauses for columns getting their default values from a sequence. */ char * -pg_get_tableschemadef_string(Oid tableRelationId, bool includeSequenceDefaults) +pg_get_tableschemadef_string(Oid tableRelationId, bool includeSequenceDefaults, + char *accessMethod) { char relationKind = 0; bool firstAttributePrinted = false; @@ -457,7 +458,11 @@ pg_get_tableschemadef_string(Oid tableRelationId, bool includeSequenceDefaults) * Add table access methods for pg12 and higher when the table is configured with an * access method */ - if (OidIsValid(relation->rd_rel->relam)) + if (accessMethod) + { + appendStringInfo(&buffer, " USING %s", quote_identifier(accessMethod)); + } + else if (OidIsValid(relation->rd_rel->relam)) { HeapTuple amTup = SearchSysCache1(AMOID, ObjectIdGetDatum( relation->rd_rel->relam)); diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index 217f272e0..ed71c7a6c 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -540,12 +540,12 @@ GetFullTableCreationCommands(Oid relationId, bool includeSequenceDefaults) List *tableDDLEventList = NIL; List *preLoadCreationCommandList = - GetPreLoadTableCreationCommands(relationId, includeSequenceDefaults); + GetPreLoadTableCreationCommands(relationId, includeSequenceDefaults, NULL); tableDDLEventList = list_concat(tableDDLEventList, preLoadCreationCommandList); List *postLoadCreationCommandList = - GetPostLoadTableCreationCommands(relationId); + GetPostLoadTableCreationCommands(relationId, true); tableDDLEventList = list_concat(tableDDLEventList, postLoadCreationCommandList); @@ -558,12 +558,16 @@ GetFullTableCreationCommands(Oid relationId, bool includeSequenceDefaults) * of DDL commands that should be applied after loading the data. */ List * -GetPostLoadTableCreationCommands(Oid relationId) +GetPostLoadTableCreationCommands(Oid relationId, bool includeIndexes) { List *tableDDLEventList = NIL; - List *indexAndConstraintCommandList = GetTableIndexAndConstraintCommands(relationId); - tableDDLEventList = list_concat(tableDDLEventList, indexAndConstraintCommandList); + if (includeIndexes) + { + List *indexAndConstraintCommandList = + GetTableIndexAndConstraintCommands(relationId); + tableDDLEventList = list_concat(tableDDLEventList, indexAndConstraintCommandList); + } List *replicaIdentityEvents = GetTableReplicaIdentityCommand(relationId); tableDDLEventList = list_concat(tableDDLEventList, replicaIdentityEvents); @@ -616,7 +620,8 @@ GetTableReplicaIdentityCommand(Oid relationId) * to facilitate faster data load. */ List * -GetPreLoadTableCreationCommands(Oid relationId, bool includeSequenceDefaults) +GetPreLoadTableCreationCommands(Oid relationId, bool includeSequenceDefaults, + char *accessMethod) { List *tableDDLEventList = NIL; @@ -640,7 +645,8 @@ GetPreLoadTableCreationCommands(Oid relationId, bool includeSequenceDefaults) /* fetch table schema and column option definitions */ char *tableSchemaDef = pg_get_tableschemadef_string(relationId, - includeSequenceDefaults); + includeSequenceDefaults, + accessMethod); char *tableColumnOptionsDef = pg_get_tablecolumnoptionsdef_string(relationId); tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString( @@ -654,7 +660,7 @@ GetPreLoadTableCreationCommands(Oid relationId, bool includeSequenceDefaults) #if PG_VERSION_NUM >= 120000 /* add columnar options for cstore tables */ - if (IsCStoreTableAmTable(relationId)) + if (accessMethod == NULL && IsCStoreTableAmTable(relationId)) { TableDDLCommand *cstoreOptionsDDL = ColumnarGetTableOptionsDDL(relationId); if (cstoreOptionsDDL != NULL) diff --git a/src/backend/distributed/operations/repair_shards.c b/src/backend/distributed/operations/repair_shards.c index 38ed0be0f..9046a5824 100644 --- a/src/backend/distributed/operations/repair_shards.c +++ b/src/backend/distributed/operations/repair_shards.c @@ -937,7 +937,7 @@ CopyShardCommandList(ShardInterval *shardInterval, const char *sourceNodeName, copyShardDataCommand->data); } - List *indexCommandList = GetPostLoadTableCreationCommands(relationId); + List *indexCommandList = GetPostLoadTableCreationCommands(relationId, true); indexCommandList = WorkerApplyShardDDLCommandList(indexCommandList, shardId); copyShardToNodeCommandsList = list_concat(copyShardToNodeCommandsList, @@ -1143,7 +1143,8 @@ RecreateTableDDLCommandList(Oid relationId) List *dropCommandList = list_make1(makeTableDDLCommandString(dropCommand->data)); List *createCommandList = GetPreLoadTableCreationCommands(relationId, - includeSequenceDefaults); + includeSequenceDefaults, + NULL); List *recreateCommandList = list_concat(dropCommandList, createCommandList); return recreateCommandList; diff --git a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql index 5ef39e451..bf8c385aa 100644 --- a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql +++ b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql @@ -6,6 +6,8 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass); #include "udfs/citus_total_relation_size/10.0-1.sql" #include "udfs/citus_tables/10.0-1.sql" #include "udfs/citus_finish_pg_upgrade/10.0-1.sql" +#include "udfs/alter_distributed_table/10.0-1.sql" +#include "udfs/alter_table_set_access_method/10.0-1.sql" #include "udfs/undistribute_table/10.0-1.sql" #include "udfs/create_citus_local_table/10.0-1.sql" #include "udfs/citus_set_coordinator_host/10.0-1.sql" @@ -24,6 +26,7 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass); #include "udfs/citus_copy_shard_placement/10.0-1.sql" #include "udfs/citus_move_shard_placement/10.0-1.sql" #include "udfs/citus_drop_trigger/10.0-1.sql" +#include "udfs/worker_change_sequence_dependency/10.0-1.sql" #include "../../columnar/sql/columnar--9.5-1--10.0-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql b/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql index 7fae38c89..28ea42d65 100644 --- a/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql @@ -6,6 +6,8 @@ #include "../../../columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql" DROP VIEW public.citus_tables; +DROP FUNCTION pg_catalog.alter_distributed_table(regclass, text, int, text, boolean); +DROP FUNCTION pg_catalog.alter_table_set_access_method(regclass, text); DROP FUNCTION pg_catalog.citus_total_relation_size(regclass,boolean); DROP FUNCTION pg_catalog.undistribute_table(regclass,boolean); DROP FUNCTION pg_catalog.create_citus_local_table(regclass,boolean); @@ -45,6 +47,7 @@ DROP VIEW pg_catalog.time_partitions; DROP FUNCTION pg_catalog.time_partition_range(regclass); DROP FUNCTION pg_catalog.citus_set_coordinator_host(text,int,noderole,name); +DROP FUNCTION pg_catalog.worker_change_sequence_dependency(regclass, regclass, regclass); CREATE FUNCTION pg_catalog.master_modify_multiple_shards(text) RETURNS integer diff --git a/src/backend/distributed/sql/udfs/alter_distributed_table/10.0-1.sql b/src/backend/distributed/sql/udfs/alter_distributed_table/10.0-1.sql new file mode 100644 index 000000000..c19e43c39 --- /dev/null +++ b/src/backend/distributed/sql/udfs/alter_distributed_table/10.0-1.sql @@ -0,0 +1,9 @@ +CREATE OR REPLACE FUNCTION pg_catalog.alter_distributed_table( + table_name regclass, distribution_column text DEFAULT NULL, shard_count int DEFAULT NULL, colocate_with text DEFAULT NULL, cascade_to_colocated boolean DEFAULT NULL) + RETURNS VOID + LANGUAGE C +AS 'MODULE_PATHNAME', $$alter_distributed_table$$; + +COMMENT ON FUNCTION pg_catalog.alter_distributed_table( + table_name regclass, distribution_column text, shard_count int, colocate_with text, cascade_to_colocated boolean) + IS 'alters a distributed table'; diff --git a/src/backend/distributed/sql/udfs/alter_distributed_table/latest.sql b/src/backend/distributed/sql/udfs/alter_distributed_table/latest.sql new file mode 100644 index 000000000..c19e43c39 --- /dev/null +++ b/src/backend/distributed/sql/udfs/alter_distributed_table/latest.sql @@ -0,0 +1,9 @@ +CREATE OR REPLACE FUNCTION pg_catalog.alter_distributed_table( + table_name regclass, distribution_column text DEFAULT NULL, shard_count int DEFAULT NULL, colocate_with text DEFAULT NULL, cascade_to_colocated boolean DEFAULT NULL) + RETURNS VOID + LANGUAGE C +AS 'MODULE_PATHNAME', $$alter_distributed_table$$; + +COMMENT ON FUNCTION pg_catalog.alter_distributed_table( + table_name regclass, distribution_column text, shard_count int, colocate_with text, cascade_to_colocated boolean) + IS 'alters a distributed table'; diff --git a/src/backend/distributed/sql/udfs/alter_table_set_access_method/10.0-1.sql b/src/backend/distributed/sql/udfs/alter_table_set_access_method/10.0-1.sql new file mode 100644 index 000000000..81fcf1b59 --- /dev/null +++ b/src/backend/distributed/sql/udfs/alter_table_set_access_method/10.0-1.sql @@ -0,0 +1,9 @@ +CREATE OR REPLACE FUNCTION pg_catalog.alter_table_set_access_method( + table_name regclass, access_method text) + RETURNS VOID + LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$alter_table_set_access_method$$; + +COMMENT ON FUNCTION pg_catalog.alter_table_set_access_method( + table_name regclass, access_method text) + IS 'alters a table''s access method'; diff --git a/src/backend/distributed/sql/udfs/alter_table_set_access_method/latest.sql b/src/backend/distributed/sql/udfs/alter_table_set_access_method/latest.sql new file mode 100644 index 000000000..81fcf1b59 --- /dev/null +++ b/src/backend/distributed/sql/udfs/alter_table_set_access_method/latest.sql @@ -0,0 +1,9 @@ +CREATE OR REPLACE FUNCTION pg_catalog.alter_table_set_access_method( + table_name regclass, access_method text) + RETURNS VOID + LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$alter_table_set_access_method$$; + +COMMENT ON FUNCTION pg_catalog.alter_table_set_access_method( + table_name regclass, access_method text) + IS 'alters a table''s access method'; diff --git a/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/10.0-1.sql b/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/10.0-1.sql new file mode 100644 index 000000000..d895abb73 --- /dev/null +++ b/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/10.0-1.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION pg_catalog.worker_change_sequence_dependency( + sequence regclass, + source_table regclass, + target_table regclass) + RETURNS VOID + LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$worker_change_sequence_dependency$$; + +COMMENT ON FUNCTION pg_catalog.worker_change_sequence_dependency( + sequence regclass, + source_table regclass, + target_table regclass) + IS 'changes sequence''s dependency from source table to target table'; diff --git a/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/latest.sql b/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/latest.sql new file mode 100644 index 000000000..d895abb73 --- /dev/null +++ b/src/backend/distributed/sql/udfs/worker_change_sequence_dependency/latest.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION pg_catalog.worker_change_sequence_dependency( + sequence regclass, + source_table regclass, + target_table regclass) + RETURNS VOID + LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$worker_change_sequence_dependency$$; + +COMMENT ON FUNCTION pg_catalog.worker_change_sequence_dependency( + sequence regclass, + source_table regclass, + target_table regclass) + IS 'changes sequence''s dependency from source table to target table'; diff --git a/src/include/columnar/cstore.h b/src/include/columnar/cstore.h index d1ee08cb4..6548887a0 100644 --- a/src/include/columnar/cstore.h +++ b/src/include/columnar/cstore.h @@ -79,6 +79,18 @@ typedef struct ColumnarOptions } ColumnarOptions; +/* + * ColumnarTableDDLContext holds the instance variable for the TableDDLCommandFunction + * instance described below. + */ +typedef struct ColumnarTableDDLContext +{ + char *schemaName; + char *relationName; + ColumnarOptions options; +} ColumnarTableDDLContext; + + /* * StripeMetadata represents information about a stripe. This information is * stored in the cstore file's footer. diff --git a/src/include/columnar/cstore_tableam.h b/src/include/columnar/cstore_tableam.h index aa320a864..71f223678 100644 --- a/src/include/columnar/cstore_tableam.h +++ b/src/include/columnar/cstore_tableam.h @@ -21,4 +21,5 @@ extern TableScanDesc cstore_beginscan_extended(Relation relation, Snapshot snaps extern bool IsCStoreTableAmTable(Oid relationId); extern TableDDLCommand * ColumnarGetTableOptionsDDL(Oid relationId); +extern char * GetShardedTableDDLCommandColumnar(uint64 shardId, void *context); #endif diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index fd3b046bf..27cfb0a4b 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -30,7 +30,8 @@ extern Oid get_extension_schema(Oid ext_oid); extern char * pg_get_serverdef_string(Oid tableRelationId); extern char * pg_get_sequencedef_string(Oid sequenceRelid); extern Form_pg_sequence pg_get_sequencedef(Oid sequenceRelationId); -extern char * pg_get_tableschemadef_string(Oid tableRelationId, bool forShardCreation); +extern char * pg_get_tableschemadef_string(Oid tableRelationId, bool forShardCreation, + char *accessMethod); extern void EnsureRelationKindSupported(Oid relationId); extern char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); extern void deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index ec29fcccc..baa40224a 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -163,6 +163,9 @@ extern bool ColumnReferencedByAnyForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId); extern List * GetReferencingForeignConstaintCommands(Oid relationOid); +extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId); +extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId); +extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId); extern bool HasForeignKeyToCitusLocalTable(Oid relationId); extern bool HasForeignKeyToReferenceTable(Oid relationOid); extern bool TableReferenced(Oid relationOid); @@ -399,10 +402,10 @@ typedef enum CascadeOperationType INVALID_OPERATION = 1 << 0, /* execute UndistributeTable on each relation */ - UNDISTRIBUTE_TABLE = 1 << 1, + CASCADE_FKEY_UNDISTRIBUTE_TABLE = 1 << 1, /* execute CreateCitusLocalTable on each relation */ - CREATE_CITUS_LOCAL_TABLE = 1 << 2, + CASCADE_FKEY_CREATE_CITUS_LOCAL_TABLE = 1 << 2, } CascadeOperationType; extern void CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE relLockMode, diff --git a/src/include/distributed/coordinator_protocol.h b/src/include/distributed/coordinator_protocol.h index 2cea9eb0c..4c30b2b04 100644 --- a/src/include/distributed/coordinator_protocol.h +++ b/src/include/distributed/coordinator_protocol.h @@ -177,9 +177,10 @@ extern uint64 GetNextShardId(void); extern uint64 GetNextPlacementId(void); extern Oid ResolveRelationId(text *relationName, bool missingOk); extern List * GetFullTableCreationCommands(Oid relationId, bool includeSequenceDefaults); -extern List * GetPostLoadTableCreationCommands(Oid relationId); +extern List * GetPostLoadTableCreationCommands(Oid relationId, bool includeIndexes); extern List * GetPreLoadTableCreationCommands(Oid relationId, - bool includeSequenceDefaults); + bool includeSequenceDefaults, + char *accessMethod); extern List * GetTableIndexAndConstraintCommands(Oid relationId); extern bool IndexImpliedByAConstraint(Form_pg_index indexForm); extern char ShardStorageType(Oid relationId); diff --git a/src/include/distributed/metadata_utility.h b/src/include/distributed/metadata_utility.h index 26c77bd62..4d41a7149 100644 --- a/src/include/distributed/metadata_utility.h +++ b/src/include/distributed/metadata_utility.h @@ -90,6 +90,74 @@ typedef struct ShardPlacement } ShardPlacement; +typedef enum CascadeToColocatedOption +{ + CASCADE_TO_COLOCATED_UNSPECIFIED, + CASCADE_TO_COLOCATED_YES, + CASCADE_TO_COLOCATED_NO, + CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED +}CascadeToColocatedOption; + +/* + * TableConversionParameters are the parameters that are given to + * table conversion UDFs: undistribute_table, alter_distributed_table, + * alter_table_set_access_method. + * + * When passing a TableConversionParameters object to one of the table + * conversion functions some of the parameters needs to be set: + * UndistributeTable: relationId + * AlterDistributedTable: relationId, distributionColumn, shardCountIsNull, + * shardCount, colocateWith, cascadeToColocated + * AlterTableSetAccessMethod: relationId, accessMethod + * + * conversionType parameter will be automatically set by the function. + * + * TableConversionState objects can be created using TableConversionParameters + * objects with CreateTableConversion function. + */ +typedef struct TableConversionParameters +{ + /* + * Determines type of conversion: UNDISTRIBUTE_TABLE, + * ALTER_DISTRIBUTED_TABLE, ALTER_TABLE_SET_ACCESS_METHOD. + */ + char conversionType; + + /* Oid of the table to do conversion on */ + Oid relationId; + + /* + * Options to do conversions on the table + * distributionColumn is the name of the new distribution column, + * shardCountIsNull is if the shardCount variable is not given + * shardCount is the new shard count, + * colocateWith is the name of the table to colocate with, 'none', or + * 'default' + * accessMethod is the name of the new accessMethod for the table + */ + char *distributionColumn; + bool shardCountIsNull; + int shardCount; + char *colocateWith; + char *accessMethod; + + /* + * cascadeToColocated determines whether the shardCount and + * colocateWith will be cascaded to the currently colocated tables + */ + CascadeToColocatedOption cascadeToColocated; + + /* + * cascadeViaForeignKeys determines if the conversion operation + * will be cascaded to the graph connected with foreign keys + * to the table + */ + bool cascadeViaForeignKeys; +} TableConversionParameters; + +typedef struct TableConversionReturn TableConversionReturn; + + /* Config variable managed via guc.c */ extern int ReplicationModel; @@ -138,10 +206,10 @@ extern void MarkShardPlacementInactive(ShardPlacement *shardPlacement); extern void UpdateShardPlacementState(uint64 placementId, char shardState); extern void DeleteShardPlacementRow(uint64 placementId); extern void CreateDistributedTable(Oid relationId, Var *distributionColumn, - char distributionMethod, char *colocateWithTableName, - bool viaDeprecatedAPI); + char distributionMethod, int shardCount, + char *colocateWithTableName, bool viaDeprecatedAPI); extern void CreateTruncateTrigger(Oid relationId); -extern void UndistributeTable(Oid relationId, bool cascadeViaForeignKeys); +extern TableConversionReturn * UndistributeTable(TableConversionParameters *params); extern void EnsureDependenciesExistOnAllNodes(const ObjectAddress *target); extern List * GetDistributableDependenciesForObject(const ObjectAddress *target); @@ -160,6 +228,7 @@ extern void EnsureSuperUser(void); extern void ErrorIfTableIsACatalogTable(Relation relation); extern void EnsureTableNotDistributed(Oid relationId); extern void EnsureReplicationSettings(Oid relationId, char replicationModel); +extern void EnsureRelationExists(Oid relationId); extern bool RegularTable(Oid relationId); extern char * ConstructQualifiedShardName(ShardInterval *shardInterval); extern uint64 GetFirstShardId(Oid relationId); diff --git a/src/test/regress/expected/alter_distributed_table.out b/src/test/regress/expected/alter_distributed_table.out new file mode 100644 index 000000000..969abec42 --- /dev/null +++ b/src/test/regress/expected/alter_distributed_table.out @@ -0,0 +1,812 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven; + server_version_above_eleven +--------------------------------------------------------------------- + t +(1 row) + +\gset +CREATE SCHEMA alter_distributed_table; +SET search_path TO alter_distributed_table; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +CREATE TABLE dist_table (a INT, b INT); +SELECT create_distributed_table ('dist_table', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO dist_table VALUES (1, 1), (2, 2), (3, 3); +CREATE TABLE colocation_table (a INT, b INT); +SELECT create_distributed_table ('colocation_table', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE colocation_table_2 (a INT, b INT); +SELECT create_distributed_table ('colocation_table_2', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | a | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering distribution column +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering shard count +SELECT alter_distributed_table('dist_table', shard_count := 6); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering colocation, note that shard count will also change +SELECT alter_distributed_table('dist_table', colocate_with := 'alter_distributed_table.colocation_table'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table, dist_table + colocation_table_2 +(2 rows) + +-- test altering shard count with cascading, note that the colocation will be kept +SELECT alter_distributed_table('dist_table', shard_count := 8, cascade_to_colocated := true); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table +NOTICE: creating a new table for alter_distributed_table.colocation_table +NOTICE: Moving the data of alter_distributed_table.colocation_table +NOTICE: Dropping the old alter_distributed_table.colocation_table +NOTICE: Renaming the new table to alter_distributed_table.colocation_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 8 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 8 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table, dist_table + colocation_table_2 +(2 rows) + +-- test altering shard count without cascading, note that the colocation will be broken +SELECT alter_distributed_table('dist_table', shard_count := 10, cascade_to_colocated := false); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 8 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 10 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test partitions +CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); +SELECT create_distributed_table('partitioned_table', 'id', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioned_table_1_5 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (5); +CREATE TABLE partitioned_table_6_10 PARTITION OF partitioned_table FOR VALUES FROM (6) TO (10); +INSERT INTO partitioned_table VALUES (2, 12), (7, 2); +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,6) + (localhost,57638,t,6) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Distribution Column | Shard Count +--------------------------------------------------------------------- + partitioned_table | id | 4 + partitioned_table_1_5 | id | 4 + partitioned_table_6_10 | id | 4 +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- test altering the parent table +SELECT alter_distributed_table('partitioned_table', shard_count := 10, distribution_column := 'a'); +NOTICE: converting the partitions of alter_distributed_table.partitioned_table +NOTICE: creating a new table for alter_distributed_table.partitioned_table_1_5 +NOTICE: Moving the data of alter_distributed_table.partitioned_table_1_5 +NOTICE: Dropping the old alter_distributed_table.partitioned_table_1_5 +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table_1_5 +NOTICE: creating a new table for alter_distributed_table.partitioned_table_6_10 +NOTICE: Moving the data of alter_distributed_table.partitioned_table_6_10 +NOTICE: Dropping the old alter_distributed_table.partitioned_table_6_10 +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table_6_10 +NOTICE: creating a new table for alter_distributed_table.partitioned_table +NOTICE: Dropping the old alter_distributed_table.partitioned_table +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test altering the partition +SELECT alter_distributed_table('partitioned_table_1_5', shard_count := 10, distribution_column := 'a'); +ERROR: cannot complete operation because table is a partition +HINT: the parent table is "partitioned_table" +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,15) + (localhost,57638,t,15) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Distribution Column | Shard Count +--------------------------------------------------------------------- + partitioned_table | a | 10 + partitioned_table_1_5 | a | 10 + partitioned_table_6_10 | a | 10 +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- test references +CREATE TABLE referenced_dist_table (a INT UNIQUE); +CREATE TABLE referenced_ref_table (a INT UNIQUE); +CREATE TABLE table_with_references (a1 INT UNIQUE REFERENCES referenced_dist_table(a), a2 INT REFERENCES referenced_ref_table(a)); +CREATE TABLE referencing_dist_table (a INT REFERENCES table_with_references(a1)); +SELECT create_distributed_table('referenced_dist_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('referenced_ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('table_with_references', 'a1', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('referencing_dist_table', 'a', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO WARNING; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) + table_with_references | FOREIGN KEY (a1) REFERENCES referenced_dist_table(a) + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) +(3 rows) + +SELECT alter_distributed_table('table_with_references', shard_count := 12, cascade_to_colocated := true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) + table_with_references | FOREIGN KEY (a1) REFERENCES referenced_dist_table(a) +(3 rows) + +SELECT alter_distributed_table('table_with_references', shard_count := 10, cascade_to_colocated := false); +WARNING: foreign key table_with_references_a1_fkey will be dropped +WARNING: foreign key referencing_dist_table_a_fkey will be dropped + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) +(1 row) + +-- check when multi shard modify mode is set to sequential +SELECT alter_distributed_table('referenced_dist_table', colocate_with:='none'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE ref_to_dist_table(a INT REFERENCES referenced_dist_table(a)); +CREATE TABLE ref_to_ref_table(a INT REFERENCES referenced_ref_table(a)); +SELECT create_distributed_table('ref_to_dist_table', 'a', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('ref_to_ref_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table referencing a reference table +SELECT alter_distributed_table('ref_to_ref_table', shard_count:=6); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- let's create a table that is not colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_dist (a INT); +SELECT create_distributed_table('col_with_ref_to_dist', 'a', colocate_with:='ref_to_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- and create a table colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_ref (a INT); +SELECT alter_distributed_table('ref_to_ref_table', colocate_with:='none'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('col_with_ref_to_ref', 'a', colocate_with:='ref_to_ref_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=8, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table colocated with a table referencing a reference table without cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=10, cascade_to_colocated:=false); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table not colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_dist', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\if :server_version_above_eleven +-- test altering columnar table +CREATE TABLE columnar_table (a INT) USING columnar; +SELECT create_distributed_table('columnar_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; + Name | Shard Count | Access Method +--------------------------------------------------------------------- + columnar_table | 4 | columnar +(1 row) + +SELECT alter_distributed_table('columnar_table', shard_count:=6); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; + Name | Shard Count | Access Method +--------------------------------------------------------------------- + columnar_table | 6 | columnar +(1 row) + +\endif +-- test with metadata sync +SET citus.replication_model TO 'streaming'; +SELECT start_metadata_sync_to_node('localhost', :worker_1_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE metadata_sync_table (a BIGSERIAL); +SELECT create_distributed_table('metadata_sync_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('metadata_sync_table', shard_count:=6); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('metadata_sync_table', shard_count:=8); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'metadata_sync_table'; + Name | Shard Count +--------------------------------------------------------------------- + metadata_sync_table | 8 +(1 row) + +SET citus.replication_model TO DEFAULT; +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +-- test complex cascade operations +CREATE TABLE cas_1 (a INT UNIQUE); +CREATE TABLE cas_2 (a INT UNIQUE); +CREATE TABLE cas_3 (a INT UNIQUE); +CREATE TABLE cas_4 (a INT UNIQUE); +CREATE TABLE cas_par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par_1 PARTITION OF cas_par FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par_2 PARTITION OF cas_par FOR VALUES FROM (5) TO (8); +CREATE TABLE cas_col (a INT UNIQUE); +-- add foreign keys from and to partitions +ALTER TABLE cas_par_1 ADD CONSTRAINT fkey_from_par_1 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_2 ADD CONSTRAINT fkey_to_par_1 FOREIGN KEY (a) REFERENCES cas_par_1(a); +ALTER TABLE cas_par ADD CONSTRAINT fkey_from_par FOREIGN KEY (a) REFERENCES cas_3(a); +ALTER TABLE cas_4 ADD CONSTRAINT fkey_to_par FOREIGN KEY (a) REFERENCES cas_par(a); +-- distribute all the tables +SELECT create_distributed_table('cas_1', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_3', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_par', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_2', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_4', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_col', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + cas_2 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_4 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_3(a) + cas_par_1 | UNIQUE (a) +(5 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +-- alter the cas_col and cascade the change +SELECT alter_distributed_table('cas_col', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + cas_2 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_4 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_3(a) + cas_par_1 | UNIQUE (a) +(5 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +SET client_min_messages TO DEFAULT; +-- test changing dist column and colocating partitioned table without changing shard count +CREATE TABLE col_table (a INT); +SELECT create_distributed_table('col_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE par_table (a BIGINT, b INT) PARTITION BY RANGE (a); +SELECT create_distributed_table('par_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE par_table_1 (a BIGINT, b INT); +SELECT create_distributed_table('par_table_1', 'a', colocate_with:='par_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE par_table ATTACH PARTITION par_table_1 FOR VALUES FROM (1) TO (5); +SELECT alter_distributed_table('par_table', distribution_column:='b', colocate_with:='col_table'); +NOTICE: converting the partitions of alter_distributed_table.par_table +NOTICE: creating a new table for alter_distributed_table.par_table_1 +NOTICE: Moving the data of alter_distributed_table.par_table_1 +NOTICE: Dropping the old alter_distributed_table.par_table_1 +NOTICE: Renaming the new table to alter_distributed_table.par_table_1 +NOTICE: creating a new table for alter_distributed_table.par_table +NOTICE: Dropping the old alter_distributed_table.par_table +NOTICE: Renaming the new table to alter_distributed_table.par_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test messages +-- test nothing to change +SELECT alter_distributed_table('dist_table'); +ERROR: you have to specify at least one of the distribution_column, shard_count or colocate_with parameters +SELECT alter_distributed_table('dist_table', cascade_to_colocated := false); +ERROR: you have to specify at least one of the distribution_column, shard_count or colocate_with parameters +-- no operation UDF calls +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +SELECT alter_distributed_table('dist_table', shard_count := 10); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +-- first colocate the tables, then try to re-colococate +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +-- test some changes while keeping others same +-- shouldn't error but should have notices about no-change parameters +SELECT alter_distributed_table('dist_table', distribution_column:='b', shard_count:=4, cascade_to_colocated:=false); +NOTICE: table is already distributed by b +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', shard_count:=4, colocate_with:='colocation_table_2'); +NOTICE: shard count of the table is already 4 +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with:='colocation_table_2', distribution_column:='a'); +NOTICE: table is already colocated with colocation_table_2 +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test cascading distribution column, should error +SELECT alter_distributed_table('dist_table', distribution_column := 'b', cascade_to_colocated := true); +ERROR: distribution_column cannot be cascaded to colocated tables +SELECT alter_distributed_table('dist_table', distribution_column := 'b', shard_count:=12, colocate_with:='colocation_table_2', cascade_to_colocated := true); +ERROR: distribution_column cannot be cascaded to colocated tables +-- test nothing to cascade +SELECT alter_distributed_table('dist_table', cascade_to_colocated := true); +ERROR: shard_count or colocate_with is necessary for cascading to colocated tables +-- test cascading colocate_with := 'none' +SELECT alter_distributed_table('dist_table', colocate_with := 'none', cascade_to_colocated := true); +ERROR: colocate_with := 'none' cannot be cascaded to colocated tables +-- test changing shard count of a colocated table without cascade_to_colocated, should error +SELECT alter_distributed_table('dist_table', shard_count := 14); +ERROR: cascade_to_colocated parameter is necessary +DETAIL: this table is colocated with some other tables +HINT: cascade_to_colocated := false will break the current colocation, cascade_to_colocated := true will change the shard count of colocated tables too. +-- test changing shard count of a non-colocated table without cascade_to_colocated, shouldn't error +SELECT alter_distributed_table('dist_table', colocate_with := 'none'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', shard_count := 14); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test altering a table into colocating with a table but giving a different shard count +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table', shard_count := 16); +ERROR: shard_count cannot be different than the shard count of the table in colocate_with +HINT: if no shard_count is specified shard count will be same with colocate_with table's +-- test colocation with distribution columns with different data types +CREATE TABLE different_type_table (a TEXT); +SELECT create_distributed_table('different_type_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with := 'different_type_table'); +ERROR: cannot colocate with different_type_table because data type of its distribution column is different than dist_table +SELECT alter_distributed_table('dist_table', distribution_column := 'a', colocate_with := 'different_type_table'); +ERROR: cannot colocate with different_type_table and change distribution column to a because data type of column a is different then the distribution column of the different_type_table +-- test shard_count := 0 +SELECT alter_distributed_table('dist_table', shard_count := 0); +ERROR: shard_count cannot be 0 +HINT: if you no longer want this to be a distributed table you can try undistribute_table() function +-- test colocating with non-distributed table +CREATE TABLE reference_table (a INT); +SELECT create_reference_table('reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with:='reference_table'); +ERROR: cannot colocate with reference_table because it is not a distributed table +-- test append table +CREATE TABLE append_table (a INT); +SELECT create_distributed_table('append_table', 'a', 'append'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('append_table', shard_count:=6); +ERROR: relation append_table should be a hash distributed table +SET client_min_messages TO WARNING; +DROP SCHEMA alter_distributed_table CASCADE; diff --git a/src/test/regress/expected/alter_distributed_table_0.out b/src/test/regress/expected/alter_distributed_table_0.out new file mode 100644 index 000000000..3a3453ac0 --- /dev/null +++ b/src/test/regress/expected/alter_distributed_table_0.out @@ -0,0 +1,791 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven; + server_version_above_eleven +--------------------------------------------------------------------- + f +(1 row) + +\gset +CREATE SCHEMA alter_distributed_table; +SET search_path TO alter_distributed_table; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +CREATE TABLE dist_table (a INT, b INT); +SELECT create_distributed_table ('dist_table', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO dist_table VALUES (1, 1), (2, 2), (3, 3); +CREATE TABLE colocation_table (a INT, b INT); +SELECT create_distributed_table ('colocation_table', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE colocation_table_2 (a INT, b INT); +SELECT create_distributed_table ('colocation_table_2', 'a', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | a | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering distribution column +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering shard count +SELECT alter_distributed_table('dist_table', shard_count := 6); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test altering colocation, note that shard count will also change +SELECT alter_distributed_table('dist_table', colocate_with := 'alter_distributed_table.colocation_table'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 4 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table, dist_table + colocation_table_2 +(2 rows) + +-- test altering shard count with cascading, note that the colocation will be kept +SELECT alter_distributed_table('dist_table', shard_count := 8, cascade_to_colocated := true); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table +NOTICE: creating a new table for alter_distributed_table.colocation_table +NOTICE: Moving the data of alter_distributed_table.colocation_table +NOTICE: Dropping the old alter_distributed_table.colocation_table +NOTICE: Renaming the new table to alter_distributed_table.colocation_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 8 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 8 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table, dist_table + colocation_table_2 +(2 rows) + +-- test altering shard count without cascading, note that the colocation will be broken +SELECT alter_distributed_table('dist_table', shard_count := 10, cascade_to_colocated := false); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + colocation_table | distributed | a | 8 + colocation_table_2 | distributed | a | 4 + dist_table | distributed | b | 10 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + colocation_table + colocation_table_2 + dist_table +(3 rows) + +-- test partitions +CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); +SELECT create_distributed_table('partitioned_table', 'id', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioned_table_1_5 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (5); +CREATE TABLE partitioned_table_6_10 PARTITION OF partitioned_table FOR VALUES FROM (6) TO (10); +INSERT INTO partitioned_table VALUES (2, 12), (7, 2); +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,6) + (localhost,57638,t,6) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Distribution Column | Shard Count +--------------------------------------------------------------------- + partitioned_table | id | 4 + partitioned_table_1_5 | id | 4 + partitioned_table_6_10 | id | 4 +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- test altering the parent table +SELECT alter_distributed_table('partitioned_table', shard_count := 10, distribution_column := 'a'); +NOTICE: converting the partitions of alter_distributed_table.partitioned_table +NOTICE: creating a new table for alter_distributed_table.partitioned_table_1_5 +NOTICE: Moving the data of alter_distributed_table.partitioned_table_1_5 +NOTICE: Dropping the old alter_distributed_table.partitioned_table_1_5 +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table_1_5 +NOTICE: creating a new table for alter_distributed_table.partitioned_table_6_10 +NOTICE: Moving the data of alter_distributed_table.partitioned_table_6_10 +NOTICE: Dropping the old alter_distributed_table.partitioned_table_6_10 +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table_6_10 +NOTICE: creating a new table for alter_distributed_table.partitioned_table +NOTICE: Dropping the old alter_distributed_table.partitioned_table +NOTICE: Renaming the new table to alter_distributed_table.partitioned_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test altering the partition +SELECT alter_distributed_table('partitioned_table_1_5', shard_count := 10, distribution_column := 'a'); +ERROR: cannot complete operation because table is a partition +HINT: the parent table is "partitioned_table" +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,15) + (localhost,57638,t,15) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Distribution Column | Shard Count +--------------------------------------------------------------------- + partitioned_table | a | 10 + partitioned_table_1_5 | a | 10 + partitioned_table_6_10 | a | 10 +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- test references +CREATE TABLE referenced_dist_table (a INT UNIQUE); +CREATE TABLE referenced_ref_table (a INT UNIQUE); +CREATE TABLE table_with_references (a1 INT UNIQUE REFERENCES referenced_dist_table(a), a2 INT REFERENCES referenced_ref_table(a)); +CREATE TABLE referencing_dist_table (a INT REFERENCES table_with_references(a1)); +SELECT create_distributed_table('referenced_dist_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_reference_table('referenced_ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('table_with_references', 'a1', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('referencing_dist_table', 'a', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO WARNING; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) + table_with_references | FOREIGN KEY (a1) REFERENCES referenced_dist_table(a) + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) +(3 rows) + +SELECT alter_distributed_table('table_with_references', shard_count := 12, cascade_to_colocated := true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) + table_with_references | FOREIGN KEY (a1) REFERENCES referenced_dist_table(a) +(3 rows) + +SELECT alter_distributed_table('table_with_references', shard_count := 10, cascade_to_colocated := false); +WARNING: foreign key table_with_references_a1_fkey will be dropped +WARNING: foreign key referencing_dist_table_a_fkey will be dropped + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) +(1 row) + +-- check when multi shard modify mode is set to sequential +SELECT alter_distributed_table('referenced_dist_table', colocate_with:='none'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE ref_to_dist_table(a INT REFERENCES referenced_dist_table(a)); +CREATE TABLE ref_to_ref_table(a INT REFERENCES referenced_ref_table(a)); +SELECT create_distributed_table('ref_to_dist_table', 'a', colocate_with:='referenced_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('ref_to_ref_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table referencing a reference table +SELECT alter_distributed_table('ref_to_ref_table', shard_count:=6); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- let's create a table that is not colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_dist (a INT); +SELECT create_distributed_table('col_with_ref_to_dist', 'a', colocate_with:='ref_to_dist_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- and create a table colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_ref (a INT); +SELECT alter_distributed_table('ref_to_ref_table', colocate_with:='none'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('col_with_ref_to_ref', 'a', colocate_with:='ref_to_ref_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=8, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table colocated with a table referencing a reference table without cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=10, cascade_to_colocated:=false); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- alter a table not colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_dist', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\if :server_version_above_eleven +-- test altering columnar table +CREATE TABLE columnar_table (a INT) USING columnar; +SELECT create_distributed_table('columnar_table', 'a', colocate_with:='none'); +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; +SELECT alter_distributed_table('columnar_table', shard_count:=6); +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; +\endif +-- test with metadata sync +SET citus.replication_model TO 'streaming'; +SELECT start_metadata_sync_to_node('localhost', :worker_1_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE metadata_sync_table (a BIGSERIAL); +SELECT create_distributed_table('metadata_sync_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('metadata_sync_table', shard_count:=6); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('metadata_sync_table', shard_count:=8); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'metadata_sync_table'; + Name | Shard Count +--------------------------------------------------------------------- + metadata_sync_table | 8 +(1 row) + +SET citus.replication_model TO DEFAULT; +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +-- test complex cascade operations +CREATE TABLE cas_1 (a INT UNIQUE); +CREATE TABLE cas_2 (a INT UNIQUE); +CREATE TABLE cas_3 (a INT UNIQUE); +CREATE TABLE cas_4 (a INT UNIQUE); +CREATE TABLE cas_par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par_1 PARTITION OF cas_par FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par_2 PARTITION OF cas_par FOR VALUES FROM (5) TO (8); +CREATE TABLE cas_col (a INT UNIQUE); +-- add foreign keys from and to partitions +ALTER TABLE cas_par_1 ADD CONSTRAINT fkey_from_par_1 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_2 ADD CONSTRAINT fkey_to_par_1 FOREIGN KEY (a) REFERENCES cas_par_1(a); +ALTER TABLE cas_par ADD CONSTRAINT fkey_from_par FOREIGN KEY (a) REFERENCES cas_3(a); +ALTER TABLE cas_4 ADD CONSTRAINT fkey_to_par FOREIGN KEY (a) REFERENCES cas_par(a); +ERROR: cannot reference partitioned table "cas_par" +-- distribute all the tables +SELECT create_distributed_table('cas_1', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_3', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_par', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_2', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_4', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('cas_col', 'a', colocate_with:='cas_1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + cas_2 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_3(a) + cas_par_1 | UNIQUE (a) +(4 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +-- alter the cas_col and cascade the change +SELECT alter_distributed_table('cas_col', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + cas_2 | FOREIGN KEY (a) REFERENCES cas_par_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_1(a) + cas_par_1 | FOREIGN KEY (a) REFERENCES cas_3(a) + cas_par_1 | UNIQUE (a) +(4 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + cas_par_1 + cas_par_2 +(2 rows) + +SET client_min_messages TO DEFAULT; +-- test changing dist column and colocating partitioned table without changing shard count +CREATE TABLE col_table (a INT); +SELECT create_distributed_table('col_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE par_table (a BIGINT, b INT) PARTITION BY RANGE (a); +SELECT create_distributed_table('par_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE par_table_1 (a BIGINT, b INT); +SELECT create_distributed_table('par_table_1', 'a', colocate_with:='par_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE par_table ATTACH PARTITION par_table_1 FOR VALUES FROM (1) TO (5); +SELECT alter_distributed_table('par_table', distribution_column:='b', colocate_with:='col_table'); +NOTICE: converting the partitions of alter_distributed_table.par_table +NOTICE: creating a new table for alter_distributed_table.par_table_1 +NOTICE: Moving the data of alter_distributed_table.par_table_1 +NOTICE: Dropping the old alter_distributed_table.par_table_1 +NOTICE: Renaming the new table to alter_distributed_table.par_table_1 +NOTICE: creating a new table for alter_distributed_table.par_table +NOTICE: Dropping the old alter_distributed_table.par_table +NOTICE: Renaming the new table to alter_distributed_table.par_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test messages +-- test nothing to change +SELECT alter_distributed_table('dist_table'); +ERROR: you have to specify at least one of the distribution_column, shard_count or colocate_with parameters +SELECT alter_distributed_table('dist_table', cascade_to_colocated := false); +ERROR: you have to specify at least one of the distribution_column, shard_count or colocate_with parameters +-- no operation UDF calls +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +SELECT alter_distributed_table('dist_table', shard_count := 10); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +-- first colocate the tables, then try to re-colococate +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); +ERROR: this call doesn't change any properties of the table +HINT: check citus_tables view to see current properties of the table +-- test some changes while keeping others same +-- shouldn't error but should have notices about no-change parameters +SELECT alter_distributed_table('dist_table', distribution_column:='b', shard_count:=4, cascade_to_colocated:=false); +NOTICE: table is already distributed by b +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', shard_count:=4, colocate_with:='colocation_table_2'); +NOTICE: shard count of the table is already 4 +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with:='colocation_table_2', distribution_column:='a'); +NOTICE: table is already colocated with colocation_table_2 +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test cascading distribution column, should error +SELECT alter_distributed_table('dist_table', distribution_column := 'b', cascade_to_colocated := true); +ERROR: distribution_column cannot be cascaded to colocated tables +SELECT alter_distributed_table('dist_table', distribution_column := 'b', shard_count:=12, colocate_with:='colocation_table_2', cascade_to_colocated := true); +ERROR: distribution_column cannot be cascaded to colocated tables +-- test nothing to cascade +SELECT alter_distributed_table('dist_table', cascade_to_colocated := true); +ERROR: shard_count or colocate_with is necessary for cascading to colocated tables +-- test cascading colocate_with := 'none' +SELECT alter_distributed_table('dist_table', colocate_with := 'none', cascade_to_colocated := true); +ERROR: colocate_with := 'none' cannot be cascaded to colocated tables +-- test changing shard count of a colocated table without cascade_to_colocated, should error +SELECT alter_distributed_table('dist_table', shard_count := 14); +ERROR: cascade_to_colocated parameter is necessary +DETAIL: this table is colocated with some other tables +HINT: cascade_to_colocated := false will break the current colocation, cascade_to_colocated := true will change the shard count of colocated tables too. +-- test changing shard count of a non-colocated table without cascade_to_colocated, shouldn't error +SELECT alter_distributed_table('dist_table', colocate_with := 'none'); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', shard_count := 14); +NOTICE: creating a new table for alter_distributed_table.dist_table +NOTICE: Moving the data of alter_distributed_table.dist_table +NOTICE: Dropping the old alter_distributed_table.dist_table +NOTICE: Renaming the new table to alter_distributed_table.dist_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test altering a table into colocating with a table but giving a different shard count +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table', shard_count := 16); +ERROR: shard_count cannot be different than the shard count of the table in colocate_with +HINT: if no shard_count is specified shard count will be same with colocate_with table's +-- test colocation with distribution columns with different data types +CREATE TABLE different_type_table (a TEXT); +SELECT create_distributed_table('different_type_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with := 'different_type_table'); +ERROR: cannot colocate with different_type_table because data type of its distribution column is different than dist_table +SELECT alter_distributed_table('dist_table', distribution_column := 'a', colocate_with := 'different_type_table'); +ERROR: cannot colocate with different_type_table and change distribution column to a because data type of column a is different then the distribution column of the different_type_table +-- test shard_count := 0 +SELECT alter_distributed_table('dist_table', shard_count := 0); +ERROR: shard_count cannot be 0 +HINT: if you no longer want this to be a distributed table you can try undistribute_table() function +-- test colocating with non-distributed table +CREATE TABLE reference_table (a INT); +SELECT create_reference_table('reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('dist_table', colocate_with:='reference_table'); +ERROR: cannot colocate with reference_table because it is not a distributed table +-- test append table +CREATE TABLE append_table (a INT); +SELECT create_distributed_table('append_table', 'a', 'append'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_distributed_table('append_table', shard_count:=6); +ERROR: relation append_table should be a hash distributed table +SET client_min_messages TO WARNING; +DROP SCHEMA alter_distributed_table CASCADE; diff --git a/src/test/regress/expected/alter_table_set_access_method.out b/src/test/regress/expected/alter_table_set_access_method.out new file mode 100644 index 000000000..990c94cc4 --- /dev/null +++ b/src/test/regress/expected/alter_table_set_access_method.out @@ -0,0 +1,394 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif +CREATE SCHEMA alter_table_set_access_method; +SET search_path TO alter_table_set_access_method; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +SELECT public.run_command_on_coordinator_and_workers($Q$ + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; +$Q$); + run_command_on_coordinator_and_workers +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE dist_table (a INT, b INT); +SELECT create_distributed_table ('dist_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO dist_table VALUES (1, 1), (2, 2), (3, 3); +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'dist_table' ORDER BY 1; + Name | Access Method +--------------------------------------------------------------------- + dist_table | heap +(1 row) + +SELECT alter_table_set_access_method('dist_table', 'columnar'); +NOTICE: any index will be dropped, because columnar tables cannot have indexes +NOTICE: creating a new table for alter_table_set_access_method.dist_table +NOTICE: Moving the data of alter_table_set_access_method.dist_table +NOTICE: Dropping the old alter_table_set_access_method.dist_table +NOTICE: Renaming the new table to alter_table_set_access_method.dist_table + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'dist_table' ORDER BY 1; + Name | Access Method +--------------------------------------------------------------------- + dist_table | columnar +(1 row) + +-- test partitions +CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); +CREATE TABLE partitioned_table_1_5 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (5); +CREATE TABLE partitioned_table_6_10 PARTITION OF partitioned_table FOR VALUES FROM (6) TO (10); +SELECT create_distributed_table('partitioned_table', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO partitioned_table VALUES (2, 12), (7, 2); +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,6) + (localhost,57638,t,6) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Access Method +--------------------------------------------------------------------- + partitioned_table | + partitioned_table_1_5 | heap + partitioned_table_6_10 | heap +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- altering partitioned tables' access methods is not supported +SELECT alter_table_set_access_method('partitioned_table', 'columnar'); +ERROR: you cannot alter access method of a partitioned table +-- test altering the partition's access method +SELECT alter_table_set_access_method('partitioned_table_1_5', 'columnar'); +NOTICE: any index will be dropped, because columnar tables cannot have indexes +NOTICE: creating a new table for alter_table_set_access_method.partitioned_table_1_5 +NOTICE: Moving the data of alter_table_set_access_method.partitioned_table_1_5 +NOTICE: Dropping the old alter_table_set_access_method.partitioned_table_1_5 +NOTICE: Renaming the new table to alter_table_set_access_method.partitioned_table_1_5 + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioned_table + partitioned_table_1_5 + partitioned_table_6_10 +(3 rows) + +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,6) + (localhost,57638,t,6) +(2 rows) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name"::text, "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; + Name | Access Method +--------------------------------------------------------------------- + partitioned_table | + partitioned_table_1_5 | columnar + partitioned_table_6_10 | heap +(3 rows) + +SELECT * FROM partitioned_table ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 + 7 | 2 +(2 rows) + +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 2 | 12 +(1 row) + +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + id | a +--------------------------------------------------------------------- + 7 | 2 +(1 row) + +-- test altering a table with index to columnar +-- the index will be dropped +CREATE TABLE index_table (a INT) ; +CREATE INDEX idx1 ON index_table (a); +SELECT indexname FROM pg_indexes WHERE schemaname = 'alter_table_set_access_method' AND tablename = 'index_table'; + indexname +--------------------------------------------------------------------- + idx1 +(1 row) + +SELECT a.amname FROM pg_class c, pg_am a where c.relname = 'index_table' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + amname +--------------------------------------------------------------------- + heap +(1 row) + +SELECT alter_table_set_access_method('index_table', 'columnar'); +NOTICE: any index will be dropped, because columnar tables cannot have indexes +NOTICE: creating a new table for alter_table_set_access_method.index_table +NOTICE: Moving the data of alter_table_set_access_method.index_table +NOTICE: Dropping the old alter_table_set_access_method.index_table +NOTICE: Renaming the new table to alter_table_set_access_method.index_table + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT indexname FROM pg_indexes WHERE schemaname = 'alter_table_set_access_method' AND tablename = 'index_table'; + indexname +--------------------------------------------------------------------- +(0 rows) + +SELECT a.amname FROM pg_class c, pg_am a where c.relname = 'index_table' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + amname +--------------------------------------------------------------------- + columnar +(1 row) + +-- test different table types +SET client_min_messages to WARNING; +SELECT 1 FROM master_add_node('localhost', :master_port, groupId := 0); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SET client_min_messages to DEFAULT; +CREATE TABLE table_type_dist (a INT); +SELECT create_distributed_table('table_type_dist', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE table_type_ref (a INT); +SELECT create_reference_table('table_type_ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE table_type_citus_local(a INT); +SELECT create_citus_local_table('table_type_citus_local'); + create_citus_local_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE table_type_pg_local (a INT); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'table\_type%' ORDER BY 1; + Name | Citus Table Type | Distribution Column | Shard Count | Access Method +--------------------------------------------------------------------- + table_type_dist | distributed | a | 4 | heap + table_type_ref | reference | | 1 | heap +(2 rows) + +SELECT c.relname, a.amname FROM pg_class c, pg_am a where c.relname SIMILAR TO 'table_type\D*' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + relname | amname +--------------------------------------------------------------------- + table_type_citus_local | heap + table_type_dist | heap + table_type_pg_local | heap + table_type_ref | heap +(4 rows) + +SELECT alter_table_set_access_method('table_type_dist', 'fake_am'); +NOTICE: creating a new table for alter_table_set_access_method.table_type_dist +WARNING: fake_scan_getnextslot +CONTEXT: SQL statement "SELECT EXISTS (SELECT 1 FROM alter_table_set_access_method.table_type_dist_1533505599)" +WARNING: fake_scan_getnextslot +NOTICE: Moving the data of alter_table_set_access_method.table_type_dist +NOTICE: Dropping the old alter_table_set_access_method.table_type_dist +NOTICE: Renaming the new table to alter_table_set_access_method.table_type_dist + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_table_set_access_method('table_type_ref', 'fake_am'); +NOTICE: creating a new table for alter_table_set_access_method.table_type_ref +WARNING: fake_scan_getnextslot +CONTEXT: SQL statement "SELECT EXISTS (SELECT 1 FROM alter_table_set_access_method.table_type_ref_1037855087)" +WARNING: fake_scan_getnextslot +NOTICE: Moving the data of alter_table_set_access_method.table_type_ref +NOTICE: Dropping the old alter_table_set_access_method.table_type_ref +NOTICE: Renaming the new table to alter_table_set_access_method.table_type_ref + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_table_set_access_method('table_type_pg_local', 'fake_am'); +NOTICE: creating a new table for alter_table_set_access_method.table_type_pg_local +NOTICE: Moving the data of alter_table_set_access_method.table_type_pg_local +NOTICE: Dropping the old alter_table_set_access_method.table_type_pg_local +NOTICE: Renaming the new table to alter_table_set_access_method.table_type_pg_local + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_table_set_access_method('table_type_citus_local', 'fake_am'); +NOTICE: creating a new table for alter_table_set_access_method.table_type_citus_local +NOTICE: Moving the data of alter_table_set_access_method.table_type_citus_local +NOTICE: Dropping the old alter_table_set_access_method.table_type_citus_local +NOTICE: Renaming the new table to alter_table_set_access_method.table_type_citus_local + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'table\_type%' ORDER BY 1; + Name | Citus Table Type | Distribution Column | Shard Count | Access Method +--------------------------------------------------------------------- + table_type_dist | distributed | a | 4 | fake_am + table_type_ref | reference | | 1 | fake_am +(2 rows) + +SELECT c.relname, a.amname FROM pg_class c, pg_am a where c.relname SIMILAR TO 'table_type\D*' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + relname | amname +--------------------------------------------------------------------- + table_type_citus_local | fake_am + table_type_dist | fake_am + table_type_pg_local | fake_am + table_type_ref | fake_am +(4 rows) + +-- test when the parent of a partition has foreign key to a reference table +CREATE TABLE ref_table (a INT UNIQUE); +SELECT create_reference_table('ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO ref_table VALUES (2), (12); +ALTER TABLE partitioned_table ADD CONSTRAINT fkey_to_ref FOREIGN KEY (a) REFERENCES ref_table(a); +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'partitioned_table_6_10'; + Name | Access Method +--------------------------------------------------------------------- + partitioned_table_6_10 | heap +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'partitioned_table_6_10') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + partitioned_table_6_10 | FOREIGN KEY (a) REFERENCES ref_table(a) +(1 row) + +SELECT alter_table_set_access_method('partitioned_table_6_10', 'columnar'); +NOTICE: any index will be dropped, because columnar tables cannot have indexes +NOTICE: creating a new table for alter_table_set_access_method.partitioned_table_6_10 +NOTICE: Moving the data of alter_table_set_access_method.partitioned_table_6_10 +NOTICE: Dropping the old alter_table_set_access_method.partitioned_table_6_10 +NOTICE: Renaming the new table to alter_table_set_access_method.partitioned_table_6_10 + alter_table_set_access_method +--------------------------------------------------------------------- + +(1 row) + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; + inhrelid +--------------------------------------------------------------------- + partitioned_table_1_5 + partitioned_table_6_10 +(2 rows) + +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'partitioned_table_6_10'; + Name | Access Method +--------------------------------------------------------------------- + partitioned_table_6_10 | columnar +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'partitioned_table_6_10') ORDER BY 1, 2; + Referencing Table | Definition +--------------------------------------------------------------------- + partitioned_table_6_10 | FOREIGN KEY (a) REFERENCES ref_table(a) +(1 row) + +SET client_min_messages TO WARNING; +DROP SCHEMA alter_table_set_access_method CASCADE; +SELECT 1 FROM master_remove_node('localhost', :master_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + diff --git a/src/test/regress/expected/alter_table_set_access_method_0.out b/src/test/regress/expected/alter_table_set_access_method_0.out new file mode 100644 index 000000000..9d92533f9 --- /dev/null +++ b/src/test/regress/expected/alter_table_set_access_method_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q diff --git a/src/test/regress/expected/citus_local_tables.out b/src/test/regress/expected/citus_local_tables.out index 4597c75de..287cb1a92 100644 --- a/src/test/regress/expected/citus_local_tables.out +++ b/src/test/regress/expected/citus_local_tables.out @@ -330,7 +330,7 @@ ERROR: Table 'citus_local_table_1' is a citus local table. Replicating shard of -- undistribute_table is supported BEGIN; SELECT undistribute_table('citus_local_table_1'); -NOTICE: creating a new local table for citus_local_tables_test_schema.citus_local_table_1 +NOTICE: creating a new table for citus_local_tables_test_schema.citus_local_table_1 NOTICE: Moving the data of citus_local_tables_test_schema.citus_local_table_1 NOTICE: executing the command locally: SELECT a FROM citus_local_tables_test_schema.citus_local_table_1_1504027 citus_local_table_1 NOTICE: Dropping the old citus_local_tables_test_schema.citus_local_table_1 diff --git a/src/test/regress/expected/coordinator_shouldhaveshards.out b/src/test/regress/expected/coordinator_shouldhaveshards.out index 00de7eea8..90b96bba2 100644 --- a/src/test/regress/expected/coordinator_shouldhaveshards.out +++ b/src/test/regress/expected/coordinator_shouldhaveshards.out @@ -618,6 +618,166 @@ SELECT 1 AS created WHERE EXISTS(SELECT * FROM pg_indexes WHERE indexname LIKE ' 1 (1 row) +-- test alter_distributed_table UDF +SET citus.shard_count TO 4; +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_table_1503060 AS citus_table_alias (a, b) VALUES (1,2), (5,6) +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_col_1503064 AS citus_table_alias (a, b) VALUES (5,6) +INSERT INTO adt_ref VALUES (3), (5); +NOTICE: executing the command locally: INSERT INTO coordinator_shouldhaveshards.adt_ref_1503068 AS citus_table_alias (a) VALUES (5) +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 4 + adt_ref | distributed | a | 4 + adt_table | distributed | a | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SET client_min_messages TO WARNING; +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO DEFAULT; +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | a | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); +NOTICE: creating a new table for coordinator_shouldhaveshards.adt_table +NOTICE: Moving the data of coordinator_shouldhaveshards.adt_table +NOTICE: Dropping the old coordinator_shouldhaveshards.adt_table +NOTICE: Renaming the new table to coordinator_shouldhaveshards.adt_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | b | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref + adt_table +(2 rows) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT * FROM adt_table ORDER BY 1; + a | b +--------------------------------------------------------------------- + 1 | 2 + 3 | 4 + 5 | 6 +(3 rows) + +SELECT * FROM adt_col ORDER BY 1; + a | b +--------------------------------------------------------------------- + 3 | 4 + 5 | 6 + 7 | 8 +(3 rows) + +SELECT * FROM adt_ref ORDER BY 1; + a +--------------------------------------------------------------------- + 3 + 5 +(2 rows) + +SET client_min_messages TO WARNING; +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT COUNT(*) FROM adt_table; + count +--------------------------------------------------------------------- + 1003 +(1 row) + +END; +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_table | distributed | a | 6 +(1 row) + +SET client_min_messages TO DEFAULT; \set VERBOSITY terse DROP TABLE ref_table; NOTICE: executing the command locally: DROP TABLE IF EXISTS coordinator_shouldhaveshards.ref_table_xxxxx CASCADE @@ -628,7 +788,7 @@ DROP TABLE ref; NOTICE: executing the command locally: DROP TABLE IF EXISTS coordinator_shouldhaveshards.ref_xxxxx CASCADE DROP TABLE test_append_table; DROP SCHEMA coordinator_shouldhaveshards CASCADE; -NOTICE: drop cascades to 4 other objects +NOTICE: drop cascades to 13 other objects SELECT 1 FROM master_set_node_property('localhost', :master_port, 'shouldhaveshards', false); ?column? --------------------------------------------------------------------- diff --git a/src/test/regress/expected/isolation_undistribute_table.out b/src/test/regress/expected/isolation_undistribute_table.out index 7d4d11a4b..9e5508a19 100644 --- a/src/test/regress/expected/isolation_undistribute_table.out +++ b/src/test/regress/expected/isolation_undistribute_table.out @@ -17,7 +17,7 @@ step s1-commit: COMMIT; step s2-undistribute: <... completed> -error in steps s1-commit s2-undistribute: ERROR: cannot undistribute table +error in steps s1-commit s2-undistribute: ERROR: cannot complete operation because no such table exists starting permutation: s1-begin s1-undistribute s2-select s1-commit step s1-begin: diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 269267fd1..d01ef799f 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -463,6 +463,8 @@ SELECT * FROM print_extension_changes(); | access method columnar | function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) | function alter_columnar_table_set(regclass,integer,integer,name,integer) + | function alter_distributed_table(regclass,text,integer,text,boolean) + | function alter_table_set_access_method(regclass,text) | function citus_activate_node(text,integer) | function citus_add_inactive_node(text,integer,integer,noderole,name) | function citus_add_node(text,integer,integer,noderole,name) @@ -493,6 +495,7 @@ SELECT * FROM print_extension_changes(); | function create_citus_local_table(regclass,boolean) | function time_partition_range(regclass) | function undistribute_table(regclass,boolean) + | function worker_change_sequence_dependency(regclass,regclass,regclass) | schema columnar | sequence columnar.storageid_seq | table columnar.columnar_skipnodes @@ -501,7 +504,7 @@ SELECT * FROM print_extension_changes(); | view citus_shards | view citus_tables | view time_partitions -(56 rows) +(59 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_extension_0.out b/src/test/regress/expected/multi_extension_0.out index c7cfcfe55..986ee0ae3 100644 --- a/src/test/regress/expected/multi_extension_0.out +++ b/src/test/regress/expected/multi_extension_0.out @@ -460,6 +460,8 @@ SELECT * FROM print_extension_changes(); function master_modify_multiple_shards(text) | function undistribute_table(regclass) | function upgrade_to_reference_table(regclass) | + | function alter_distributed_table(regclass,text,integer,text,boolean) + | function alter_table_set_access_method(regclass,text) | function citus_activate_node(text,integer) | function citus_add_inactive_node(text,integer,integer,noderole,name) | function citus_add_node(text,integer,integer,noderole,name) @@ -489,6 +491,7 @@ SELECT * FROM print_extension_changes(); | function create_citus_local_table(regclass,boolean) | function time_partition_range(regclass) | function undistribute_table(regclass,boolean) + | function worker_change_sequence_dependency(regclass,regclass,regclass) | schema columnar | sequence columnar.storageid_seq | table columnar.columnar_skipnodes @@ -497,7 +500,7 @@ SELECT * FROM print_extension_changes(); | view citus_shards | view citus_tables | view time_partitions -(52 rows) +(55 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_mx_alter_distributed_table.out b/src/test/regress/expected/multi_mx_alter_distributed_table.out new file mode 100644 index 000000000..b56c477b9 --- /dev/null +++ b/src/test/regress/expected/multi_mx_alter_distributed_table.out @@ -0,0 +1,163 @@ +CREATE SCHEMA mx_alter_distributed_table; +SET search_path TO mx_alter_distributed_table; +SET citus.shard_replication_factor TO 1; +-- test alter_distributed_table UDF +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +INSERT INTO adt_ref VALUES (3), (5); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 4 + adt_ref | distributed | a | 4 + adt_table | distributed | a | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SET client_min_messages TO WARNING; +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO DEFAULT; +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | a | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); +NOTICE: creating a new table for mx_alter_distributed_table.adt_table +NOTICE: Moving the data of mx_alter_distributed_table.adt_table +NOTICE: Dropping the old mx_alter_distributed_table.adt_table +NOTICE: Renaming the new table to mx_alter_distributed_table.adt_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | b | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref + adt_table +(2 rows) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT * FROM adt_table ORDER BY 1; + a | b +--------------------------------------------------------------------- + 1 | 2 + 3 | 4 + 5 | 6 +(3 rows) + +SELECT * FROM adt_col ORDER BY 1; + a | b +--------------------------------------------------------------------- + 3 | 4 + 5 | 6 + 7 | 8 +(3 rows) + +SELECT * FROM adt_ref ORDER BY 1; + a +--------------------------------------------------------------------- + 3 + 5 +(2 rows) + +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); +NOTICE: creating a new table for mx_alter_distributed_table.adt_table +NOTICE: Moving the data of mx_alter_distributed_table.adt_table +NOTICE: Dropping the old mx_alter_distributed_table.adt_table +NOTICE: Renaming the new table to mx_alter_distributed_table.adt_table + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT COUNT(*) FROM adt_table; + count +--------------------------------------------------------------------- + 1003 +(1 row) + +END; +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_table | distributed | a | 6 +(1 row) + +SET client_min_messages TO WARNING; +DROP SCHEMA mx_alter_distributed_table CASCADE; diff --git a/src/test/regress/expected/pg13_with_ties.out b/src/test/regress/expected/pg13_with_ties.out index 51f2471ef..38b692fc6 100644 --- a/src/test/regress/expected/pg13_with_ties.out +++ b/src/test/regress/expected/pg13_with_ties.out @@ -154,7 +154,7 @@ SELECT * FROM with_ties_table_2 ORDER BY a, b; TRUNCATE with_ties_table_2; -- test INSERT SELECTs into distributed table with a different distribution column SELECT undistribute_table('with_ties_table_2'); -NOTICE: creating a new local table for public.with_ties_table_2 +NOTICE: creating a new table for public.with_ties_table_2 NOTICE: Moving the data of public.with_ties_table_2 NOTICE: Dropping the old public.with_ties_table_2 NOTICE: Renaming the new table to public.with_ties_table_2 diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index 84c4d6f6d..c6e7f6965 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -280,7 +280,7 @@ INSERT INTO upsert_test (part_key, other_col) SELECT part_key, other_col FROM up COMMIT; -- to test citus local tables select undistribute_table('upsert_test'); -NOTICE: creating a new local table for single_node.upsert_test +NOTICE: creating a new table for single_node.upsert_test NOTICE: Moving the data of single_node.upsert_test NOTICE: Dropping the old single_node.upsert_test NOTICE: Renaming the new table to single_node.upsert_test @@ -1116,7 +1116,7 @@ RESET citus.task_executor_type; -- make sure undistribute table works fine ALTER TABLE test DROP CONSTRAINT foreign_key; SELECT undistribute_table('test_2'); -NOTICE: creating a new local table for single_node.test_2 +NOTICE: creating a new table for single_node.test_2 NOTICE: Moving the data of single_node.test_2 NOTICE: Dropping the old single_node.test_2 NOTICE: Renaming the new table to single_node.test_2 @@ -1166,28 +1166,27 @@ ALTER TABLE distributed_table_1 ADD CONSTRAINT fkey_3 FOREIGN KEY (col_1) REFERE ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_4 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_5 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_2); SELECT undistribute_table('partitioned_table_1', cascade_via_foreign_keys=>true); -NOTICE: undistributing the partitions of single_node.partitioned_table_1 -NOTICE: creating a new local table for single_node.partitioned_table_1_100_200 +NOTICE: converting the partitions of single_node.partitioned_table_1 +NOTICE: creating a new table for single_node.partitioned_table_1_100_200 NOTICE: Moving the data of single_node.partitioned_table_1_100_200 NOTICE: Dropping the old single_node.partitioned_table_1_100_200 NOTICE: Renaming the new table to single_node.partitioned_table_1_100_200 -NOTICE: creating a new local table for single_node.partitioned_table_1_200_300 +NOTICE: creating a new table for single_node.partitioned_table_1_200_300 NOTICE: Moving the data of single_node.partitioned_table_1_200_300 NOTICE: Dropping the old single_node.partitioned_table_1_200_300 NOTICE: Renaming the new table to single_node.partitioned_table_1_200_300 -NOTICE: creating a new local table for single_node.partitioned_table_1 -NOTICE: Moving the data of single_node.partitioned_table_1 +NOTICE: creating a new table for single_node.partitioned_table_1 NOTICE: Dropping the old single_node.partitioned_table_1 NOTICE: Renaming the new table to single_node.partitioned_table_1 -NOTICE: creating a new local table for single_node.reference_table_1 +NOTICE: creating a new table for single_node.reference_table_1 NOTICE: Moving the data of single_node.reference_table_1 NOTICE: Dropping the old single_node.reference_table_1 NOTICE: Renaming the new table to single_node.reference_table_1 -NOTICE: creating a new local table for single_node.distributed_table_1 +NOTICE: creating a new table for single_node.distributed_table_1 NOTICE: Moving the data of single_node.distributed_table_1 NOTICE: Dropping the old single_node.distributed_table_1 NOTICE: Renaming the new table to single_node.distributed_table_1 -NOTICE: creating a new local table for single_node.citus_local_table_1 +NOTICE: creating a new table for single_node.citus_local_table_1 NOTICE: Moving the data of single_node.citus_local_table_1 NOTICE: Dropping the old single_node.citus_local_table_1 NOTICE: Renaming the new table to single_node.citus_local_table_1 @@ -1289,6 +1288,154 @@ SELECT pg_reload_conf(); t (1 row) +-- test alter_distributed_table UDF +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +INSERT INTO adt_ref VALUES (3), (5); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 4 + adt_ref | distributed | a | 4 + adt_table | distributed | a | 4 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | a | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref, adt_table +(1 row) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_col | distributed | a | 6 + adt_ref | distributed | a | 6 + adt_table | distributed | b | 6 +(3 rows) + +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; + Colocation Groups +--------------------------------------------------------------------- + adt_col, adt_ref + adt_table +(2 rows) + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + Referencing Table | Definition +--------------------------------------------------------------------- + adt_col | UNIQUE (a) + adt_ref | FOREIGN KEY (a) REFERENCES adt_col(a) +(2 rows) + +SELECT * FROM adt_table ORDER BY 1; + a | b +--------------------------------------------------------------------- + 1 | 2 + 3 | 4 + 5 | 6 +(3 rows) + +SELECT * FROM adt_col ORDER BY 1; + a | b +--------------------------------------------------------------------- + 3 | 4 + 5 | 6 + 7 | 8 +(3 rows) + +SELECT * FROM adt_ref ORDER BY 1; + a +--------------------------------------------------------------------- + 3 + 5 +(2 rows) + +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); + alter_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT COUNT(*) FROM adt_table; + count +--------------------------------------------------------------------- + 1003 +(1 row) + +END; +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; + Name | Citus Table Type | Distribution Column | Shard Count +--------------------------------------------------------------------- + adt_table | distributed | a | 6 +(1 row) + \c - - - :master_port -- sometimes Postgres is a little slow to terminate the backends -- even if PGFinish is sent. So, to prevent any flaky tests, sleep diff --git a/src/test/regress/expected/undistribute_table.out b/src/test/regress/expected/undistribute_table.out index 16c16de00..9ef8f69a0 100644 --- a/src/test/regress/expected/undistribute_table.out +++ b/src/test/regress/expected/undistribute_table.out @@ -34,13 +34,13 @@ SELECT * FROM dist_table ORDER BY 1, 2, 3; -- we cannot immediately convert in the same statement, because -- the name->OID conversion happens at parse time. SELECT undistribute_table('dist_table'), create_distributed_table('dist_table', 'a'); -NOTICE: creating a new local table for undistribute_table.dist_table +NOTICE: creating a new table for undistribute_table.dist_table NOTICE: Moving the data of undistribute_table.dist_table NOTICE: Dropping the old undistribute_table.dist_table NOTICE: Renaming the new table to undistribute_table.dist_table ERROR: relation with OID XXXX does not exist SELECT undistribute_table('dist_table'); -NOTICE: creating a new local table for undistribute_table.dist_table +NOTICE: creating a new table for undistribute_table.dist_table NOTICE: Moving the data of undistribute_table.dist_table NOTICE: Dropping the old undistribute_table.dist_table NOTICE: Renaming the new table to undistribute_table.dist_table @@ -87,7 +87,7 @@ SELECT * FROM pg_indexes WHERE tablename = 'dist_table'; (1 row) SELECT undistribute_table('dist_table'); -NOTICE: creating a new local table for undistribute_table.dist_table +NOTICE: creating a new table for undistribute_table.dist_table NOTICE: Moving the data of undistribute_table.dist_table NOTICE: Dropping the old undistribute_table.dist_table NOTICE: Renaming the new table to undistribute_table.dist_table @@ -122,10 +122,10 @@ SELECT create_distributed_table('referencing_table', 'id'); INSERT INTO referencing_table VALUES (4, 6, 'cba'), (1, 1, 'dcba'), (2, 3, 'aaa'); SELECT undistribute_table('referenced_table'); -ERROR: cannot undistribute table because a foreign key references to it +ERROR: cannot complete operation because table referenced_table is referenced by a foreign key HINT: Use cascade option to undistribute all the relations involved in a foreign key relationship with undistribute_table.referenced_table by executing SELECT undistribute_table($$undistribute_table.referenced_table$$, cascade_via_foreign_keys=>true) SELECT undistribute_table('referencing_table'); -ERROR: cannot undistribute table because it has a foreign key +ERROR: cannot complete operation because table referencing_table has a foreign key HINT: Use cascade option to undistribute all the relations involved in a foreign key relationship with undistribute_table.referencing_table by executing SELECT undistribute_table($$undistribute_table.referencing_table$$, cascade_via_foreign_keys=>true) DROP TABLE referenced_table, referencing_table; -- test distributed foreign tables @@ -142,7 +142,7 @@ NOTICE: foreign-data wrapper "fake_fdw" does not have an extension defined (1 row) SELECT undistribute_table('foreign_table'); -ERROR: cannot undistribute table because it is a foreign table +ERROR: cannot complete operation because it is a foreign table DROP FOREIGN TABLE foreign_table; -- test partitioned tables CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); @@ -198,21 +198,20 @@ SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; -- undistributing partitions are not supported SELECT undistribute_table('partitioned_table_1_5'); -ERROR: cannot undistribute table because it is a partition -HINT: undistribute the partitioned table "partitioned_table" instead +ERROR: cannot complete operation because table is a partition +HINT: the parent table is "partitioned_table" -- we can undistribute partitioned parent tables SELECT undistribute_table('partitioned_table'); -NOTICE: undistributing the partitions of undistribute_table.partitioned_table -NOTICE: creating a new local table for undistribute_table.partitioned_table_1_5 +NOTICE: converting the partitions of undistribute_table.partitioned_table +NOTICE: creating a new table for undistribute_table.partitioned_table_1_5 NOTICE: Moving the data of undistribute_table.partitioned_table_1_5 NOTICE: Dropping the old undistribute_table.partitioned_table_1_5 NOTICE: Renaming the new table to undistribute_table.partitioned_table_1_5 -NOTICE: creating a new local table for undistribute_table.partitioned_table_6_10 +NOTICE: creating a new table for undistribute_table.partitioned_table_6_10 NOTICE: Moving the data of undistribute_table.partitioned_table_6_10 NOTICE: Dropping the old undistribute_table.partitioned_table_6_10 NOTICE: Renaming the new table to undistribute_table.partitioned_table_6_10 -NOTICE: creating a new local table for undistribute_table.partitioned_table -NOTICE: Moving the data of undistribute_table.partitioned_table +NOTICE: creating a new table for undistribute_table.partitioned_table NOTICE: Dropping the old undistribute_table.partitioned_table NOTICE: Renaming the new table to undistribute_table.partitioned_table undistribute_table @@ -283,7 +282,7 @@ SELECT * FROM seq_table ORDER BY a; (3 rows) SELECT undistribute_table('seq_table'); -NOTICE: creating a new local table for undistribute_table.seq_table +NOTICE: creating a new table for undistribute_table.seq_table NOTICE: Moving the data of undistribute_table.seq_table NOTICE: Dropping the old undistribute_table.seq_table NOTICE: Renaming the new table to undistribute_table.seq_table @@ -348,7 +347,7 @@ SELECT * FROM another_schema.undis_view3 ORDER BY 1, 2; (3 rows) SELECT undistribute_table('view_table'); -NOTICE: creating a new local table for undistribute_table.view_table +NOTICE: creating a new table for undistribute_table.view_table NOTICE: Moving the data of undistribute_table.view_table NOTICE: Dropping the old undistribute_table.view_table NOTICE: drop cascades to 3 other objects diff --git a/src/test/regress/expected/undistribute_table_cascade.out b/src/test/regress/expected/undistribute_table_cascade.out index 67942b831..269dddc16 100644 --- a/src/test/regress/expected/undistribute_table_cascade.out +++ b/src/test/regress/expected/undistribute_table_cascade.out @@ -80,11 +80,11 @@ ALTER TABLE distributed_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFER ALTER TABLE reference_table_1 ADD CONSTRAINT fkey_12 FOREIGN KEY (col_2) REFERENCES reference_table_1(col_1); -- show that all of below fails as we didn't provide cascade=true SELECT undistribute_table('distributed_table_1'); -ERROR: cannot undistribute table because it has a foreign key +ERROR: cannot complete operation because table distributed_table_1 has a foreign key SELECT undistribute_table('citus_local_table_1', cascade_via_foreign_keys=>false); -ERROR: cannot undistribute table because it has a foreign key +ERROR: cannot complete operation because table citus_local_table_1 has a foreign key SELECT undistribute_table('reference_table_2'); -ERROR: cannot undistribute table because it has a foreign key +ERROR: cannot complete operation because table reference_table_2 has a foreign key -- In each of below transation blocks, show that we preserve foreign keys. -- Also show that we don't have any citus tables in current schema after -- undistribute_table(cascade). @@ -304,6 +304,17 @@ SELECT create_reference_table('reference_table_3'); ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_9 FOREIGN KEY (col_1) REFERENCES reference_table_3(col_2); ALTER TABLE partitioned_table_2 ADD CONSTRAINT fkey_10 FOREIGN KEY (col_1) REFERENCES reference_table_3(col_2); +-- show that we properly handle cases where undistribute_table is not supported +-- error out when table is a partition table +SELECT undistribute_table('partitioned_table_2_100_200', cascade_via_foreign_keys=>true); +ERROR: cannot complete operation because table is a partition +-- error if table does not exist +SELECT undistribute_table('non_existent_table', cascade_via_foreign_keys=>true); +ERROR: relation "non_existent_table" does not exist at character 27 +-- error if table is a postgres table +CREATE TABLE local_table(a int); +SELECT undistribute_table('local_table', cascade_via_foreign_keys=>true); +ERROR: cannot undistribute table because the table is not distributed -- as pg < 12 doesn't support foreign keys between partitioned tables, -- define below foreign key conditionally instead of adding another -- test output @@ -376,7 +387,7 @@ BEGIN; set citus.multi_shard_modify_mode to 'sequential'; ALTER TABLE partitioned_table_1_100_200 ADD CONSTRAINT non_inherited_fkey FOREIGN KEY(col_1) REFERENCES partitioned_table_2_100_200(col_1); SELECT undistribute_table('partitioned_table_2', cascade_via_foreign_keys=>true); -ERROR: cannot cascade operation via foreign keys as partition table undistribute_table_cascade.partitioned_table_1_100_200 involved in a foreign key relationship that is not inherited from it's parent table +ERROR: cannot cascade operation via foreign keys as partition table undistribute_table_cascade.partitioned_table_2_100_200 involved in a foreign key relationship that is not inherited from it's parent table ROLLBACK; BEGIN; set citus.multi_shard_modify_mode to 'sequential'; diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 83a644aef..841a307fa 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -20,7 +20,9 @@ ORDER BY 1; event trigger citus_cascade_to_partition function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) function alter_columnar_table_set(regclass,integer,integer,name,integer) + function alter_distributed_table(regclass,text,integer,text,boolean) function alter_role_if_exists(text,text) + function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) function array_cat_agg(anyarray) @@ -175,6 +177,7 @@ ORDER BY 1; function worker_apply_sequence_command(text,regtype) function worker_apply_shard_ddl_command(bigint,text) function worker_apply_shard_ddl_command(bigint,text,text) + function worker_change_sequence_dependency(regclass,regclass,regclass) function worker_cleanup_job_schema_cache() function worker_create_or_alter_role(text,text,text) function worker_create_or_replace_object(text) @@ -233,5 +236,5 @@ ORDER BY 1; view citus_worker_stat_activity view pg_dist_shard_placement view time_partitions -(217 rows) +(220 rows) diff --git a/src/test/regress/expected/upgrade_list_citus_objects_0.out b/src/test/regress/expected/upgrade_list_citus_objects_0.out index 8219ef2bf..9771cdb3e 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects_0.out +++ b/src/test/regress/expected/upgrade_list_citus_objects_0.out @@ -17,7 +17,9 @@ ORDER BY 1; description --------------------------------------------------------------------- event trigger citus_cascade_to_partition + function alter_distributed_table(regclass,text,integer,text,boolean) function alter_role_if_exists(text,text) + function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) function array_cat_agg(anyarray) @@ -171,6 +173,7 @@ ORDER BY 1; function worker_apply_sequence_command(text,regtype) function worker_apply_shard_ddl_command(bigint,text) function worker_apply_shard_ddl_command(bigint,text,text) + function worker_change_sequence_dependency(regclass,regclass,regclass) function worker_cleanup_job_schema_cache() function worker_create_or_alter_role(text,text,text) function worker_create_or_replace_object(text) @@ -229,5 +232,5 @@ ORDER BY 1; view citus_worker_stat_activity view pg_dist_shard_placement view time_partitions -(213 rows) +(216 rows) diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule index b19eafc97..5a1f1f7e2 100644 --- a/src/test/regress/multi_mx_schedule +++ b/src/test/regress/multi_mx_schedule @@ -51,6 +51,7 @@ test: multi_mx_explain test: multi_mx_reference_table test: multi_mx_insert_select_repartition test: locally_execute_intermediate_results +test: multi_mx_alter_distributed_table # test that no tests leaked intermediate results. This should always be last test: ensure_no_intermediate_data_leak diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index f776b9993..8bbad2220 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -124,6 +124,8 @@ test: multi_task_assignment_policy multi_cross_shard test: multi_utility_statements test: multi_dropped_column_aliases foreign_key_restriction_enforcement test: multi_binary_master_copy_format binary_protocol +test: alter_table_set_access_method +test: alter_distributed_table # ---------- # Parallel TPC-H tests to check our distributed execution behavior diff --git a/src/test/regress/sql/alter_distributed_table.sql b/src/test/regress/sql/alter_distributed_table.sql new file mode 100644 index 000000000..cbac8b461 --- /dev/null +++ b/src/test/regress/sql/alter_distributed_table.sql @@ -0,0 +1,276 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven; +\gset + +CREATE SCHEMA alter_distributed_table; +SET search_path TO alter_distributed_table; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; + +CREATE TABLE dist_table (a INT, b INT); +SELECT create_distributed_table ('dist_table', 'a', colocate_with := 'none'); +INSERT INTO dist_table VALUES (1, 1), (2, 2), (3, 3); + +CREATE TABLE colocation_table (a INT, b INT); +SELECT create_distributed_table ('colocation_table', 'a', colocate_with := 'none'); + +CREATE TABLE colocation_table_2 (a INT, b INT); +SELECT create_distributed_table ('colocation_table_2', 'a', colocate_with := 'none'); + + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + +-- test altering distribution column +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + +-- test altering shard count +SELECT alter_distributed_table('dist_table', shard_count := 6); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + +-- test altering colocation, note that shard count will also change +SELECT alter_distributed_table('dist_table', colocate_with := 'alter_distributed_table.colocation_table'); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + +-- test altering shard count with cascading, note that the colocation will be kept +SELECT alter_distributed_table('dist_table', shard_count := 8, cascade_to_colocated := true); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + +-- test altering shard count without cascading, note that the colocation will be broken +SELECT alter_distributed_table('dist_table', shard_count := 10, cascade_to_colocated := false); +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2'); +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables + WHERE "Name" IN ('dist_table', 'colocation_table', 'colocation_table_2') GROUP BY "Colocation ID" ORDER BY 1; + + +-- test partitions +CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); +SELECT create_distributed_table('partitioned_table', 'id', colocate_with := 'none'); +CREATE TABLE partitioned_table_1_5 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (5); +CREATE TABLE partitioned_table_6_10 PARTITION OF partitioned_table FOR VALUES FROM (6) TO (10); +INSERT INTO partitioned_table VALUES (2, 12), (7, 2); + +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT * FROM partitioned_table ORDER BY 1, 2; +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + +-- test altering the parent table +SELECT alter_distributed_table('partitioned_table', shard_count := 10, distribution_column := 'a'); + +-- test altering the partition +SELECT alter_distributed_table('partitioned_table_1_5', shard_count := 10, distribution_column := 'a'); + +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name"::text, "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT * FROM partitioned_table ORDER BY 1, 2; +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + + +-- test references +CREATE TABLE referenced_dist_table (a INT UNIQUE); +CREATE TABLE referenced_ref_table (a INT UNIQUE); +CREATE TABLE table_with_references (a1 INT UNIQUE REFERENCES referenced_dist_table(a), a2 INT REFERENCES referenced_ref_table(a)); +CREATE TABLE referencing_dist_table (a INT REFERENCES table_with_references(a1)); + +SELECT create_distributed_table('referenced_dist_table', 'a', colocate_with:='none'); +SELECT create_reference_table('referenced_ref_table'); +SELECT create_distributed_table('table_with_references', 'a1', colocate_with:='referenced_dist_table'); +SELECT create_distributed_table('referencing_dist_table', 'a', colocate_with:='referenced_dist_table'); + +SET client_min_messages TO WARNING; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; +SELECT alter_distributed_table('table_with_references', shard_count := 12, cascade_to_colocated := true); +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; +SELECT alter_distributed_table('table_with_references', shard_count := 10, cascade_to_colocated := false); +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + + +-- check when multi shard modify mode is set to sequential +SELECT alter_distributed_table('referenced_dist_table', colocate_with:='none'); +CREATE TABLE ref_to_dist_table(a INT REFERENCES referenced_dist_table(a)); +CREATE TABLE ref_to_ref_table(a INT REFERENCES referenced_ref_table(a)); +SELECT create_distributed_table('ref_to_dist_table', 'a', colocate_with:='referenced_dist_table'); +SELECT create_distributed_table('ref_to_ref_table', 'a', colocate_with:='none'); + +-- alter a table referencing a reference table +SELECT alter_distributed_table('ref_to_ref_table', shard_count:=6); + +-- let's create a table that is not colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_dist (a INT); +SELECT create_distributed_table('col_with_ref_to_dist', 'a', colocate_with:='ref_to_dist_table'); +-- and create a table colocated with a table that references a reference table +CREATE TABLE col_with_ref_to_ref (a INT); +SELECT alter_distributed_table('ref_to_ref_table', colocate_with:='none'); +SELECT create_distributed_table('col_with_ref_to_ref', 'a', colocate_with:='ref_to_ref_table'); + +-- alter a table colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=8, cascade_to_colocated:=true); +-- alter a table colocated with a table referencing a reference table without cascading +SELECT alter_distributed_table('col_with_ref_to_ref', shard_count:=10, cascade_to_colocated:=false); +-- alter a table not colocated with a table referencing a reference table with cascading +SELECT alter_distributed_table('col_with_ref_to_dist', shard_count:=6, cascade_to_colocated:=true); + + +\if :server_version_above_eleven +-- test altering columnar table +CREATE TABLE columnar_table (a INT) USING columnar; +SELECT create_distributed_table('columnar_table', 'a', colocate_with:='none'); +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; +SELECT alter_distributed_table('columnar_table', shard_count:=6); +SELECT "Name"::text, "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'columnar_table'; +\endif + + +-- test with metadata sync +SET citus.replication_model TO 'streaming'; +SELECT start_metadata_sync_to_node('localhost', :worker_1_port); + +CREATE TABLE metadata_sync_table (a BIGSERIAL); +SELECT create_distributed_table('metadata_sync_table', 'a', colocate_with:='none'); + +SELECT alter_distributed_table('metadata_sync_table', shard_count:=6); +SELECT alter_distributed_table('metadata_sync_table', shard_count:=8); + +SELECT "Name", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'metadata_sync_table'; + +SET citus.replication_model TO DEFAULT; +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); + +-- test complex cascade operations +CREATE TABLE cas_1 (a INT UNIQUE); +CREATE TABLE cas_2 (a INT UNIQUE); + +CREATE TABLE cas_3 (a INT UNIQUE); +CREATE TABLE cas_4 (a INT UNIQUE); + +CREATE TABLE cas_par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE cas_par_1 PARTITION OF cas_par FOR VALUES FROM (1) TO (4); +CREATE TABLE cas_par_2 PARTITION OF cas_par FOR VALUES FROM (5) TO (8); + +CREATE TABLE cas_col (a INT UNIQUE); + +-- add foreign keys from and to partitions +ALTER TABLE cas_par_1 ADD CONSTRAINT fkey_from_par_1 FOREIGN KEY (a) REFERENCES cas_1(a); +ALTER TABLE cas_2 ADD CONSTRAINT fkey_to_par_1 FOREIGN KEY (a) REFERENCES cas_par_1(a); + +ALTER TABLE cas_par ADD CONSTRAINT fkey_from_par FOREIGN KEY (a) REFERENCES cas_3(a); +ALTER TABLE cas_4 ADD CONSTRAINT fkey_to_par FOREIGN KEY (a) REFERENCES cas_par(a); + +-- distribute all the tables +SELECT create_distributed_table('cas_1', 'a', colocate_with:='none'); +SELECT create_distributed_table('cas_3', 'a', colocate_with:='cas_1'); +SELECT create_distributed_table('cas_par', 'a', colocate_with:='cas_1'); +SELECT create_distributed_table('cas_2', 'a', colocate_with:='cas_1'); +SELECT create_distributed_table('cas_4', 'a', colocate_with:='cas_1'); +SELECT create_distributed_table('cas_col', 'a', colocate_with:='cas_1'); + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; + +-- alter the cas_col and cascade the change +SELECT alter_distributed_table('cas_col', shard_count:=6, cascade_to_colocated:=true); + +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'cas_par_1' OR confrelid::regclass::text = 'cas_par_1') ORDER BY 1, 2; +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'cas_par'::regclass ORDER BY 1; +SET client_min_messages TO DEFAULT; + + +-- test changing dist column and colocating partitioned table without changing shard count +CREATE TABLE col_table (a INT); +SELECT create_distributed_table('col_table', 'a', colocate_with:='none'); +CREATE TABLE par_table (a BIGINT, b INT) PARTITION BY RANGE (a); +SELECT create_distributed_table('par_table', 'a', colocate_with:='none'); +CREATE TABLE par_table_1 (a BIGINT, b INT); +SELECT create_distributed_table('par_table_1', 'a', colocate_with:='par_table'); +ALTER TABLE par_table ATTACH PARTITION par_table_1 FOR VALUES FROM (1) TO (5); + +SELECT alter_distributed_table('par_table', distribution_column:='b', colocate_with:='col_table'); + +-- test messages +-- test nothing to change +SELECT alter_distributed_table('dist_table'); +SELECT alter_distributed_table('dist_table', cascade_to_colocated := false); + +-- no operation UDF calls +SELECT alter_distributed_table('dist_table', distribution_column := 'b'); +SELECT alter_distributed_table('dist_table', shard_count := 10); +-- first colocate the tables, then try to re-colococate +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table'); + +-- test some changes while keeping others same +-- shouldn't error but should have notices about no-change parameters +SELECT alter_distributed_table('dist_table', distribution_column:='b', shard_count:=4, cascade_to_colocated:=false); +SELECT alter_distributed_table('dist_table', shard_count:=4, colocate_with:='colocation_table_2'); +SELECT alter_distributed_table('dist_table', colocate_with:='colocation_table_2', distribution_column:='a'); + +-- test cascading distribution column, should error +SELECT alter_distributed_table('dist_table', distribution_column := 'b', cascade_to_colocated := true); +SELECT alter_distributed_table('dist_table', distribution_column := 'b', shard_count:=12, colocate_with:='colocation_table_2', cascade_to_colocated := true); + +-- test nothing to cascade +SELECT alter_distributed_table('dist_table', cascade_to_colocated := true); + +-- test cascading colocate_with := 'none' +SELECT alter_distributed_table('dist_table', colocate_with := 'none', cascade_to_colocated := true); + +-- test changing shard count of a colocated table without cascade_to_colocated, should error +SELECT alter_distributed_table('dist_table', shard_count := 14); + +-- test changing shard count of a non-colocated table without cascade_to_colocated, shouldn't error +SELECT alter_distributed_table('dist_table', colocate_with := 'none'); +SELECT alter_distributed_table('dist_table', shard_count := 14); + +-- test altering a table into colocating with a table but giving a different shard count +SELECT alter_distributed_table('dist_table', colocate_with := 'colocation_table', shard_count := 16); + +-- test colocation with distribution columns with different data types +CREATE TABLE different_type_table (a TEXT); +SELECT create_distributed_table('different_type_table', 'a'); + +SELECT alter_distributed_table('dist_table', colocate_with := 'different_type_table'); +SELECT alter_distributed_table('dist_table', distribution_column := 'a', colocate_with := 'different_type_table'); + +-- test shard_count := 0 +SELECT alter_distributed_table('dist_table', shard_count := 0); + +-- test colocating with non-distributed table +CREATE TABLE reference_table (a INT); +SELECT create_reference_table('reference_table'); +SELECT alter_distributed_table('dist_table', colocate_with:='reference_table'); + +-- test append table +CREATE TABLE append_table (a INT); +SELECT create_distributed_table('append_table', 'a', 'append'); +SELECT alter_distributed_table('append_table', shard_count:=6); + +SET client_min_messages TO WARNING; +DROP SCHEMA alter_distributed_table CASCADE; diff --git a/src/test/regress/sql/alter_table_set_access_method.sql b/src/test/regress/sql/alter_table_set_access_method.sql new file mode 100644 index 000000000..72f529fc5 --- /dev/null +++ b/src/test/regress/sql/alter_table_set_access_method.sql @@ -0,0 +1,115 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif + + +CREATE SCHEMA alter_table_set_access_method; +SET search_path TO alter_table_set_access_method; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; + +SELECT public.run_command_on_coordinator_and_workers($Q$ + CREATE FUNCTION fake_am_handler(internal) + RETURNS table_am_handler + AS 'citus' + LANGUAGE C; + CREATE ACCESS METHOD fake_am TYPE TABLE HANDLER fake_am_handler; +$Q$); + +CREATE TABLE dist_table (a INT, b INT); +SELECT create_distributed_table ('dist_table', 'a'); +INSERT INTO dist_table VALUES (1, 1), (2, 2), (3, 3); + +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'dist_table' ORDER BY 1; +SELECT alter_table_set_access_method('dist_table', 'columnar'); +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'dist_table' ORDER BY 1; + + +-- test partitions +CREATE TABLE partitioned_table (id INT, a INT) PARTITION BY RANGE (id); +CREATE TABLE partitioned_table_1_5 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (5); +CREATE TABLE partitioned_table_6_10 PARTITION OF partitioned_table FOR VALUES FROM (6) TO (10); +SELECT create_distributed_table('partitioned_table', 'id'); +INSERT INTO partitioned_table VALUES (2, 12), (7, 2); + +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name"::text, "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT * FROM partitioned_table ORDER BY 1, 2; +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + +-- altering partitioned tables' access methods is not supported +SELECT alter_table_set_access_method('partitioned_table', 'columnar'); +-- test altering the partition's access method +SELECT alter_table_set_access_method('partitioned_table_1_5', 'columnar'); + +SELECT logicalrelid::text FROM pg_dist_partition WHERE logicalrelid::regclass::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT run_command_on_workers($$SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE 'partitioned\_table%'$$); +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name"::text, "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'partitioned\_table%' ORDER BY 1; +SELECT * FROM partitioned_table ORDER BY 1, 2; +SELECT * FROM partitioned_table_1_5 ORDER BY 1, 2; +SELECT * FROM partitioned_table_6_10 ORDER BY 1, 2; + + +-- test altering a table with index to columnar +-- the index will be dropped +CREATE TABLE index_table (a INT) USING heap; +CREATE INDEX idx1 ON index_table (a); +SELECT indexname FROM pg_indexes WHERE schemaname = 'alter_table_set_access_method' AND tablename = 'index_table'; +SELECT a.amname FROM pg_class c, pg_am a where c.relname = 'index_table' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; +SELECT alter_table_set_access_method('index_table', 'columnar'); +SELECT indexname FROM pg_indexes WHERE schemaname = 'alter_table_set_access_method' AND tablename = 'index_table'; +SELECT a.amname FROM pg_class c, pg_am a where c.relname = 'index_table' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + + +-- test different table types +SET client_min_messages to WARNING; +SELECT 1 FROM master_add_node('localhost', :master_port, groupId := 0); +SET client_min_messages to DEFAULT; +CREATE TABLE table_type_dist (a INT); +SELECT create_distributed_table('table_type_dist', 'a'); +CREATE TABLE table_type_ref (a INT); +SELECT create_reference_table('table_type_ref'); +CREATE TABLE table_type_citus_local(a INT); +SELECT create_citus_local_table('table_type_citus_local'); +CREATE TABLE table_type_pg_local (a INT); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'table\_type%' ORDER BY 1; +SELECT c.relname, a.amname FROM pg_class c, pg_am a where c.relname SIMILAR TO 'table_type\D*' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + +SELECT alter_table_set_access_method('table_type_dist', 'fake_am'); +SELECT alter_table_set_access_method('table_type_ref', 'fake_am'); +SELECT alter_table_set_access_method('table_type_pg_local', 'fake_am'); +SELECT alter_table_set_access_method('table_type_citus_local', 'fake_am'); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count", "Access Method" FROM public.citus_tables WHERE "Name"::text LIKE 'table\_type%' ORDER BY 1; +SELECT c.relname, a.amname FROM pg_class c, pg_am a where c.relname SIMILAR TO 'table_type\D*' AND c.relnamespace = 'alter_table_set_access_method'::regnamespace AND c.relam = a.oid; + +-- test when the parent of a partition has foreign key to a reference table +CREATE TABLE ref_table (a INT UNIQUE); +SELECT create_reference_table('ref_table'); +INSERT INTO ref_table VALUES (2), (12); +ALTER TABLE partitioned_table ADD CONSTRAINT fkey_to_ref FOREIGN KEY (a) REFERENCES ref_table(a); + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'partitioned_table_6_10'; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'partitioned_table_6_10') ORDER BY 1, 2; + +SELECT alter_table_set_access_method('partitioned_table_6_10', 'columnar'); + +SELECT inhrelid::regclass::text FROM pg_catalog.pg_inherits WHERE inhparent = 'partitioned_table'::regclass ORDER BY 1; +SELECT "Name", "Access Method" FROM public.citus_tables WHERE "Name"::text = 'partitioned_table_6_10'; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'partitioned_table_6_10') ORDER BY 1, 2; + +SET client_min_messages TO WARNING; +DROP SCHEMA alter_table_set_access_method CASCADE; +SELECT 1 FROM master_remove_node('localhost', :master_port); diff --git a/src/test/regress/sql/coordinator_shouldhaveshards.sql b/src/test/regress/sql/coordinator_shouldhaveshards.sql index 42c5744ce..6b4f73975 100644 --- a/src/test/regress/sql/coordinator_shouldhaveshards.sql +++ b/src/test/regress/sql/coordinator_shouldhaveshards.sql @@ -272,6 +272,57 @@ CREATE INDEX ix_test_index_creation5 ON test_index_creation1 -- test if indexes are created SELECT 1 AS created WHERE EXISTS(SELECT * FROM pg_indexes WHERE indexname LIKE '%test_index_creation%'); + +-- test alter_distributed_table UDF +SET citus.shard_count TO 4; +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); + +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +INSERT INTO adt_ref VALUES (3), (5); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SET client_min_messages TO WARNING; +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); +SET client_min_messages TO DEFAULT; + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT * FROM adt_table ORDER BY 1; +SELECT * FROM adt_col ORDER BY 1; +SELECT * FROM adt_ref ORDER BY 1; + +SET client_min_messages TO WARNING; +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); +SELECT COUNT(*) FROM adt_table; +END; + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; +SET client_min_messages TO DEFAULT; + + \set VERBOSITY terse DROP TABLE ref_table; diff --git a/src/test/regress/sql/multi_mx_alter_distributed_table.sql b/src/test/regress/sql/multi_mx_alter_distributed_table.sql new file mode 100644 index 000000000..e754595af --- /dev/null +++ b/src/test/regress/sql/multi_mx_alter_distributed_table.sql @@ -0,0 +1,52 @@ +CREATE SCHEMA mx_alter_distributed_table; +SET search_path TO mx_alter_distributed_table; +SET citus.shard_replication_factor TO 1; + +-- test alter_distributed_table UDF +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); + +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +INSERT INTO adt_ref VALUES (3), (5); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SET client_min_messages TO WARNING; +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); +SET client_min_messages TO DEFAULT; + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT * FROM adt_table ORDER BY 1; +SELECT * FROM adt_col ORDER BY 1; +SELECT * FROM adt_ref ORDER BY 1; + +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); +SELECT COUNT(*) FROM adt_table; +END; + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; + +SET client_min_messages TO WARNING; +DROP SCHEMA mx_alter_distributed_table CASCADE; diff --git a/src/test/regress/sql/single_node.sql b/src/test/regress/sql/single_node.sql index 1a9128157..b660ccb22 100644 --- a/src/test/regress/sql/single_node.sql +++ b/src/test/regress/sql/single_node.sql @@ -701,6 +701,50 @@ SELECT pg_sleep(0.1); ALTER SYSTEM SET citus.recover_2pc_interval TO '-1'; SELECT pg_reload_conf(); +-- test alter_distributed_table UDF +CREATE TABLE adt_table (a INT, b INT); +CREATE TABLE adt_col (a INT UNIQUE, b INT); +CREATE TABLE adt_ref (a INT REFERENCES adt_col(a)); + +SELECT create_distributed_table('adt_table', 'a', colocate_with:='none'); +SELECT create_distributed_table('adt_col', 'a', colocate_with:='adt_table'); +SELECT create_distributed_table('adt_ref', 'a', colocate_with:='adt_table'); + +INSERT INTO adt_table VALUES (1, 2), (3, 4), (5, 6); +INSERT INTO adt_col VALUES (3, 4), (5, 6), (7, 8); +INSERT INTO adt_ref VALUES (3), (5); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT alter_distributed_table('adt_table', shard_count:=6, cascade_to_colocated:=true); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT alter_distributed_table('adt_table', distribution_column:='b', colocate_with:='none'); + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%'; +SELECT STRING_AGG("Name"::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM public.citus_tables WHERE "Name"::text LIKE 'adt%' GROUP BY "Colocation ID" ORDER BY 1; +SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint + WHERE (conrelid::regclass::text = 'adt_col' OR confrelid::regclass::text = 'adt_col') ORDER BY 1; + +SELECT * FROM adt_table ORDER BY 1; +SELECT * FROM adt_col ORDER BY 1; +SELECT * FROM adt_ref ORDER BY 1; + +BEGIN; +INSERT INTO adt_table SELECT x, x+1 FROM generate_series(1, 1000) x; +SELECT alter_distributed_table('adt_table', distribution_column:='a'); +SELECT COUNT(*) FROM adt_table; +END; + +SELECT "Name", "Citus Table Type", "Distribution Column", "Shard Count" FROM public.citus_tables WHERE "Name"::text = 'adt_table'; + \c - - - :master_port -- sometimes Postgres is a little slow to terminate the backends diff --git a/src/test/regress/sql/undistribute_table_cascade.sql b/src/test/regress/sql/undistribute_table_cascade.sql index d7d1cfea4..20d43ead0 100644 --- a/src/test/regress/sql/undistribute_table_cascade.sql +++ b/src/test/regress/sql/undistribute_table_cascade.sql @@ -175,6 +175,15 @@ SELECT create_reference_table('reference_table_3'); ALTER TABLE partitioned_table_1 ADD CONSTRAINT fkey_9 FOREIGN KEY (col_1) REFERENCES reference_table_3(col_2); ALTER TABLE partitioned_table_2 ADD CONSTRAINT fkey_10 FOREIGN KEY (col_1) REFERENCES reference_table_3(col_2); +-- show that we properly handle cases where undistribute_table is not supported +-- error out when table is a partition table +SELECT undistribute_table('partitioned_table_2_100_200', cascade_via_foreign_keys=>true); +-- error if table does not exist +SELECT undistribute_table('non_existent_table', cascade_via_foreign_keys=>true); +-- error if table is a postgres table +CREATE TABLE local_table(a int); +SELECT undistribute_table('local_table', cascade_via_foreign_keys=>true); + -- as pg < 12 doesn't support foreign keys between partitioned tables, -- define below foreign key conditionally instead of adding another -- test output