diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index ae8d9d725..aecf8876c 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -157,7 +157,6 @@ jobs: apt-get update -y ## Install required packages to execute packaging tools for deb based distros - apt install python3-dev python3-pip -y - sudo apt-get purge -y python3-yaml - python3 -m pip install --upgrade pip setuptools==57.5.0 + apt-get install python3-dev python3-pip -y + apt-get purge -y python3-yaml ./.github/packaging/validate_build_output.sh "deb" diff --git a/configure b/configure index 1d954457c..cd8cb3745 100755 --- a/configure +++ b/configure @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='11.2.0' -PACKAGE_STRING='Citus 11.2.0' +PACKAGE_VERSION='11.2.1' +PACKAGE_STRING='Citus 11.2.1' PACKAGE_BUGREPORT='' PACKAGE_URL='' diff --git a/configure.ac b/configure.ac index e8d7054d8..b60bd7913 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [11.2.0]) +AC_INIT([Citus], [11.2.1]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control index 0d6bb3a9b..8e8181737 100644 --- a/src/backend/distributed/citus.control +++ b/src/backend/distributed/citus.control @@ -1,6 +1,6 @@ # Citus extension comment = 'Citus distributed database' -default_version = '11.2-1' +default_version = '11.2-2' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index 7db1da813..b8884e2b1 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -913,6 +913,13 @@ CopyTableConversionReturnIntoCurrentContext(TableConversionReturn *tableConversi static TableConversionReturn * ConvertTable(TableConversionState *con) { + /* + * We do not allow alter_distributed_table and undistribute_table operations + * for tables with identity columns. This is because we do not have a proper way + * of keeping sequence states consistent across the cluster. + */ + ErrorIfTableHasIdentityColumn(con->relationId); + /* * when there are many partitions or colocated tables, memory usage is * accumulated. Free context for each call to ConvertTable. @@ -1588,96 +1595,6 @@ CreateMaterializedViewDDLCommand(Oid matViewOid) } -/* - * This function marks all the identity sequences as distributed on the given table. - */ -static void -MarkIdentitiesAsDistributed(Oid targetRelationId) -{ - Relation relation = relation_open(targetRelationId, AccessShareLock); - TupleDesc tupleDescriptor = RelationGetDescr(relation); - relation_close(relation, NoLock); - - bool missingSequenceOk = false; - - for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; - attributeIndex++) - { - Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); - - if (attributeForm->attidentity) - { - Oid seqOid = getIdentitySequence(targetRelationId, attributeForm->attnum, - missingSequenceOk); - - ObjectAddress seqAddress = { 0 }; - ObjectAddressSet(seqAddress, RelationRelationId, seqOid); - MarkObjectDistributed(&seqAddress); - } - } -} - - -/* - * This function returns sql statements to rename identites on the given table - */ -static void -PrepareRenameIdentitiesCommands(Oid sourceRelationId, Oid targetRelationId, - List **outCoordinatorCommands, List **outWorkerCommands) -{ - Relation targetRelation = relation_open(targetRelationId, AccessShareLock); - TupleDesc targetTupleDescriptor = RelationGetDescr(targetRelation); - relation_close(targetRelation, NoLock); - - bool missingSequenceOk = false; - - for (int attributeIndex = 0; attributeIndex < targetTupleDescriptor->natts; - attributeIndex++) - { - Form_pg_attribute attributeForm = TupleDescAttr(targetTupleDescriptor, - attributeIndex); - - if (attributeForm->attidentity) - { - char *columnName = NameStr(attributeForm->attname); - - Oid targetSequenceOid = getIdentitySequence(targetRelationId, - attributeForm->attnum, - missingSequenceOk); - char *targetSequenceName = generate_relation_name(targetSequenceOid, NIL); - - Oid sourceSequenceOid = getIdentitySequence(sourceRelationId, - attributeForm->attnum, - missingSequenceOk); - char *sourceSequenceName = generate_relation_name(sourceSequenceOid, NIL); - - /* to rename sequence on the coordinator */ - *outCoordinatorCommands = lappend(*outCoordinatorCommands, psprintf( - "SET citus.enable_ddl_propagation TO OFF; ALTER SEQUENCE %s RENAME TO %s; RESET citus.enable_ddl_propagation;", - quote_identifier( - targetSequenceName), - quote_identifier( - sourceSequenceName))); - - /* update workers to use existing sequence and drop the new one generated by PG */ - bool missingTableOk = true; - *outWorkerCommands = lappend(*outWorkerCommands, - GetAlterColumnWithNextvalDefaultCmd( - sourceSequenceOid, sourceRelationId, - columnName, - missingTableOk)); - - - /* drop the sequence generated by identity column */ - *outWorkerCommands = lappend(*outWorkerCommands, psprintf( - "DROP SEQUENCE IF EXISTS %s", - quote_identifier( - targetSequenceName))); - } - } -} - - /* * ReplaceTable replaces the source table with the target table. * It moves all the rows of the source table to target table with INSERT SELECT. @@ -1736,24 +1653,6 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, ExecuteQueryViaSPI(query->data, SPI_OK_INSERT); } - /* - * Drop identity dependencies (sequences marked as DEPENDENCY_INTERNAL) on the workers - * to keep their states after the source table is dropped. - */ - List *ownedIdentitySequences = getOwnedSequences_internal(sourceId, 0, - DEPENDENCY_INTERNAL); - if (ownedIdentitySequences != NIL && ShouldSyncTableMetadata(sourceId)) - { - char *qualifiedTableName = quote_qualified_identifier(schemaName, sourceName); - StringInfo command = makeStringInfo(); - - appendStringInfo(command, - "SELECT pg_catalog.worker_drop_sequence_dependency(%s);", - quote_literal_cstr(qualifiedTableName)); - - SendCommandToWorkersWithMetadata(command->data); - } - /* * Modify regular sequence dependencies (sequences marked as DEPENDENCY_AUTO) */ @@ -1813,23 +1712,6 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, quote_qualified_identifier(schemaName, sourceName)))); } - /* - * We need to prepare rename identities commands before dropping the original table, - * otherwise we can't find the original names of the identity sequences. - * We prepare separate commands for the coordinator and the workers because: - * In the coordinator, we simply need to rename the identity sequences - * to their names on the old table, because right now the identity - * sequences have default names generated by Postgres with the creation of the new table - * In the workers, we have not dropped the original identity sequences, - * so what we do is we alter the columns and set their default to the - * original identity sequences, and after that we drop the new sequences. - */ - List *coordinatorCommandsToRenameIdentites = NIL; - List *workerCommandsToRenameIdentites = NIL; - PrepareRenameIdentitiesCommands(sourceId, targetId, - &coordinatorCommandsToRenameIdentites, - &workerCommandsToRenameIdentites); - resetStringInfo(query); appendStringInfo(query, "DROP %sTABLE %s CASCADE", IsForeignTable(sourceId) ? "FOREIGN " : "", @@ -1847,27 +1729,6 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, quote_qualified_identifier(schemaName, targetName), quote_identifier(sourceName)); ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); - - char *coordinatorCommand = NULL; - foreach_ptr(coordinatorCommand, coordinatorCommandsToRenameIdentites) - { - ExecuteQueryViaSPI(coordinatorCommand, SPI_OK_UTILITY); - } - - char *workerCommand = NULL; - foreach_ptr(workerCommand, workerCommandsToRenameIdentites) - { - SendCommandToWorkersWithMetadata(workerCommand); - } - - /* - * To preserve identity sequences states in case of redistributing the table again, - * we don't drop them when we undistribute a table. To maintain consistency and - * avoid future problems if we redistribute the table, we want to apply all changes happening to - * the identity sequence in the coordinator to their corresponding sequences in the workers as well. - * That's why we have to mark identity sequences as distributed - */ - MarkIdentitiesAsDistributed(targetId); } diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index bb4ab7473..fdf36fbbb 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -1131,7 +1131,7 @@ DropIdentitiesOnTable(Oid relationId) { Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); - relation_close(relation, NoLock); + List *dropCommandList = NIL; for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) @@ -1151,15 +1151,23 @@ DropIdentitiesOnTable(Oid relationId) qualifiedTableName, columnName); - /* - * We need to disable/enable ddl propagation for this command, to prevent - * sending unnecessary ALTER COLUMN commands for partitions, to MX workers. - */ - ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, - dropCommand->data, - ENABLE_DDL_PROPAGATION)); + dropCommandList = lappend(dropCommandList, dropCommand->data); } } + + relation_close(relation, NoLock); + + char *dropCommand = NULL; + foreach_ptr(dropCommand, dropCommandList) + { + /* + * We need to disable/enable ddl propagation for this command, to prevent + * sending unnecessary ALTER COLUMN commands for partitions, to MX workers. + */ + ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, + dropCommand, + ENABLE_DDL_PROPAGATION)); + } } diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 0bea11034..8006ae895 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -1190,7 +1190,7 @@ EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid ownerRelationId foreach_oid(citusTableId, citusTableIdList) { List *seqInfoList = NIL; - GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0); + GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0, DEPENDENCY_AUTO); SequenceInfo *seqInfo = NULL; foreach_ptr(seqInfo, seqInfoList) @@ -1267,7 +1267,7 @@ EnsureRelationHasCompatibleSequenceTypes(Oid relationId) { List *seqInfoList = NIL; - GetDependentSequencesWithRelation(relationId, &seqInfoList, 0); + GetDependentSequencesWithRelation(relationId, &seqInfoList, 0, DEPENDENCY_AUTO); EnsureDistributedSequencesHaveOneType(relationId, seqInfoList); } @@ -1608,6 +1608,8 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, { Oid parentRelationId = InvalidOid; + ErrorIfTableHasUnsupportedIdentityColumn(relationId); + EnsureLocalTableEmptyIfNecessary(relationId, distributionMethod); /* user really wants triggers? */ diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 735449973..d09cacc19 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -370,7 +370,7 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) bool creatingShellTableOnRemoteNode = true; List *tableDDLCommands = GetFullTableCreationCommands(relationId, WORKER_NEXTVAL_SEQUENCE_DEFAULTS, - INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS, + INCLUDE_IDENTITY, creatingShellTableOnRemoteNode); TableDDLCommand *tableDDLCommand = NULL; foreach_ptr(tableDDLCommand, tableDDLCommands) diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index e8c217bb5..f1757bb62 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -33,7 +33,8 @@ /* Local functions forward declarations for helper functions */ static bool OptionsSpecifyOwnedBy(List *optionList, Oid *ownedByTableId); -static Oid SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress); +static Oid SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char + depType); static List * FilterDistributedSequences(GrantStmt *stmt); @@ -183,7 +184,7 @@ ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList, char *columnName = NameStr(attributeForm->attname); List *columnOwnedSequences = - getOwnedSequences_internal(relationId, attributeIndex + 1, 0); + getOwnedSequences_internal(relationId, attributeIndex + 1, DEPENDENCY_AUTO); if (attributeForm->atthasdef && list_length(columnOwnedSequences) == 0) { @@ -453,21 +454,22 @@ PreprocessAlterSequenceStmt(Node *node, const char *queryString, /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); + /* We have already asserted that we have exactly 1 address in the addresses. */ + ObjectAddress *address = linitial(addresses); + /* error out if the sequence is distributed */ - if (IsAnyObjectDistributed(addresses)) + if (IsAnyObjectDistributed(addresses) || SequenceUsedInDistributedTable(address, + DEPENDENCY_INTERNAL)) { ereport(ERROR, (errmsg( "Altering a distributed sequence is currently not supported."))); } - /* We have already asserted that we have exactly 1 address in the addresses. */ - ObjectAddress *address = linitial(addresses); - /* * error out if the sequence is used in a distributed table * and this is an ALTER SEQUENCE .. AS .. statement */ - Oid citusTableId = SequenceUsedInDistributedTable(address); + Oid citusTableId = SequenceUsedInDistributedTable(address, DEPENDENCY_AUTO); if (citusTableId != InvalidOid) { List *options = stmt->options; @@ -497,16 +499,19 @@ PreprocessAlterSequenceStmt(Node *node, const char *queryString, * SequenceUsedInDistributedTable returns true if the argument sequence * is used as the default value of a column in a distributed table. * Returns false otherwise + * See DependencyType for the possible values of depType. + * We use DEPENDENCY_INTERNAL for sequences created by identity column. + * DEPENDENCY_AUTO for regular sequences. */ static Oid -SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress) +SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char depType) { List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid citusTableId = InvalidOid; foreach_oid(citusTableId, citusTableIdList) { List *seqInfoList = NIL; - GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0); + GetDependentSequencesWithRelation(citusTableId, &seqInfoList, 0, depType); SequenceInfo *seqInfo = NULL; foreach_ptr(seqInfo, seqInfoList) { diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 39a652f10..f1d533ae9 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -1378,29 +1378,6 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, } } - /* - * We check for ADD COLUMN .. GENERATED .. AS IDENTITY expr - * since it uses a sequence as an internal dependency - * we should deparse the statement - */ - constraint = NULL; - foreach_ptr(constraint, columnConstraints) - { - if (constraint->contype == CONSTR_IDENTITY) - { - deparseAT = true; - useInitialDDLCommandString = false; - - /* - * Since we don't support constraints for AT_AddColumn - * we have to set is_not_null to true explicitly for identity columns - */ - ColumnDef *newColDef = copyObject(columnDefinition); - newColDef->constraints = NULL; - newColDef->is_not_null = true; - newCmd->def = (Node *) newColDef; - } - } /* * We check for ADD COLUMN .. SERIAL pseudo-type @@ -2539,34 +2516,6 @@ PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) } } } - - /* - * We check for ADD COLUMN .. GENERATED AS IDENTITY expr - * since it uses a seqeunce as an internal dependency - */ - constraint = NULL; - foreach_ptr(constraint, columnConstraints) - { - if (constraint->contype == CONSTR_IDENTITY) - { - AttrNumber attnum = get_attnum(relationId, - columnDefinition->colname); - bool missing_ok = false; - Oid seqOid = getIdentitySequence(relationId, attnum, missing_ok); - - if (ShouldSyncTableMetadata(relationId)) - { - needMetadataSyncForNewSequences = true; - alterTableDefaultNextvalCmd = - GetAddColumnWithNextvalDefaultCmd(seqOid, - relationId, - columnDefinition - ->colname, - columnDefinition - ->typeName); - } - } - } } /* * We check for ALTER COLUMN .. SET DEFAULT nextval('user_defined_seq') @@ -3222,6 +3171,17 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) { if (columnConstraint->contype == CONSTR_IDENTITY) { + /* + * We currently don't support adding an identity column for an MX table + */ + if (ShouldSyncTableMetadata(relationId)) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot execute ADD COLUMN commands involving identity" + " columns when metadata is synchronized to workers"))); + } + /* * Currently we don't support backfilling the new identity column with default values * if the table is not empty @@ -3352,7 +3312,8 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) */ AttrNumber attnum = get_attnum(relationId, command->name); List *seqInfoList = NIL; - GetDependentSequencesWithRelation(relationId, &seqInfoList, attnum); + GetDependentSequencesWithRelation(relationId, &seqInfoList, attnum, + DEPENDENCY_AUTO); if (seqInfoList != NIL) { ereport(ERROR, (errmsg("cannot execute ALTER COLUMN TYPE .. command " @@ -4011,3 +3972,59 @@ MakeNameListFromRangeVar(const RangeVar *rel) return list_make1(makeString(rel->relname)); } } + + +/* + * ErrorIfTableHasUnsupportedIdentityColumn errors out if the given table has any identity column other than bigint identity column. + */ +void +ErrorIfTableHasUnsupportedIdentityColumn(Oid relationId) +{ + Relation relation = relation_open(relationId, AccessShareLock); + TupleDesc tupleDescriptor = RelationGetDescr(relation); + + for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; + attributeIndex++) + { + Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); + + if (attributeForm->attidentity && attributeForm->atttypid != INT8OID) + { + char *qualifiedRelationName = generate_qualified_relation_name(relationId); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot complete operation on %s with smallint/int identity column", + qualifiedRelationName), + errhint( + "Use bigint identity column instead."))); + } + } + + relation_close(relation, NoLock); +} + + +/* + * ErrorIfTableHasIdentityColumn errors out if the given table has identity column + */ +void +ErrorIfTableHasIdentityColumn(Oid relationId) +{ + Relation relation = relation_open(relationId, AccessShareLock); + TupleDesc tupleDescriptor = RelationGetDescr(relation); + + for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; + attributeIndex++) + { + Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); + + if (attributeForm->attidentity) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot complete operation on a table with identity column"))); + } + } + + relation_close(relation, NoLock); +} diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index ada77b098..05e483766 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -304,10 +304,7 @@ pg_get_sequencedef(Oid sequenceRelationId) * When it's WORKER_NEXTVAL_SEQUENCE_DEFAULTS, the function creates the DEFAULT * clause using worker_nextval('sequence') and not nextval('sequence') * When IncludeIdentities is NO_IDENTITY, the function does not include identity column - * specifications. When it's INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS, the function - * uses sequences and set them as default values for identity columns by using exactly - * the same approach with worker_nextval('sequence') & nextval('sequence') logic - * desribed above. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. + * specifications. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. */ char * pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults @@ -403,26 +400,9 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults Oid seqOid = getIdentitySequence(RelationGetRelid(relation), attributeForm->attnum, missing_ok); - char *sequenceName = generate_qualified_relation_name(seqOid); - - if (includeIdentityDefaults == INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS) - { - if (pg_get_sequencedef(seqOid)->seqtypid != INT8OID) - { - appendStringInfo(&buffer, - " DEFAULT worker_nextval(%s::regclass)", - quote_literal_cstr(sequenceName)); - } - else - { - appendStringInfo(&buffer, " DEFAULT nextval(%s::regclass)", - quote_literal_cstr(sequenceName)); - } - } - else if (includeIdentityDefaults == INCLUDE_IDENTITY) + if (includeIdentityDefaults == INCLUDE_IDENTITY) { Form_pg_sequence pgSequenceForm = pg_get_sequencedef(seqOid); - uint64 sequenceStart = nextval_internal(seqOid, false); char *sequenceDef = psprintf( " GENERATED %s AS IDENTITY (INCREMENT BY " INT64_FORMAT \ " MINVALUE " INT64_FORMAT " MAXVALUE " @@ -433,7 +413,8 @@ pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults "ALWAYS" : "BY DEFAULT", pgSequenceForm->seqincrement, pgSequenceForm->seqmin, - pgSequenceForm->seqmax, sequenceStart, + pgSequenceForm->seqmax, + pgSequenceForm->seqstart, pgSequenceForm->seqcache, pgSequenceForm->seqcycle ? "" : "NO "); @@ -1391,7 +1372,7 @@ convert_aclright_to_string(int aclright) /* * contain_nextval_expression_walker walks over expression tree and returns - * true if it contains call to 'nextval' function. + * true if it contains call to 'nextval' function or it has an identity column. */ bool contain_nextval_expression_walker(Node *node, void *context) @@ -1401,6 +1382,13 @@ contain_nextval_expression_walker(Node *node, void *context) return false; } + /* check if the node contains an identity column */ + if (IsA(node, NextValueExpr)) + { + return true; + } + + /* check if the node contains call to 'nextval' */ if (IsA(node, FuncExpr)) { FuncExpr *funcExpr = (FuncExpr *) node; diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index a67c8fed0..dffeb482a 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -1834,7 +1834,7 @@ static List * GetRelationSequenceDependencyList(Oid relationId) { List *seqInfoList = NIL; - GetDependentSequencesWithRelation(relationId, &seqInfoList, 0); + GetDependentSequencesWithRelation(relationId, &seqInfoList, 0, DEPENDENCY_AUTO); List *seqIdList = NIL; SequenceInfo *seqInfo = NULL; diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 6a5840f78..08c0de255 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1586,10 +1586,13 @@ GetAttributeTypeOid(Oid relationId, AttrNumber attnum) * For both cases, we use the intermediate AttrDefault object from pg_depend. * If attnum is specified, we only return the sequences related to that * attribute of the relationId. + * See DependencyType for the possible values of depType. + * We use DEPENDENCY_INTERNAL for sequences created by identity column. + * DEPENDENCY_AUTO for regular sequences. */ void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, - AttrNumber attnum) + AttrNumber attnum, char depType) { Assert(*seqInfoList == NIL); @@ -1626,7 +1629,7 @@ GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, if (deprec->classid == AttrDefaultRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && - deprec->deptype == DEPENDENCY_AUTO) + deprec->deptype == depType) { /* * We are going to generate corresponding SequenceInfo @@ -1635,8 +1638,7 @@ GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, attrdefResult = lappend_oid(attrdefResult, deprec->objid); attrdefAttnumResult = lappend_int(attrdefAttnumResult, deprec->refobjsubid); } - else if ((deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == - DEPENDENCY_INTERNAL) && + else if (deprec->deptype == depType && deprec->refobjsubid != 0 && deprec->classid == RelationRelationId && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) @@ -1883,6 +1885,53 @@ SequenceDependencyCommandList(Oid relationId) } +/* + * IdentitySequenceDependencyCommandList generate a command to execute + * a UDF (WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES) on workers to modify the identity + * columns min/max values to produce unique values on workers. + */ +List * +IdentitySequenceDependencyCommandList(Oid targetRelationId) +{ + List *commandList = NIL; + + Relation relation = relation_open(targetRelationId, AccessShareLock); + TupleDesc tupleDescriptor = RelationGetDescr(relation); + + bool tableHasIdentityColumn = false; + for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; + attributeIndex++) + { + Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); + + if (attributeForm->attidentity) + { + tableHasIdentityColumn = true; + break; + } + } + + relation_close(relation, NoLock); + + if (tableHasIdentityColumn) + { + StringInfo stringInfo = makeStringInfo(); + char *tableName = generate_qualified_relation_name(targetRelationId); + + appendStringInfo(stringInfo, + WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES, + quote_literal_cstr(tableName)); + + + commandList = lappend(commandList, + makeTableDDLCommandString( + stringInfo->data)); + } + + return commandList; +} + + /* * CreateSequenceDependencyCommand generates a query string for calling * worker_record_sequence_dependency on the worker to recreate a sequence->table @@ -2605,8 +2654,7 @@ CreateShellTableOnWorkers(Oid relationId) List *commandList = list_make1(DISABLE_DDL_PROPAGATION); IncludeSequenceDefaults includeSequenceDefaults = WORKER_NEXTVAL_SEQUENCE_DEFAULTS; - IncludeIdentities includeIdentityDefaults = - INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS; + IncludeIdentities includeIdentityDefaults = INCLUDE_IDENTITY; bool creatingShellTableOnRemoteNode = true; List *tableDDLCommands = GetFullTableCreationCommands(relationId, diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index 172a2a303..dca9906a6 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -461,10 +461,7 @@ ResolveRelationId(text *relationName, bool missingOk) * definition, optional column storage and statistics definitions, and index * constraint and trigger definitions. * When IncludeIdentities is NO_IDENTITY, the function does not include identity column - * specifications. When it's INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS, the function - * uses sequences and set them as default values for identity columns by using exactly - * the same approach with worker_nextval('sequence') & nextval('sequence') logic - * desribed above. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. + * specifications. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. */ List * GetFullTableCreationCommands(Oid relationId, @@ -500,6 +497,15 @@ GetFullTableCreationCommands(Oid relationId, tableDDLEventList = lappend(tableDDLEventList, truncateTriggerCommand); } + + /* + * For identity column sequences, we only need to modify + * their min/max values to produce unique values on the worker nodes. + */ + List *identitySequenceDependencyCommandList = + IdentitySequenceDependencyCommandList(relationId); + tableDDLEventList = list_concat(tableDDLEventList, + identitySequenceDependencyCommandList); } tableDDLEventList = list_concat(tableDDLEventList, postLoadCreationCommandList); diff --git a/src/backend/distributed/sql/citus--11.2-1--11.2-2.sql b/src/backend/distributed/sql/citus--11.2-1--11.2-2.sql new file mode 100644 index 000000000..574af7099 --- /dev/null +++ b/src/backend/distributed/sql/citus--11.2-1--11.2-2.sql @@ -0,0 +1,5 @@ +-- citus--11.2-1--11.2-2 +-- Since we backported the UDF below from version 11.3, +-- the version portion of this file does not match with +-- the version of the included file. +#include "udfs/worker_adjust_identity_column_seq_ranges/11.3-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--11.2-2--11.2-1.sql b/src/backend/distributed/sql/downgrades/citus--11.2-2--11.2-1.sql new file mode 100644 index 000000000..92ba854da --- /dev/null +++ b/src/backend/distributed/sql/downgrades/citus--11.2-2--11.2-1.sql @@ -0,0 +1,2 @@ +-- citus--11.2-2--11.2-1 +DROP FUNCTION IF EXISTS pg_catalog.worker_adjust_identity_column_seq_ranges(regclass); diff --git a/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/11.3-1.sql b/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/11.3-1.sql new file mode 100644 index 000000000..aecc43704 --- /dev/null +++ b/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/11.3-1.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) + RETURNS VOID + LANGUAGE C STRICT + AS 'MODULE_PATHNAME', $$worker_adjust_identity_column_seq_ranges$$; +COMMENT ON FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) + IS 'modify identity column seq ranges to produce globally unique values'; + diff --git a/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/latest.sql b/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/latest.sql new file mode 100644 index 000000000..aecc43704 --- /dev/null +++ b/src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/latest.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) + RETURNS VOID + LANGUAGE C STRICT + AS 'MODULE_PATHNAME', $$worker_adjust_identity_column_seq_ranges$$; +COMMENT ON FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) + IS 'modify identity column seq ranges to produce globally unique values'; + diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index d563c443b..11fdda287 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -70,6 +70,7 @@ static void AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequence PG_FUNCTION_INFO_V1(worker_apply_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_inter_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_sequence_command); +PG_FUNCTION_INFO_V1(worker_adjust_identity_column_seq_ranges); PG_FUNCTION_INFO_V1(worker_append_table_to_shard); PG_FUNCTION_INFO_V1(worker_nextval); @@ -133,6 +134,60 @@ worker_apply_inter_shard_ddl_command(PG_FUNCTION_ARGS) } +/* + * worker_adjust_identity_column_seq_ranges takes a table oid, runs an ALTER SEQUENCE statement + * for each identity column to adjust the minvalue and maxvalue of the sequence owned by + * identity column such that the sequence creates globally unique values. + * We use table oid instead of sequence name to avoid any potential conflicts between sequences of different tables. This way, we can safely iterate through identity columns on a specific table without any issues. While this may introduce a small amount of business logic to workers, it's a much safer approach overall. + */ +Datum +worker_adjust_identity_column_seq_ranges(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Oid tableRelationId = PG_GETARG_OID(0); + + EnsureTableOwner(tableRelationId); + + Relation tableRelation = relation_open(tableRelationId, AccessShareLock); + TupleDesc tableTupleDesc = RelationGetDescr(tableRelation); + + bool missingSequenceOk = false; + + for (int attributeIndex = 0; attributeIndex < tableTupleDesc->natts; + attributeIndex++) + { + Form_pg_attribute attributeForm = TupleDescAttr(tableTupleDesc, + attributeIndex); + + /* skip dropped columns */ + if (attributeForm->attisdropped) + { + continue; + } + + if (attributeForm->attidentity) + { + Oid sequenceOid = getIdentitySequence(tableRelationId, + attributeForm->attnum, + missingSequenceOk); + + Oid sequenceSchemaOid = get_rel_namespace(sequenceOid); + char *sequenceSchemaName = get_namespace_name(sequenceSchemaOid); + char *sequenceName = get_rel_name(sequenceOid); + Oid sequenceTypeId = pg_get_sequencedef(sequenceOid)->seqtypid; + + AlterSequenceMinMax(sequenceOid, sequenceSchemaName, sequenceName, + sequenceTypeId); + } + } + + relation_close(tableRelation, NoLock); + + PG_RETURN_VOID(); +} + + /* * worker_apply_sequence_command takes a CREATE SEQUENCE command string, runs the * CREATE SEQUENCE command then creates and runs an ALTER SEQUENCE statement diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index c3ec4fafb..e9eab1c35 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -566,6 +566,9 @@ extern bool ConstrTypeCitusCanDefaultName(ConstrType constrType); extern char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, bool missingTableOk); +extern void ErrorIfTableHasUnsupportedIdentityColumn(Oid relationId); +extern void ErrorIfTableHasIdentityColumn(Oid relationId); + /* text_search.c - forward declarations */ extern List * GetCreateTextSearchConfigStatements(const ObjectAddress *address); extern List * GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); diff --git a/src/include/distributed/coordinator_protocol.h b/src/include/distributed/coordinator_protocol.h index 1444bff91..ab3264e5d 100644 --- a/src/include/distributed/coordinator_protocol.h +++ b/src/include/distributed/coordinator_protocol.h @@ -124,8 +124,7 @@ typedef enum IncludeSequenceDefaults typedef enum IncludeIdentities { NO_IDENTITY = 0, /* don't include identities */ - INCLUDE_IDENTITY_AS_SEQUENCE_DEFAULTS = 1, /* include identities as sequences */ - INCLUDE_IDENTITY = 2 /* include identities as-is*/ + INCLUDE_IDENTITY = 1 /* include identities as-is*/ } IncludeIdentities; diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index e06b5268f..11140beff 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -101,11 +101,12 @@ extern void SyncNodeMetadataToNodesMain(Datum main_arg); extern void SignalMetadataSyncDaemon(Oid database, int sig); extern bool ShouldInitiateMetadataSync(bool *lockFailure); extern List * SequenceDependencyCommandList(Oid relationId); +extern List * IdentitySequenceDependencyCommandList(Oid targetRelationId); extern List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName); extern List * GetSequencesFromAttrDef(Oid attrdefOid); extern void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, - AttrNumber attnum); + AttrNumber attnum, char depType); extern List * GetDependentFunctionsWithRelation(Oid relationId); extern Oid GetAttributeTypeOid(Oid relationId, AttrNumber attnum); extern void SetLocalEnableMetadataSync(bool state); @@ -146,6 +147,8 @@ extern void SyncDeleteColocationGroupToNodes(uint32 colocationId); "placementid = EXCLUDED.placementid" #define METADATA_SYNC_CHANNEL "metadata_sync" +#define WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES \ + "SELECT pg_catalog.worker_adjust_identity_column_seq_ranges(%s)" /* controlled via GUC */ extern char *EnableManualMetadataChangesForUser; diff --git a/src/test/regress/expected/generated_identity.out b/src/test/regress/expected/generated_identity.out index 23a87af3d..865012af0 100644 --- a/src/test/regress/expected/generated_identity.out +++ b/src/test/regress/expected/generated_identity.out @@ -1,525 +1,431 @@ +-- This test file has an alternative output because of error messages vary for PG13 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int <= 13 AS server_version_le_13; + server_version_le_13 +--------------------------------------------------------------------- + f +(1 row) + CREATE SCHEMA generated_identities; SET search_path TO generated_identities; SET client_min_messages to ERROR; +SET citus.shard_replication_factor TO 1; SELECT 1 from citus_add_node('localhost', :master_port, groupId=>0); ?column? --------------------------------------------------------------------- 1 (1 row) -DROP TABLE IF EXISTS generated_identities_test; --- create a partitioned table for testing. -CREATE TABLE generated_identities_test ( - a int CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY, - b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), - c smallint GENERATED BY DEFAULT AS IDENTITY, - d serial, - e bigserial, - f smallserial, - g int -) -PARTITION BY RANGE (a); -CREATE TABLE generated_identities_test_1_5 PARTITION OF generated_identities_test FOR VALUES FROM (1) TO (5); -CREATE TABLE generated_identities_test_5_50 PARTITION OF generated_identities_test FOR VALUES FROM (5) TO (50); --- local tables -SELECT citus_add_local_table_to_metadata('generated_identities_test'); +-- smallint identity column can not be distributed +CREATE TABLE smallint_identity_column ( + a smallint GENERATED BY DEFAULT AS IDENTITY +); +SELECT create_distributed_table('smallint_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.smallint_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_distributed_table_concurrently('smallint_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.smallint_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_reference_table('smallint_identity_column'); +ERROR: cannot complete operation on a table with identity column +SELECT citus_add_local_table_to_metadata('smallint_identity_column'); citus_add_local_table_to_metadata --------------------------------------------------------------------- (1 row) -\d generated_identities_test - Partitioned table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - a | integer | | not null | generated by default as identity - b | bigint | | not null | generated always as identity - c | smallint | | not null | generated by default as identity - d | integer | | not null | nextval('generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities_test_e_seq'::regclass) - f | smallint | | not null | nextval('generated_identities_test_f_seq'::regclass) - g | integer | | | -Partition key: RANGE (a) -Number of partitions: 2 (Use \d+ to list them.) - -\c - - - :worker_1_port -\d generated_identities.generated_identities_test - Partitioned table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - a | integer | | not null | worker_nextval('generated_identities.generated_identities_test_a_seq'::regclass) - b | bigint | | not null | nextval('generated_identities.generated_identities_test_b_seq'::regclass) - c | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_c_seq'::regclass) - d | integer | | not null | worker_nextval('generated_identities.generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities.generated_identities_test_e_seq'::regclass) - f | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_f_seq'::regclass) - g | integer | | | -Partition key: RANGE (a) -Number of partitions: 2 (Use \d+ to list them.) - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -SELECT undistribute_table('generated_identities_test'); - undistribute_table +DROP TABLE smallint_identity_column; +-- int identity column can not be distributed +CREATE TABLE int_identity_column ( + a int GENERATED BY DEFAULT AS IDENTITY +); +SELECT create_distributed_table('int_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.int_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_distributed_table_concurrently('int_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.int_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_reference_table('int_identity_column'); +ERROR: cannot complete operation on a table with identity column +SELECT citus_add_local_table_to_metadata('int_identity_column'); + citus_add_local_table_to_metadata --------------------------------------------------------------------- (1 row) -SELECT citus_remove_node('localhost', :master_port); - citus_remove_node +DROP TABLE int_identity_column; +RESET citus.shard_replication_factor; +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT citus_add_local_table_to_metadata('bigint_identity_column'); + citus_add_local_table_to_metadata --------------------------------------------------------------------- (1 row) -SELECT create_distributed_table('generated_identities_test', 'a'); +DROP TABLE bigint_identity_column; +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT create_distributed_table('bigint_identity_column', 'a'); create_distributed_table --------------------------------------------------------------------- (1 row) -\d generated_identities_test - Partitioned table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default +\d bigint_identity_column + Table "generated_identities.bigint_identity_column" + Column | Type | Collation | Nullable | Default --------------------------------------------------------------------- - a | integer | | not null | generated by default as identity - b | bigint | | not null | generated always as identity - c | smallint | | not null | generated by default as identity - d | integer | | not null | nextval('generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities_test_e_seq'::regclass) - f | smallint | | not null | nextval('generated_identities_test_f_seq'::regclass) - g | integer | | | -Partition key: RANGE (a) -Number of partitions: 2 (Use \d+ to list them.) + a | bigint | | not null | generated by default as identity + b | integer | | | \c - - - :worker_1_port -\d generated_identities.generated_identities_test - Partitioned table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO bigint_identity_column (b) +SELECT s FROM generate_series(1,10) s; +\d generated_identities.bigint_identity_column + Table "generated_identities.bigint_identity_column" + Column | Type | Collation | Nullable | Default --------------------------------------------------------------------- - a | integer | | not null | worker_nextval('generated_identities.generated_identities_test_a_seq'::regclass) - b | bigint | | not null | nextval('generated_identities.generated_identities_test_b_seq'::regclass) - c | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_c_seq'::regclass) - d | integer | | not null | worker_nextval('generated_identities.generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities.generated_identities_test_e_seq'::regclass) - f | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_f_seq'::regclass) - g | integer | | | -Partition key: RANGE (a) -Number of partitions: 2 (Use \d+ to list them.) + a | bigint | | not null | generated by default as identity + b | integer | | | \c - - - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; -insert into generated_identities_test (g) values (1); -insert into generated_identities_test (g) SELECT 2; -INSERT INTO generated_identities_test (g) +INSERT INTO bigint_identity_column (b) +SELECT s FROM generate_series(11,20) s; +SELECT * FROM bigint_identity_column ORDER BY B ASC; + a | b +--------------------------------------------------------------------- + 3940649673949185 | 1 + 3940649673949186 | 2 + 3940649673949187 | 3 + 3940649673949188 | 4 + 3940649673949189 | 5 + 3940649673949190 | 6 + 3940649673949191 | 7 + 3940649673949192 | 8 + 3940649673949193 | 9 + 3940649673949194 | 10 + 1 | 11 + 2 | 12 + 3 | 13 + 4 | 14 + 5 | 15 + 6 | 16 + 7 | 17 + 8 | 18 + 9 | 19 + 10 | 20 +(20 rows) + +-- table with identity column cannot be altered. +SELECT alter_distributed_table('bigint_identity_column', 'b'); +ERROR: cannot complete operation on a table with identity column +-- table with identity column cannot be undistributed. +SELECT undistribute_table('bigint_identity_column'); +ERROR: cannot complete operation on a table with identity column +DROP TABLE bigint_identity_column; +-- create a partitioned table for testing. +CREATE TABLE partitioned_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE partitioned_table_1_50 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); +CREATE TABLE partitioned_table_50_500 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); +SELECT create_distributed_table('partitioned_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\d partitioned_table + Partitioned table "generated_identities.partitioned_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Partition key: RANGE (c) +Number of partitions: 2 (Use \d+ to list them.) + +\c - - - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +\d generated_identities.partitioned_table + Partitioned table "generated_identities.partitioned_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Partition key: RANGE (c) +Number of partitions: 2 (Use \d+ to list them.) + +insert into partitioned_table (c) values (1); +insert into partitioned_table (c) SELECT 2; +INSERT INTO partitioned_table (c) SELECT s FROM generate_series(3,7) s; -SELECT * FROM generated_identities_test ORDER BY 1; - a | b | c | d | e | f | g ---------------------------------------------------------------------- - 1 | 10 | 1 | 1 | 1 | 1 | 1 - 2 | 20 | 2 | 2 | 2 | 2 | 2 - 3 | 30 | 3 | 3 | 3 | 3 | 3 - 4 | 40 | 4 | 4 | 4 | 4 | 4 - 5 | 50 | 5 | 5 | 5 | 5 | 5 - 6 | 60 | 6 | 6 | 6 | 6 | 6 - 7 | 70 | 7 | 7 | 7 | 7 | 7 -(7 rows) - -SELECT undistribute_table('generated_identities_test'); - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -SELECT * FROM generated_identities_test ORDER BY 1; - a | b | c | d | e | f | g ---------------------------------------------------------------------- - 1 | 10 | 1 | 1 | 1 | 1 | 1 - 2 | 20 | 2 | 2 | 2 | 2 | 2 - 3 | 30 | 3 | 3 | 3 | 3 | 3 - 4 | 40 | 4 | 4 | 4 | 4 | 4 - 5 | 50 | 5 | 5 | 5 | 5 | 5 - 6 | 60 | 6 | 6 | 6 | 6 | 6 - 7 | 70 | 7 | 7 | 7 | 7 | 7 -(7 rows) - -\d generated_identities_test - Partitioned table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - a | integer | | not null | generated by default as identity - b | bigint | | not null | generated always as identity - c | smallint | | not null | generated by default as identity - d | integer | | not null | nextval('generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities_test_e_seq'::regclass) - f | smallint | | not null | nextval('generated_identities_test_f_seq'::regclass) - g | integer | | | -Partition key: RANGE (a) -Number of partitions: 2 (Use \d+ to list them.) - -\c - - - :worker_1_port -\d generated_identities.generated_identities_test \c - - - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; -INSERT INTO generated_identities_test (g) -SELECT s FROM generate_series(8,10) s; -SELECT * FROM generated_identities_test ORDER BY 1; - a | b | c | d | e | f | g +INSERT INTO partitioned_table (c) +SELECT s FROM generate_series(10,20) s; +INSERT INTO partitioned_table (a,c) VALUES (998,998); +INSERT INTO partitioned_table (a,b,c) OVERRIDING SYSTEM VALUE VALUES (999,999,999); +SELECT * FROM partitioned_table ORDER BY c ASC; + a | b | c --------------------------------------------------------------------- - 1 | 10 | 1 | 1 | 1 | 1 | 1 - 2 | 20 | 2 | 2 | 2 | 2 | 2 - 3 | 30 | 3 | 3 | 3 | 3 | 3 - 4 | 40 | 4 | 4 | 4 | 4 | 4 - 5 | 50 | 5 | 5 | 5 | 5 | 5 - 6 | 60 | 6 | 6 | 6 | 6 | 6 - 7 | 70 | 7 | 7 | 7 | 7 | 7 - 8 | 80 | 8 | 8 | 8 | 8 | 8 - 9 | 90 | 9 | 9 | 9 | 9 | 9 - 10 | 100 | 10 | 10 | 10 | 10 | 10 -(10 rows) - --- distributed table -SELECT create_distributed_table('generated_identities_test', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 10 | 10 | 10 + 20 | 20 | 11 + 30 | 30 | 12 + 40 | 40 | 13 + 50 | 50 | 14 + 60 | 60 | 15 + 70 | 70 | 16 + 80 | 80 | 17 + 90 | 90 | 18 + 100 | 100 | 19 + 110 | 110 | 20 + 998 | 120 | 998 + 999 | 999 | 999 +(20 rows) -- alter table .. alter column .. add is unsupported -ALTER TABLE generated_identities_test ALTER COLUMN g ADD GENERATED ALWAYS AS IDENTITY; +ALTER TABLE partitioned_table ALTER COLUMN g ADD GENERATED ALWAYS AS IDENTITY; ERROR: alter table command is currently unsupported DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. -- alter table .. alter column is unsupported -ALTER TABLE generated_identities_test ALTER COLUMN b TYPE int; +ALTER TABLE partitioned_table ALTER COLUMN b TYPE int; ERROR: cannot execute ALTER COLUMN command involving identity column -SELECT alter_distributed_table('generated_identities_test', 'g'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT alter_distributed_table('generated_identities_test', 'b'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT alter_distributed_table('generated_identities_test', 'c'); - alter_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT undistribute_table('generated_identities_test'); - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -SELECT * FROM generated_identities_test ORDER BY g; - a | b | c | d | e | f | g ---------------------------------------------------------------------- - 1 | 10 | 1 | 1 | 1 | 1 | 1 - 2 | 20 | 2 | 2 | 2 | 2 | 2 - 3 | 30 | 3 | 3 | 3 | 3 | 3 - 4 | 40 | 4 | 4 | 4 | 4 | 4 - 5 | 50 | 5 | 5 | 5 | 5 | 5 - 6 | 60 | 6 | 6 | 6 | 6 | 6 - 7 | 70 | 7 | 7 | 7 | 7 | 7 - 8 | 80 | 8 | 8 | 8 | 8 | 8 - 9 | 90 | 9 | 9 | 9 | 9 | 9 - 10 | 100 | 10 | 10 | 10 | 10 | 10 -(10 rows) - --- reference table -DROP TABLE generated_identities_test; -CREATE TABLE generated_identities_test ( - a int GENERATED BY DEFAULT AS IDENTITY, - b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), - c smallint GENERATED BY DEFAULT AS IDENTITY, - d serial, - e bigserial, - f smallserial, - g int +DROP TABLE partitioned_table; +-- create a table for reference table testing. +CREATE TABLE reference_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10) UNIQUE, + c int ); -SELECT create_reference_table('generated_identities_test'); +SELECT create_reference_table('reference_table'); create_reference_table --------------------------------------------------------------------- (1 row) -\d generated_identities_test - Table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default +\d reference_table + Table "generated_identities.reference_table" + Column | Type | Collation | Nullable | Default --------------------------------------------------------------------- - a | integer | | not null | generated by default as identity - b | bigint | | not null | generated always as identity - c | smallint | | not null | generated by default as identity - d | integer | | not null | nextval('generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities_test_e_seq'::regclass) - f | smallint | | not null | nextval('generated_identities_test_f_seq'::regclass) - g | integer | | | + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Indexes: + "reference_table_b_key" UNIQUE CONSTRAINT, btree (b) \c - - - :worker_1_port -\d generated_identities.generated_identities_test - Table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default +SET search_path TO generated_identities; +\d generated_identities.reference_table + Table "generated_identities.reference_table" + Column | Type | Collation | Nullable | Default --------------------------------------------------------------------- - a | integer | | not null | worker_nextval('generated_identities.generated_identities_test_a_seq'::regclass) - b | bigint | | not null | nextval('generated_identities.generated_identities_test_b_seq'::regclass) - c | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_c_seq'::regclass) - d | integer | | not null | worker_nextval('generated_identities.generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities.generated_identities_test_e_seq'::regclass) - f | smallint | | not null | worker_nextval('generated_identities.generated_identities_test_f_seq'::regclass) - g | integer | | | + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Indexes: + "reference_table_b_key" UNIQUE CONSTRAINT, btree (b) + +INSERT INTO reference_table (c) +SELECT s FROM generate_series(1,10) s; +--on master +select * from reference_table; + a | b | c +--------------------------------------------------------------------- + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 3940649673949255 | 3940649673949255 | 8 + 3940649673949265 | 3940649673949265 | 9 + 3940649673949275 | 3940649673949275 | 10 +(10 rows) \c - - - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; -INSERT INTO generated_identities_test (g) +INSERT INTO reference_table (c) SELECT s FROM generate_series(11,20) s; -SELECT * FROM generated_identities_test ORDER BY g; - a | b | c | d | e | f | g +SELECT * FROM reference_table ORDER BY c ASC; + a | b | c --------------------------------------------------------------------- - 1 | 10 | 1 | 1 | 1 | 1 | 11 - 2 | 20 | 2 | 2 | 2 | 2 | 12 - 3 | 30 | 3 | 3 | 3 | 3 | 13 - 4 | 40 | 4 | 4 | 4 | 4 | 14 - 5 | 50 | 5 | 5 | 5 | 5 | 15 - 6 | 60 | 6 | 6 | 6 | 6 | 16 - 7 | 70 | 7 | 7 | 7 | 7 | 17 - 8 | 80 | 8 | 8 | 8 | 8 | 18 - 9 | 90 | 9 | 9 | 9 | 9 | 19 - 10 | 100 | 10 | 10 | 10 | 10 | 20 -(10 rows) + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 3940649673949255 | 3940649673949255 | 8 + 3940649673949265 | 3940649673949265 | 9 + 3940649673949275 | 3940649673949275 | 10 + 10 | 10 | 11 + 20 | 20 | 12 + 30 | 30 | 13 + 40 | 40 | 14 + 50 | 50 | 15 + 60 | 60 | 16 + 70 | 70 | 17 + 80 | 80 | 18 + 90 | 90 | 19 + 100 | 100 | 20 +(20 rows) -SELECT undistribute_table('generated_identities_test'); - undistribute_table +DROP TABLE reference_table; +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +); +-- https://github.com/citusdata/citus/issues/6694 +CREATE USER identity_test_user; +GRANT INSERT ON color TO identity_test_user; +GRANT USAGE ON SCHEMA generated_identities TO identity_test_user; +SET ROLE identity_test_user; +SELECT create_distributed_table('color', 'color_id'); +ERROR: must be owner of table color +SET ROLE postgres; +SET citus.shard_replication_factor TO 1; +SELECT create_distributed_table_concurrently('color', 'color_id'); + create_distributed_table_concurrently --------------------------------------------------------------------- (1 row) -\d generated_identities_test - Table "generated_identities.generated_identities_test" - Column | Type | Collation | Nullable | Default +RESET citus.shard_replication_factor; +\c - identity_test_user - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO color(color_name) VALUES ('Blue'); +\c - postgres - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +SET citus.next_shard_id TO 12400000; +DROP TABLE Color; +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +) USING columnar; +SELECT create_distributed_table('color', 'color_id'); + create_distributed_table --------------------------------------------------------------------- - a | integer | | not null | generated by default as identity - b | bigint | | not null | generated always as identity - c | smallint | | not null | generated by default as identity - d | integer | | not null | nextval('generated_identities_test_d_seq'::regclass) - e | bigint | | not null | nextval('generated_identities_test_e_seq'::regclass) - f | smallint | | not null | nextval('generated_identities_test_f_seq'::regclass) - g | integer | | | + +(1 row) + +INSERT INTO color(color_name) VALUES ('Blue'); +\d+ color + Table "generated_identities.color" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + color_id | bigint | | not null | generated always as identity | plain | | + color_name | character varying | | not null | | extended | | +Indexes: + "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) \c - - - :worker_1_port -\d generated_identities.generated_identities_test -\c - - - :master_port +SET search_path TO generated_identities; +\d+ color + Table "generated_identities.color" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + color_id | bigint | | not null | generated always as identity | plain | | + color_name | character varying | | not null | | extended | | +Indexes: + "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) + +INSERT INTO color(color_name) VALUES ('Red'); +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +ERROR: Altering a distributed sequence is currently not supported. +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +ERROR: cannot insert a non-DEFAULT value into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +ERROR: cannot insert a non-DEFAULT value into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +ERROR: duplicate key value violates unique constraint "color_color_id_key_12400000" +DETAIL: Key (color_id)=(1) already exists. +CONTEXT: while executing command on localhost:xxxxx +-- update null or custom value +UPDATE color SET color_id = NULL; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +UPDATE color SET color_id = 1; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +\c - postgres - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; -- alter table .. add column .. GENERATED .. AS IDENTITY -DROP TABLE IF EXISTS color; -CREATE TABLE color ( - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_name'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - ALTER TABLE color ADD COLUMN color_id BIGINT GENERATED ALWAYS AS IDENTITY; -INSERT INTO color(color_name) VALUES ('Red'); -ALTER TABLE color ADD COLUMN color_id_1 BIGINT GENERATED ALWAYS AS IDENTITY; -ERROR: Cannot add an identity column because the table is not empty -DROP TABLE color; --- insert data from workers -CREATE TABLE color ( - color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_id'); +ERROR: cannot execute ADD COLUMN commands involving identity columns when metadata is synchronized to workers +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +ERROR: Altering a distributed sequence is currently not supported. +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +ERROR: cannot insert a non-DEFAULT value into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +ERROR: cannot insert a non-DEFAULT value into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +ERROR: duplicate key value violates unique constraint "color_color_id_key_12400000" +DETAIL: Key (color_id)=(1) already exists. +CONTEXT: while executing command on localhost:xxxxx +-- update null or custom value +UPDATE color SET color_id = NULL; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +UPDATE color SET color_id = 1; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +DROP TABLE IF EXISTS test; +CREATE TABLE test (x int, y int, z bigint generated by default as identity); +SELECT create_distributed_table('test', 'x', colocate_with := 'none'); create_distributed_table --------------------------------------------------------------------- (1 row) -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -SELECT undistribute_table('color'); - undistribute_table +INSERT INTO test VALUES (1,2); +INSERT INTO test SELECT x, y FROM test WHERE x = 1; +SELECT * FROM test; + x | y | z --------------------------------------------------------------------- + 1 | 2 | 1 + 1 | 2 | 2 +(2 rows) -(1 row) - -SELECT create_distributed_table('color', 'color_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -INSERT INTO color(color_name) VALUES ('Red'); -SELECT count(*) from color; - count ---------------------------------------------------------------------- - 3 -(1 row) - --- modify sequence & alter table -DROP TABLE color; -CREATE TABLE color ( - color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -SELECT undistribute_table('color'); - undistribute_table ---------------------------------------------------------------------- - -(1 row) - -ALTER SEQUENCE color_color_id_seq RENAME TO myseq; -SELECT create_distributed_table('color', 'color_id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -\ds+ myseq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- - generated_identities | myseq | sequence | postgres | permanent | 8192 bytes | -(1 row) - -\ds+ color_color_id_seq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- -(0 rows) - -\d color - Table "generated_identities.color" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - color_id | bigint | | not null | generated always as identity - color_name | character varying | | not null | -Indexes: - "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -\ds+ myseq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- - generated_identities | myseq | sequence | postgres | permanent | 8192 bytes | -(1 row) - -\ds+ color_color_id_seq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- -(0 rows) - -\d color - Table "generated_identities.color" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - color_id | bigint | | not null | nextval('myseq'::regclass) - color_name | character varying | | not null | -Indexes: - "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) - -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -ALTER SEQUENCE myseq RENAME TO color_color_id_seq; -\ds+ myseq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- -(0 rows) - -\ds+ color_color_id_seq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- - generated_identities | color_color_id_seq | sequence | postgres | permanent | 8192 bytes | -(1 row) - -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -\ds+ myseq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- -(0 rows) - -\ds+ color_color_id_seq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- - generated_identities | color_color_id_seq | sequence | postgres | permanent | 8192 bytes | -(1 row) - -\d color - Table "generated_identities.color" - Column | Type | Collation | Nullable | Default ---------------------------------------------------------------------- - color_id | bigint | | not null | nextval('color_color_id_seq'::regclass) - color_name | character varying | | not null | -Indexes: - "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) - -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -SELECT alter_distributed_table('co23423lor', shard_count := 6); -ERROR: relation "co23423lor" does not exist -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; -\ds+ color_color_id_seq - List of relations - Schema | Name | Type | Owner | Persistence | Size | Description ---------------------------------------------------------------------- - generated_identities | color_color_id_seq | sequence | postgres | permanent | 8192 bytes | -(1 row) - -INSERT INTO color(color_name) VALUES ('Red'); -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; DROP SCHEMA generated_identities CASCADE; +DROP USER identity_test_user; diff --git a/src/test/regress/expected/generated_identity_0.out b/src/test/regress/expected/generated_identity_0.out new file mode 100644 index 000000000..1bff7f68f --- /dev/null +++ b/src/test/regress/expected/generated_identity_0.out @@ -0,0 +1,431 @@ +-- This test file has an alternative output because of error messages vary for PG13 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int <= 13 AS server_version_le_13; + server_version_le_13 +--------------------------------------------------------------------- + t +(1 row) + +CREATE SCHEMA generated_identities; +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +SET citus.shard_replication_factor TO 1; +SELECT 1 from citus_add_node('localhost', :master_port, groupId=>0); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- smallint identity column can not be distributed +CREATE TABLE smallint_identity_column ( + a smallint GENERATED BY DEFAULT AS IDENTITY +); +SELECT create_distributed_table('smallint_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.smallint_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_distributed_table_concurrently('smallint_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.smallint_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_reference_table('smallint_identity_column'); +ERROR: cannot complete operation on a table with identity column +SELECT citus_add_local_table_to_metadata('smallint_identity_column'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE smallint_identity_column; +-- int identity column can not be distributed +CREATE TABLE int_identity_column ( + a int GENERATED BY DEFAULT AS IDENTITY +); +SELECT create_distributed_table('int_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.int_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_distributed_table_concurrently('int_identity_column', 'a'); +ERROR: cannot complete operation on generated_identities.int_identity_column with smallint/int identity column +HINT: Use bigint identity column instead. +SELECT create_reference_table('int_identity_column'); +ERROR: cannot complete operation on a table with identity column +SELECT citus_add_local_table_to_metadata('int_identity_column'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE int_identity_column; +RESET citus.shard_replication_factor; +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT citus_add_local_table_to_metadata('bigint_identity_column'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE bigint_identity_column; +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT create_distributed_table('bigint_identity_column', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\d bigint_identity_column + Table "generated_identities.bigint_identity_column" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + +\c - - - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO bigint_identity_column (b) +SELECT s FROM generate_series(1,10) s; +\d generated_identities.bigint_identity_column + Table "generated_identities.bigint_identity_column" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | integer | | | + +\c - - - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO bigint_identity_column (b) +SELECT s FROM generate_series(11,20) s; +SELECT * FROM bigint_identity_column ORDER BY B ASC; + a | b +--------------------------------------------------------------------- + 3940649673949185 | 1 + 3940649673949186 | 2 + 3940649673949187 | 3 + 3940649673949188 | 4 + 3940649673949189 | 5 + 3940649673949190 | 6 + 3940649673949191 | 7 + 3940649673949192 | 8 + 3940649673949193 | 9 + 3940649673949194 | 10 + 1 | 11 + 2 | 12 + 3 | 13 + 4 | 14 + 5 | 15 + 6 | 16 + 7 | 17 + 8 | 18 + 9 | 19 + 10 | 20 +(20 rows) + +-- table with identity column cannot be altered. +SELECT alter_distributed_table('bigint_identity_column', 'b'); +ERROR: cannot complete operation on a table with identity column +-- table with identity column cannot be undistributed. +SELECT undistribute_table('bigint_identity_column'); +ERROR: cannot complete operation on a table with identity column +DROP TABLE bigint_identity_column; +-- create a partitioned table for testing. +CREATE TABLE partitioned_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE partitioned_table_1_50 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); +CREATE TABLE partitioned_table_50_500 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); +SELECT create_distributed_table('partitioned_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\d partitioned_table + Partitioned table "generated_identities.partitioned_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Partition key: RANGE (c) +Number of partitions: 2 (Use \d+ to list them.) + +\c - - - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +\d generated_identities.partitioned_table + Partitioned table "generated_identities.partitioned_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Partition key: RANGE (c) +Number of partitions: 2 (Use \d+ to list them.) + +insert into partitioned_table (c) values (1); +insert into partitioned_table (c) SELECT 2; +INSERT INTO partitioned_table (c) +SELECT s FROM generate_series(3,7) s; +\c - - - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO partitioned_table (c) +SELECT s FROM generate_series(10,20) s; +INSERT INTO partitioned_table (a,c) VALUES (998,998); +INSERT INTO partitioned_table (a,b,c) OVERRIDING SYSTEM VALUE VALUES (999,999,999); +SELECT * FROM partitioned_table ORDER BY c ASC; + a | b | c +--------------------------------------------------------------------- + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 10 | 10 | 10 + 20 | 20 | 11 + 30 | 30 | 12 + 40 | 40 | 13 + 50 | 50 | 14 + 60 | 60 | 15 + 70 | 70 | 16 + 80 | 80 | 17 + 90 | 90 | 18 + 100 | 100 | 19 + 110 | 110 | 20 + 998 | 120 | 998 + 999 | 999 | 999 +(20 rows) + +-- alter table .. alter column .. add is unsupported +ALTER TABLE partitioned_table ALTER COLUMN g ADD GENERATED ALWAYS AS IDENTITY; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported. +-- alter table .. alter column is unsupported +ALTER TABLE partitioned_table ALTER COLUMN b TYPE int; +ERROR: cannot execute ALTER COLUMN command involving identity column +DROP TABLE partitioned_table; +-- create a table for reference table testing. +CREATE TABLE reference_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10) UNIQUE, + c int +); +SELECT create_reference_table('reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +\d reference_table + Table "generated_identities.reference_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Indexes: + "reference_table_b_key" UNIQUE CONSTRAINT, btree (b) + +\c - - - :worker_1_port +SET search_path TO generated_identities; +\d generated_identities.reference_table + Table "generated_identities.reference_table" + Column | Type | Collation | Nullable | Default +--------------------------------------------------------------------- + a | bigint | | not null | generated by default as identity + b | bigint | | not null | generated always as identity + c | integer | | | +Indexes: + "reference_table_b_key" UNIQUE CONSTRAINT, btree (b) + +INSERT INTO reference_table (c) +SELECT s FROM generate_series(1,10) s; +--on master +select * from reference_table; + a | b | c +--------------------------------------------------------------------- + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 3940649673949255 | 3940649673949255 | 8 + 3940649673949265 | 3940649673949265 | 9 + 3940649673949275 | 3940649673949275 | 10 +(10 rows) + +\c - - - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO reference_table (c) +SELECT s FROM generate_series(11,20) s; +SELECT * FROM reference_table ORDER BY c ASC; + a | b | c +--------------------------------------------------------------------- + 3940649673949185 | 3940649673949185 | 1 + 3940649673949195 | 3940649673949195 | 2 + 3940649673949205 | 3940649673949205 | 3 + 3940649673949215 | 3940649673949215 | 4 + 3940649673949225 | 3940649673949225 | 5 + 3940649673949235 | 3940649673949235 | 6 + 3940649673949245 | 3940649673949245 | 7 + 3940649673949255 | 3940649673949255 | 8 + 3940649673949265 | 3940649673949265 | 9 + 3940649673949275 | 3940649673949275 | 10 + 10 | 10 | 11 + 20 | 20 | 12 + 30 | 30 | 13 + 40 | 40 | 14 + 50 | 50 | 15 + 60 | 60 | 16 + 70 | 70 | 17 + 80 | 80 | 18 + 90 | 90 | 19 + 100 | 100 | 20 +(20 rows) + +DROP TABLE reference_table; +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +); +-- https://github.com/citusdata/citus/issues/6694 +CREATE USER identity_test_user; +GRANT INSERT ON color TO identity_test_user; +GRANT USAGE ON SCHEMA generated_identities TO identity_test_user; +SET ROLE identity_test_user; +SELECT create_distributed_table('color', 'color_id'); +ERROR: must be owner of table color +SET ROLE postgres; +SET citus.shard_replication_factor TO 1; +SELECT create_distributed_table_concurrently('color', 'color_id'); + create_distributed_table_concurrently +--------------------------------------------------------------------- + +(1 row) + +RESET citus.shard_replication_factor; +\c - identity_test_user - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +INSERT INTO color(color_name) VALUES ('Blue'); +\c - postgres - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +SET citus.next_shard_id TO 12400000; +DROP TABLE Color; +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +) USING columnar; +SELECT create_distributed_table('color', 'color_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO color(color_name) VALUES ('Blue'); +\d+ color + Table "generated_identities.color" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + color_id | bigint | | not null | generated always as identity | plain | | + color_name | character varying | | not null | | extended | | +Indexes: + "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) + +\c - - - :worker_1_port +SET search_path TO generated_identities; +\d+ color + Table "generated_identities.color" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + color_id | bigint | | not null | generated always as identity | plain | | + color_name | character varying | | not null | | extended | | +Indexes: + "color_color_id_key" UNIQUE CONSTRAINT, btree (color_id) + +INSERT INTO color(color_name) VALUES ('Red'); +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +ERROR: Altering a distributed sequence is currently not supported. +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +ERROR: cannot insert into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +ERROR: cannot insert into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +ERROR: duplicate key value violates unique constraint "color_color_id_key_12400000" +DETAIL: Key (color_id)=(1) already exists. +CONTEXT: while executing command on localhost:xxxxx +-- update null or custom value +UPDATE color SET color_id = NULL; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +UPDATE color SET color_id = 1; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +\c - postgres - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +-- alter table .. add column .. GENERATED .. AS IDENTITY +ALTER TABLE color ADD COLUMN color_id BIGINT GENERATED ALWAYS AS IDENTITY; +ERROR: cannot execute ADD COLUMN commands involving identity columns when metadata is synchronized to workers +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +ERROR: Altering a distributed sequence is currently not supported. +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +ERROR: cannot insert into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +ERROR: cannot insert into column "color_id" +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +ERROR: duplicate key value violates unique constraint "color_color_id_key_12400000" +DETAIL: Key (color_id)=(1) already exists. +CONTEXT: while executing command on localhost:xxxxx +-- update null or custom value +UPDATE color SET color_id = NULL; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +UPDATE color SET color_id = 1; +ERROR: column "color_id" can only be updated to DEFAULT +DETAIL: Column "color_id" is an identity column defined as GENERATED ALWAYS. +DROP TABLE IF EXISTS test; +CREATE TABLE test (x int, y int, z bigint generated by default as identity); +SELECT create_distributed_table('test', 'x', colocate_with := 'none'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test VALUES (1,2); +INSERT INTO test SELECT x, y FROM test WHERE x = 1; +SELECT * FROM test; + x | y | z +--------------------------------------------------------------------- + 1 | 2 | 1 + 1 | 2 | 2 +(2 rows) + +DROP SCHEMA generated_identities CASCADE; +DROP USER identity_test_user; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index d47c6cf63..45bfbcca7 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1303,14 +1303,23 @@ SELECT * FROM multi_extension.print_extension_changes(); | type cluster_clock (38 rows) -DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version SHOW citus.version; citus.version --------------------------------------------------------------------- - 11.2.0 + 11.2.1 (1 row) +-- Snapshot of state at 11.2-2 +ALTER EXTENSION citus UPDATE TO '11.2-2'; +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + | function worker_adjust_identity_column_seq_ranges(regclass) void +(1 row) + +-- Test downgrade to 11.2-1 from 11.2-2 +ALTER EXTENSION citus UPDATE TO '11.2-1'; -- ensure no unexpected objects were created outside pg_catalog SELECT pgio.type, pgio.identity FROM pg_depend AS pgd, @@ -1326,6 +1335,7 @@ ORDER BY 1, 2; view | public.citus_tables (1 row) +DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- see incompatible version errors out RESET citus.enable_version_checks; RESET columnar.enable_version_checks; diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 7cd2f63c8..6328873bb 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -230,6 +230,7 @@ ORDER BY 1; function truncate_local_data_after_distributing_table(regclass) function undistribute_table(regclass,boolean) function update_distributed_table_colocation(regclass,text) + function worker_adjust_identity_column_seq_ranges(regclass) function worker_apply_inter_shard_ddl_command(bigint,text,bigint,text,text) function worker_apply_sequence_command(text) function worker_apply_sequence_command(text,regtype) @@ -318,5 +319,5 @@ ORDER BY 1; view citus_stat_statements view pg_dist_shard_placement view time_partitions -(310 rows) +(311 rows) diff --git a/src/test/regress/sql/generated_identity.sql b/src/test/regress/sql/generated_identity.sql index 004f45b40..c2980d0bd 100644 --- a/src/test/regress/sql/generated_identity.sql +++ b/src/test/regress/sql/generated_identity.sql @@ -1,266 +1,235 @@ +-- This test file has an alternative output because of error messages vary for PG13 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int <= 13 AS server_version_le_13; + CREATE SCHEMA generated_identities; SET search_path TO generated_identities; SET client_min_messages to ERROR; +SET citus.shard_replication_factor TO 1; SELECT 1 from citus_add_node('localhost', :master_port, groupId=>0); -DROP TABLE IF EXISTS generated_identities_test; - --- create a partitioned table for testing. -CREATE TABLE generated_identities_test ( - a int CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY, - b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), - c smallint GENERATED BY DEFAULT AS IDENTITY, - d serial, - e bigserial, - f smallserial, - g int -) -PARTITION BY RANGE (a); -CREATE TABLE generated_identities_test_1_5 PARTITION OF generated_identities_test FOR VALUES FROM (1) TO (5); -CREATE TABLE generated_identities_test_5_50 PARTITION OF generated_identities_test FOR VALUES FROM (5) TO (50); - --- local tables -SELECT citus_add_local_table_to_metadata('generated_identities_test'); - -\d generated_identities_test - -\c - - - :worker_1_port - -\d generated_identities.generated_identities_test - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -SELECT undistribute_table('generated_identities_test'); - -SELECT citus_remove_node('localhost', :master_port); - -SELECT create_distributed_table('generated_identities_test', 'a'); - -\d generated_identities_test - -\c - - - :worker_1_port - -\d generated_identities.generated_identities_test - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -insert into generated_identities_test (g) values (1); - -insert into generated_identities_test (g) SELECT 2; - -INSERT INTO generated_identities_test (g) -SELECT s FROM generate_series(3,7) s; - -SELECT * FROM generated_identities_test ORDER BY 1; - -SELECT undistribute_table('generated_identities_test'); - -SELECT * FROM generated_identities_test ORDER BY 1; - -\d generated_identities_test - -\c - - - :worker_1_port - -\d generated_identities.generated_identities_test - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -INSERT INTO generated_identities_test (g) -SELECT s FROM generate_series(8,10) s; - -SELECT * FROM generated_identities_test ORDER BY 1; - --- distributed table -SELECT create_distributed_table('generated_identities_test', 'a'); - --- alter table .. alter column .. add is unsupported -ALTER TABLE generated_identities_test ALTER COLUMN g ADD GENERATED ALWAYS AS IDENTITY; - --- alter table .. alter column is unsupported -ALTER TABLE generated_identities_test ALTER COLUMN b TYPE int; - -SELECT alter_distributed_table('generated_identities_test', 'g'); - -SELECT alter_distributed_table('generated_identities_test', 'b'); - -SELECT alter_distributed_table('generated_identities_test', 'c'); - -SELECT undistribute_table('generated_identities_test'); - -SELECT * FROM generated_identities_test ORDER BY g; - --- reference table - -DROP TABLE generated_identities_test; - -CREATE TABLE generated_identities_test ( - a int GENERATED BY DEFAULT AS IDENTITY, - b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), - c smallint GENERATED BY DEFAULT AS IDENTITY, - d serial, - e bigserial, - f smallserial, - g int +-- smallint identity column can not be distributed +CREATE TABLE smallint_identity_column ( + a smallint GENERATED BY DEFAULT AS IDENTITY ); +SELECT create_distributed_table('smallint_identity_column', 'a'); +SELECT create_distributed_table_concurrently('smallint_identity_column', 'a'); +SELECT create_reference_table('smallint_identity_column'); +SELECT citus_add_local_table_to_metadata('smallint_identity_column'); -SELECT create_reference_table('generated_identities_test'); +DROP TABLE smallint_identity_column; -\d generated_identities_test +-- int identity column can not be distributed +CREATE TABLE int_identity_column ( + a int GENERATED BY DEFAULT AS IDENTITY +); +SELECT create_distributed_table('int_identity_column', 'a'); +SELECT create_distributed_table_concurrently('int_identity_column', 'a'); +SELECT create_reference_table('int_identity_column'); +SELECT citus_add_local_table_to_metadata('int_identity_column'); +DROP TABLE int_identity_column; +RESET citus.shard_replication_factor; + + +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT citus_add_local_table_to_metadata('bigint_identity_column'); +DROP TABLE bigint_identity_column; + +CREATE TABLE bigint_identity_column ( + a bigint GENERATED BY DEFAULT AS IDENTITY, + b int +); +SELECT create_distributed_table('bigint_identity_column', 'a'); + +\d bigint_identity_column \c - - - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; -\d generated_identities.generated_identities_test +INSERT INTO bigint_identity_column (b) +SELECT s FROM generate_series(1,10) s; + +\d generated_identities.bigint_identity_column \c - - - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; -INSERT INTO generated_identities_test (g) +INSERT INTO bigint_identity_column (b) SELECT s FROM generate_series(11,20) s; -SELECT * FROM generated_identities_test ORDER BY g; +SELECT * FROM bigint_identity_column ORDER BY B ASC; -SELECT undistribute_table('generated_identities_test'); +-- table with identity column cannot be altered. +SELECT alter_distributed_table('bigint_identity_column', 'b'); -\d generated_identities_test +-- table with identity column cannot be undistributed. +SELECT undistribute_table('bigint_identity_column'); + +DROP TABLE bigint_identity_column; + +-- create a partitioned table for testing. +CREATE TABLE partitioned_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10), + c int +) +PARTITION BY RANGE (c); +CREATE TABLE partitioned_table_1_50 PARTITION OF partitioned_table FOR VALUES FROM (1) TO (50); +CREATE TABLE partitioned_table_50_500 PARTITION OF partitioned_table FOR VALUES FROM (50) TO (1000); + +SELECT create_distributed_table('partitioned_table', 'a'); + +\d partitioned_table \c - - - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; -\d generated_identities.generated_identities_test +\d generated_identities.partitioned_table + +insert into partitioned_table (c) values (1); + +insert into partitioned_table (c) SELECT 2; + +INSERT INTO partitioned_table (c) +SELECT s FROM generate_series(3,7) s; \c - - - :master_port SET search_path TO generated_identities; SET client_min_messages to ERROR; +INSERT INTO partitioned_table (c) +SELECT s FROM generate_series(10,20) s; + +INSERT INTO partitioned_table (a,c) VALUES (998,998); + +INSERT INTO partitioned_table (a,b,c) OVERRIDING SYSTEM VALUE VALUES (999,999,999); + +SELECT * FROM partitioned_table ORDER BY c ASC; + +-- alter table .. alter column .. add is unsupported +ALTER TABLE partitioned_table ALTER COLUMN g ADD GENERATED ALWAYS AS IDENTITY; + +-- alter table .. alter column is unsupported +ALTER TABLE partitioned_table ALTER COLUMN b TYPE int; + +DROP TABLE partitioned_table; + +-- create a table for reference table testing. +CREATE TABLE reference_table ( + a bigint CONSTRAINT myconname GENERATED BY DEFAULT AS IDENTITY (START WITH 10 INCREMENT BY 10), + b bigint GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 10) UNIQUE, + c int +); + +SELECT create_reference_table('reference_table'); + +\d reference_table + +\c - - - :worker_1_port +SET search_path TO generated_identities; + +\d generated_identities.reference_table + +INSERT INTO reference_table (c) +SELECT s FROM generate_series(1,10) s; + +--on master +select * from reference_table; + +\c - - - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; + +INSERT INTO reference_table (c) +SELECT s FROM generate_series(11,20) s; + +SELECT * FROM reference_table ORDER BY c ASC; + +DROP TABLE reference_table; + +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +); + +-- https://github.com/citusdata/citus/issues/6694 +CREATE USER identity_test_user; +GRANT INSERT ON color TO identity_test_user; +GRANT USAGE ON SCHEMA generated_identities TO identity_test_user; + +SET ROLE identity_test_user; +SELECT create_distributed_table('color', 'color_id'); + +SET ROLE postgres; +SET citus.shard_replication_factor TO 1; +SELECT create_distributed_table_concurrently('color', 'color_id'); +RESET citus.shard_replication_factor; + +\c - identity_test_user - :worker_1_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; + +INSERT INTO color(color_name) VALUES ('Blue'); + +\c - postgres - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; +SET citus.next_shard_id TO 12400000; + +DROP TABLE Color; +CREATE TABLE color ( + color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + color_name VARCHAR NOT NULL +) USING columnar; +SELECT create_distributed_table('color', 'color_id'); +INSERT INTO color(color_name) VALUES ('Blue'); +\d+ color + +\c - - - :worker_1_port +SET search_path TO generated_identities; +\d+ color +INSERT INTO color(color_name) VALUES ('Red'); +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +-- update null or custom value +UPDATE color SET color_id = NULL; +UPDATE color SET color_id = 1; + +\c - postgres - :master_port +SET search_path TO generated_identities; +SET client_min_messages to ERROR; + + -- alter table .. add column .. GENERATED .. AS IDENTITY -DROP TABLE IF EXISTS color; -CREATE TABLE color ( - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_name'); ALTER TABLE color ADD COLUMN color_id BIGINT GENERATED ALWAYS AS IDENTITY; -INSERT INTO color(color_name) VALUES ('Red'); -ALTER TABLE color ADD COLUMN color_id_1 BIGINT GENERATED ALWAYS AS IDENTITY; -DROP TABLE color; --- insert data from workers -CREATE TABLE color ( - color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_id'); +-- alter sequence .. restart +ALTER SEQUENCE color_color_id_seq RESTART WITH 1000; +-- override system value +INSERT INTO color(color_id, color_name) VALUES (1, 'Red'); +INSERT INTO color(color_id, color_name) VALUES (NULL, 'Red'); +INSERT INTO color(color_id, color_name) OVERRIDING SYSTEM VALUE VALUES (1, 'Red'); +-- update null or custom value +UPDATE color SET color_id = NULL; +UPDATE color SET color_id = 1; -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -SELECT undistribute_table('color'); -SELECT create_distributed_table('color', 'color_id'); - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -INSERT INTO color(color_name) VALUES ('Red'); - -SELECT count(*) from color; - --- modify sequence & alter table -DROP TABLE color; - -CREATE TABLE color ( - color_id BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, - color_name VARCHAR NOT NULL -); -SELECT create_distributed_table('color', 'color_id'); - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -SELECT undistribute_table('color'); - -ALTER SEQUENCE color_color_id_seq RENAME TO myseq; - -SELECT create_distributed_table('color', 'color_id'); -\ds+ myseq -\ds+ color_color_id_seq -\d color - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -\ds+ myseq -\ds+ color_color_id_seq -\d color - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -ALTER SEQUENCE myseq RENAME TO color_color_id_seq; - -\ds+ myseq -\ds+ color_color_id_seq - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -\ds+ myseq -\ds+ color_color_id_seq -\d color - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -SELECT alter_distributed_table('co23423lor', shard_count := 6); - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :worker_1_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; - -\ds+ color_color_id_seq - -INSERT INTO color(color_name) VALUES ('Red'); - -\c - - - :master_port -SET search_path TO generated_identities; -SET client_min_messages to ERROR; +DROP TABLE IF EXISTS test; +CREATE TABLE test (x int, y int, z bigint generated by default as identity); +SELECT create_distributed_table('test', 'x', colocate_with := 'none'); +INSERT INTO test VALUES (1,2); +INSERT INTO test SELECT x, y FROM test WHERE x = 1; +SELECT * FROM test; DROP SCHEMA generated_identities CASCADE; +DROP USER identity_test_user; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index a44282521..3cf3194e7 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -563,11 +563,17 @@ RESET client_min_messages; SELECT * FROM multi_extension.print_extension_changes(); -DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; - -- show running version SHOW citus.version; +-- Snapshot of state at 11.2-2 +ALTER EXTENSION citus UPDATE TO '11.2-2'; + +SELECT * FROM multi_extension.print_extension_changes(); + +-- Test downgrade to 11.2-1 from 11.2-2 +ALTER EXTENSION citus UPDATE TO '11.2-1'; + -- ensure no unexpected objects were created outside pg_catalog SELECT pgio.type, pgio.identity FROM pg_depend AS pgd, @@ -579,6 +585,8 @@ WHERE pgd.refclassid = 'pg_extension'::regclass AND pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal') ORDER BY 1, 2; +DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; + -- see incompatible version errors out RESET citus.enable_version_checks; RESET columnar.enable_version_checks;