diff --git a/.github/packaging/validate_build_output.sh b/.github/packaging/validate_build_output.sh index 4ae588574..200b1a188 100755 --- a/.github/packaging/validate_build_output.sh +++ b/.github/packaging/validate_build_output.sh @@ -1,6 +1,20 @@ package_type=${1} -git clone -b v0.8.23 --depth=1 https://github.com/citusdata/tools.git tools + +# Since $HOME is set in GH_Actions as /github/home, pyenv fails to create virtualenvs. +# For this script, we set $HOME to /root and then set it back to /github/home. +GITHUB_HOME="${HOME}" +export HOME="/root" + +eval "$(pyenv init -)" +pyenv versions +pyenv virtualenv ${PACKAGING_PYTHON_VERSION} packaging_env +pyenv activate packaging_env + +git clone -b v0.8.24 --depth=1 https://github.com/citusdata/tools.git tools python3 -m pip install -r tools/packaging_automation/requirements.txt python3 -m tools.packaging_automation.validate_build_output --output_file output.log \ --ignore_file .github/packaging/packaging_ignore.yml \ --package_type ${package_type} +pyenv deactivate +# Set $HOME back to /github/home +export HOME=${GITHUB_HOME} diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 0233f2c1a..ae8d9d725 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -49,14 +49,17 @@ jobs: container: image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }} + options: --user root steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Add Postgres installation directory into PATH for rpm based distros + - name: Set Postgres and python parameters for rpm based distros run: | echo "/usr/pgsql-${{ matrix.POSTGRES_VERSION }}/bin" >> $GITHUB_PATH + echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH + echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV - name: Configure run: | @@ -115,14 +118,17 @@ jobs: container: image: citus/packaging:${{ matrix.packaging_docker_image }} + options: --user root steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set pg_config path to related Postgres version + - name: Set pg_config path and python parameters for deb based distros run: | echo "PG_CONFIG=/usr/lib/postgresql/${{ matrix.POSTGRES_VERSION }}/bin/pg_config" >> $GITHUB_ENV + echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH + echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV - name: Configure run: | @@ -154,5 +160,4 @@ jobs: apt install python3-dev python3-pip -y sudo apt-get purge -y python3-yaml python3 -m pip install --upgrade pip setuptools==57.5.0 - ./.github/packaging/validate_build_output.sh "deb" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd737ac3..428598d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,107 @@ +### citus v11.2.0 (January 30, 2023) ### + +* Adds support for outer joins with reference tables / complex subquery-CTEs + in the outer side of the join (e.g., \ LEFT JOIN + \) + +* Adds support for creating `PRIMARY KEY`s and `UNIQUE`/`EXCLUSION`/`CHECK`/ + `FOREIGN KEY` constraints via `ALTER TABLE` command without providing a + constraint name + +* Adds support for using identity columns on Citus tables + +* Adds support for `MERGE` command on local tables + +* Adds `citus_job_list()`, `citus_job_status()` and `citus_rebalance_status()` + UDFs that allow monitoring rebalancer progress + +* Adds `citus_task_wait()` UDF to wait on desired task status + +* Adds `source_lsn`, `target_lsn` and `status` fields into + `get_rebalance_progress()` + +* Introduces `citus_copy_shard_placement()` UDF with node id + +* Introduces `citus_move_shard_placement()` UDF with node id + +* Propagates `BEGIN` properties to worker nodes + +* Propagates `DROP OWNED BY` to worker nodes + +* Deprecates `citus.replicate_reference_tables_on_activate` and makes it + always `off` + +* Drops GUC `citus.defer_drop_after_shard_move` + +* Drops GUC `citus.defer_drop_after_shard_split` + +* Drops `SHARD_STATE_TO_DELETE` state and uses the cleanup records instead + +* Allows `citus_update_node()` to work with nodes from different clusters + +* Adds signal handlers for queue monitor to gracefully shutdown, cancel and to + see config changes + +* Defers cleanup after a failure in shard move or split + +* Extends cleanup process for replication artifacts + +* Improves a query that terminates compelling backends from + `citus_update_node()` + +* Includes Citus global pid in all internal `application_name`s + +* Avoids leaking `search_path` to workers when executing DDL commands + +* Fixes `alter_table_set_access_method error()` for views + +* Fixes `citus_drain_node()` to allow draining the specified worker only + +* Fixes a bug in global pid assignment for connections opened by rebalancer + internally + +* Fixes a bug that causes background rebalancer to fail when a reference table + doesn't have a primary key + +* Fixes a bug that might cause failing to query the views based on tables that + have renamed columns + +* Fixes a bug that might cause incorrectly planning the sublinks in query tree + +* Fixes a floating point exception during + `create_distributed_table_concurrently()` + +* Fixes a rebalancer failure due to integer overflow in subscription and role + creation + +* Fixes a regression in allowed foreign keys on distributed tables + +* Fixes a use-after-free bug in connection management + +* Fixes an unexpected foreign table error by disallowing to drop the + table_name option + +* Fixes an uninitialized memory access in `create_distributed_function()` + +* Fixes crash that happens when trying to replicate a reference table that is + actually dropped + +* Make sure to cleanup the shard on the target node in case of a + failed/aborted shard move + +* Makes sure to create replication artifacts with unique names + +* Makes sure to disallow triggers that depend on extensions + +* Makes sure to quote all identifiers used for logical replication to prevent + potential issues + +* Makes sure to skip foreign key validations at the end of shard moves + +* Prevents crashes on `UPDATE` with certain `RETURNING` clauses + +* Propagates column aliases in the shard-level commands + ### citus v11.1.5 (December 12, 2022) ### * Fixes two potential dangling pointer issues diff --git a/configure b/configure index d17d2caad..7a0bd7b9e 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Citus 11.2devel. +# Generated by GNU Autoconf 2.69 for Citus 11.3devel. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' -PACKAGE_VERSION='11.2devel' -PACKAGE_STRING='Citus 11.2devel' +PACKAGE_VERSION='11.3devel' +PACKAGE_STRING='Citus 11.3devel' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Citus 11.2devel to adapt to many kinds of systems. +\`configure' configures Citus 11.3devel to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1324,7 +1324,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Citus 11.2devel:";; + short | recursive ) echo "Configuration of Citus 11.3devel:";; esac cat <<\_ACEOF @@ -1429,7 +1429,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Citus configure 11.2devel +Citus configure 11.3devel generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Citus $as_me 11.2devel, which was +It was created by Citus $as_me 11.3devel, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Citus $as_me 11.2devel, which was +This file was extended by Citus $as_me 11.3devel, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5455,7 +5455,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Citus config.status 11.2devel +Citus config.status 11.3devel configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 75cf2cf71..1f63078dd 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([Citus], [11.2devel]) +AC_INIT([Citus], [11.3devel]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control index 0d6bb3a9b..5367c84a9 100644 --- a/src/backend/distributed/citus.control +++ b/src/backend/distributed/citus.control @@ -1,6 +1,6 @@ # Citus extension comment = 'Citus distributed database' -default_version = '11.2-1' +default_version = '11.3-1' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog diff --git a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c index cc3b64e4a..bb4ab7473 100644 --- a/src/backend/distributed/commands/citus_add_local_table_to_metadata.c +++ b/src/backend/distributed/commands/citus_add_local_table_to_metadata.c @@ -526,7 +526,7 @@ ErrorIfUnsupportedCreateCitusLocalTable(Relation relation) * ServerUsesPostgresFdw gets a foreign server Oid and returns true if the FDW that * the server depends on is postgres_fdw. Returns false otherwise. */ -static bool +bool ServerUsesPostgresFdw(Oid serverId) { ForeignServer *server = GetForeignServer(serverId); @@ -585,6 +585,30 @@ ErrorIfOptionListHasNoTableName(List *optionList) } +/* + * ForeignTableDropsTableNameOption returns true if given option list contains + * (DROP table_name). + */ +bool +ForeignTableDropsTableNameOption(List *optionList) +{ + char *table_nameString = "table_name"; + DefElem *option = NULL; + foreach_ptr(option, optionList) + { + char *optionName = option->defname; + DefElemAction optionAction = option->defaction; + if (strcmp(optionName, table_nameString) == 0 && + optionAction == DEFELEM_DROP) + { + return true; + } + } + + return false; +} + + /* * ErrorIfUnsupportedCitusLocalTableKind errors out if the relation kind of * relation with relationId is not supported for citus local table creation. diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index ba75bb101..39a652f10 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -41,6 +41,7 @@ #include "distributed/resource_lock.h" #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" +#include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "parser/parse_expr.h" @@ -119,6 +120,8 @@ static Oid get_attrdef_oid(Oid relationId, AttrNumber attnum); static char * GetAddColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, TypeName *typeName); +static void ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid + relationId); /* @@ -3078,6 +3081,42 @@ ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, } +/* + * ErrorIfAlterTableDropTableNameFromPostgresFdw errors if given alter foreign table + * option list drops 'table_name' from a postgresfdw foreign table which is + * inside metadata. + */ +static void +ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid relationId) +{ + char relationKind PG_USED_FOR_ASSERTS_ONLY = + get_rel_relkind(relationId); + Assert(relationKind == RELKIND_FOREIGN_TABLE); + + ForeignTable *foreignTable = GetForeignTable(relationId); + Oid serverId = foreignTable->serverid; + if (!ServerUsesPostgresFdw(serverId)) + { + return; + } + + if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE) && + ForeignTableDropsTableNameOption(optionList)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "alter foreign table alter options (drop table_name) command " + "is not allowed for Citus tables"), + errdetail( + "Table_name option can not be dropped from a foreign table " + "which is inside metadata."), + errhint( + "Try to undistribute foreign table before dropping table_name option."))); + } +} + + /* * ErrorIfUnsupportedAlterTableStmt checks if the corresponding alter table * statement is supported for distributed tables and errors out if it is not. @@ -3490,6 +3529,8 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) { if (IsForeignTable(relationId)) { + List *optionList = (List *) command->def; + ErrorIfAlterTableDropTableNameFromPostgresFdw(optionList, relationId); break; } } diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index adebdb90c..899384ad5 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -219,6 +219,23 @@ multi_ProcessUtility(PlannedStmt *pstmt, PreprocessCreateExtensionStmtForCitusColumnar(parsetree); } + /* + * Make sure that on DROP DATABASE we terminate the background daemon + * associated with it. + */ + if (IsA(parsetree, DropdbStmt)) + { + const bool missingOK = true; + DropdbStmt *dropDbStatement = (DropdbStmt *) parsetree; + char *dbname = dropDbStatement->dbname; + Oid databaseOid = get_database_oid(dbname, missingOK); + + if (OidIsValid(databaseOid)) + { + StopMaintenanceDaemon(databaseOid); + } + } + if (!CitusHasBeenLoaded()) { /* @@ -678,22 +695,9 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } /* - * Make sure that on DROP DATABASE we terminate the background daemon + * Make sure that on DROP EXTENSION we terminate the background daemon * associated with it. */ - if (IsA(parsetree, DropdbStmt)) - { - const bool missingOK = true; - DropdbStmt *dropDbStatement = (DropdbStmt *) parsetree; - char *dbname = dropDbStatement->dbname; - Oid databaseOid = get_database_oid(dbname, missingOK); - - if (OidIsValid(databaseOid)) - { - StopMaintenanceDaemon(databaseOid); - } - } - if (IsDropCitusExtensionStmt(parsetree)) { StopMaintenanceDaemon(MyDatabaseId); diff --git a/src/backend/distributed/connection/connection_management.c b/src/backend/distributed/connection/connection_management.c index c5b300fd4..8ab35ca42 100644 --- a/src/backend/distributed/connection/connection_management.c +++ b/src/backend/distributed/connection/connection_management.c @@ -1454,6 +1454,9 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit) { ShutdownConnection(connection); + /* remove from transactionlist before free-ing */ + ResetRemoteTransaction(connection); + /* unlink from list */ dlist_delete(iter.cur); diff --git a/src/backend/distributed/operations/shard_rebalancer.c b/src/backend/distributed/operations/shard_rebalancer.c index d26880b9b..d24936925 100644 --- a/src/backend/distributed/operations/shard_rebalancer.c +++ b/src/backend/distributed/operations/shard_rebalancer.c @@ -1937,7 +1937,10 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo if (HasNodesWithMissingReferenceTables(&referenceTableIdList)) { - VerifyTablesHaveReplicaIdentity(referenceTableIdList); + if (shardTransferMode == TRANSFER_MODE_AUTOMATIC) + { + VerifyTablesHaveReplicaIdentity(referenceTableIdList); + } /* * Reference tables need to be copied to (newly-added) nodes, this needs to be the diff --git a/src/backend/distributed/sql/citus--11.2-1--11.3-1.sql b/src/backend/distributed/sql/citus--11.2-1--11.3-1.sql new file mode 100644 index 000000000..981c5f375 --- /dev/null +++ b/src/backend/distributed/sql/citus--11.2-1--11.3-1.sql @@ -0,0 +1,4 @@ +-- citus--11.2-1--11.3-1 + +-- bump version to 11.3-1 + diff --git a/src/backend/distributed/sql/downgrades/citus--11.3-1--11.2-1.sql b/src/backend/distributed/sql/downgrades/citus--11.3-1--11.2-1.sql new file mode 100644 index 000000000..7d71235d7 --- /dev/null +++ b/src/backend/distributed/sql/downgrades/citus--11.3-1--11.2-1.sql @@ -0,0 +1,2 @@ +-- citus--11.3-1--11.2-1 +-- this is an empty downgrade path since citus--11.2-1--11.3-1.sql is empty for now diff --git a/src/backend/distributed/sql/udfs/citus_job_status/11.2-1.sql b/src/backend/distributed/sql/udfs/citus_job_status/11.2-1.sql index 07709a614..93496203a 100644 --- a/src/backend/distributed/sql/udfs/citus_job_status/11.2-1.sql +++ b/src/backend/distributed/sql/udfs/citus_job_status/11.2-1.sql @@ -74,7 +74,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( WHERE j.job_id = $1 AND t.status = 'running' ), - errored_task_details AS ( + errored_or_retried_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), @@ -85,7 +85,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( pg_dist_background_task t JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND NOT EXISTS (SELECT 1 FROM rp WHERE rp.sessionid = t.pid) - AND t.status = 'error' + AND (t.status = 'error' OR (t.status = 'runnable' AND t.retry_count > 0)) ) SELECT job_id, @@ -97,7 +97,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( jsonb_build_object( 'task_state_counts', (SELECT jsonb_object_agg(status, count) FROM task_state_occurence_counts), 'tasks', (COALESCE((SELECT tasks FROM running_task_details),'[]'::jsonb) || - COALESCE((SELECT tasks FROM errored_task_details),'[]'::jsonb))) AS details + COALESCE((SELECT tasks FROM errored_or_retried_task_details),'[]'::jsonb))) AS details FROM pg_dist_background_job j WHERE j.job_id = $1 $fn$; diff --git a/src/backend/distributed/sql/udfs/citus_job_status/latest.sql b/src/backend/distributed/sql/udfs/citus_job_status/latest.sql index 07709a614..93496203a 100644 --- a/src/backend/distributed/sql/udfs/citus_job_status/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_job_status/latest.sql @@ -74,7 +74,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( WHERE j.job_id = $1 AND t.status = 'running' ), - errored_task_details AS ( + errored_or_retried_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), @@ -85,7 +85,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( pg_dist_background_task t JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND NOT EXISTS (SELECT 1 FROM rp WHERE rp.sessionid = t.pid) - AND t.status = 'error' + AND (t.status = 'error' OR (t.status = 'runnable' AND t.retry_count > 0)) ) SELECT job_id, @@ -97,7 +97,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( jsonb_build_object( 'task_state_counts', (SELECT jsonb_object_agg(status, count) FROM task_state_occurence_counts), 'tasks', (COALESCE((SELECT tasks FROM running_task_details),'[]'::jsonb) || - COALESCE((SELECT tasks FROM errored_task_details),'[]'::jsonb))) AS details + COALESCE((SELECT tasks FROM errored_or_retried_task_details),'[]'::jsonb))) AS details FROM pg_dist_background_job j WHERE j.job_id = $1 $fn$; diff --git a/src/backend/distributed/transaction/remote_transaction.c b/src/backend/distributed/transaction/remote_transaction.c index aff357fb3..0f6241793 100644 --- a/src/backend/distributed/transaction/remote_transaction.c +++ b/src/backend/distributed/transaction/remote_transaction.c @@ -80,6 +80,7 @@ StartRemoteTransactionBegin(struct MultiConnection *connection) /* remember transaction as being in-progress */ dlist_push_tail(&InProgressTransactions, &connection->transactionNode); + connection->transactionInProgress = true; transaction->transactionState = REMOTE_TRANS_STARTING; @@ -865,11 +866,13 @@ ResetRemoteTransaction(struct MultiConnection *connection) RemoteTransaction *transaction = &connection->remoteTransaction; /* unlink from list of open transactions, if necessary */ - if (transaction->transactionState != REMOTE_TRANS_NOT_STARTED) + if (connection->transactionInProgress) { /* XXX: Should we error out for a critical transaction? */ dlist_delete(&connection->transactionNode); + connection->transactionInProgress = false; + memset(&connection->transactionNode, 0, sizeof(connection->transactionNode)); } /* just reset the entire state, relying on 0 being invalid/false */ diff --git a/src/include/distributed/connection_management.h b/src/include/distributed/connection_management.h index f95fb612d..4ffb83a86 100644 --- a/src/include/distributed/connection_management.h +++ b/src/include/distributed/connection_management.h @@ -189,8 +189,12 @@ typedef struct MultiConnection /* information about the associated remote transaction */ RemoteTransaction remoteTransaction; - /* membership in list of in-progress transactions */ + /* + * membership in list of in-progress transactions and a flag to indicate + * that the connection was added to this list + */ dlist_node transactionNode; + bool transactionInProgress; /* list of all placements referenced by this connection */ dlist_head referencedPlacements; diff --git a/src/include/distributed/metadata_utility.h b/src/include/distributed/metadata_utility.h index 82576d681..ceea51678 100644 --- a/src/include/distributed/metadata_utility.h +++ b/src/include/distributed/metadata_utility.h @@ -353,6 +353,8 @@ extern void EnsureRelationExists(Oid relationId); extern bool RegularTable(Oid relationId); extern bool TableEmpty(Oid tableId); extern bool IsForeignTable(Oid relationId); +extern bool ForeignTableDropsTableNameOption(List *optionList); +extern bool ServerUsesPostgresFdw(Oid serverId); extern char * ConstructQualifiedShardName(ShardInterval *shardInterval); extern uint64 GetFirstShardId(Oid relationId); extern Datum StringToDatum(char *inputString, Oid dataType); diff --git a/src/test/regress/citus_tests/config.py b/src/test/regress/citus_tests/config.py index 35b569b9b..72a05ecef 100644 --- a/src/test/regress/citus_tests/config.py +++ b/src/test/regress/citus_tests/config.py @@ -38,7 +38,7 @@ CITUS_ARBITRARY_TEST_DIR = "./tmp_citus_test" MASTER = "master" # This should be updated when citus version changes -MASTER_VERSION = "11.2" +MASTER_VERSION = "11.3" HOME = expanduser("~") diff --git a/src/test/regress/expected/background_rebalance.out b/src/test/regress/expected/background_rebalance.out index bc1db54ac..c82078d6f 100644 --- a/src/test/regress/expected/background_rebalance.out +++ b/src/test/regress/expected/background_rebalance.out @@ -210,6 +210,100 @@ SELECT citus_rebalance_stop(); (1 row) RESET ROLE; +CREATE TABLE ref_no_pk(a int); +SELECT create_reference_table('ref_no_pk'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE ref_with_pk(a int primary key); +SELECT create_reference_table('ref_with_pk'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- Add coordinator so there's a node which doesn't have the reference tables +SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0); +NOTICE: localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- fails +BEGIN; +SELECT 1 FROM citus_rebalance_start(); +ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY +DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY. +HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'. +ROLLBACK; +-- success +BEGIN; +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical'); +NOTICE: Scheduled 1 moves as job xxx +DETAIL: Rebalance scheduled as background job +HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status(); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +ROLLBACK; +-- success +BEGIN; +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes'); +NOTICE: Scheduled 1 moves as job xxx +DETAIL: Rebalance scheduled as background job +HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status(); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +ROLLBACK; +-- fails +SELECT 1 FROM citus_rebalance_start(); +ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY +DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY. +HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'. +-- succeeds +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical'); +NOTICE: Scheduled 1 moves as job xxx +DETAIL: Rebalance scheduled as background job +HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status(); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- wait for success +SELECT citus_rebalance_wait(); + citus_rebalance_wait +--------------------------------------------------------------------- + +(1 row) + +SELECT state, details from citus_rebalance_status(); + state | details +--------------------------------------------------------------------- + finished | {"tasks": [], "task_state_counts": {"done": 2}} +(1 row) + +-- Remove coordinator again to allow rerunning of this test +SELECT 1 FROM citus_remove_node('localhost', :master_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT public.wait_until_metadata_sync(30000); + wait_until_metadata_sync +--------------------------------------------------------------------- + +(1 row) + SET client_min_messages TO WARNING; DROP SCHEMA background_rebalance CASCADE; DROP USER non_super_user_rebalance; diff --git a/src/test/regress/expected/drop_database.out b/src/test/regress/expected/drop_database.out new file mode 100644 index 000000000..d150cc8d3 --- /dev/null +++ b/src/test/regress/expected/drop_database.out @@ -0,0 +1,43 @@ +-- coordinator +CREATE SCHEMA drop_database; +SET search_path TO drop_database; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 35137400; +CREATE DATABASE citus_created; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c citus_created +CREATE EXTENSION citus; +CREATE DATABASE citus_not_created; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c citus_not_created +DROP DATABASE citus_created; +\c regression +DROP DATABASE citus_not_created; +-- worker1 +\c - - - :worker_1_port +SET search_path TO drop_database; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 35137400; +CREATE DATABASE citus_created; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c citus_created +CREATE EXTENSION citus; +CREATE DATABASE citus_not_created; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c citus_not_created +DROP DATABASE citus_created; +\c regression +DROP DATABASE citus_not_created; +\c - - - :master_port +SET client_min_messages TO WARNING; +DROP SCHEMA drop_database CASCADE; diff --git a/src/test/regress/expected/foreign_tables_mx.out b/src/test/regress/expected/foreign_tables_mx.out index b2574432c..4bcddac5a 100644 --- a/src/test/regress/expected/foreign_tables_mx.out +++ b/src/test/regress/expected/foreign_tables_mx.out @@ -416,6 +416,65 @@ NOTICE: renaming the new table to foreign_tables_schema_mx.foreign_table_local_ (1 row) DROP FOREIGN TABLE foreign_table_local; +-- disallow dropping table_name when foreign table is in metadata +CREATE TABLE table_name_drop(id int); +CREATE FOREIGN TABLE foreign_table_name_drop_fails ( + id INT +) + SERVER foreign_server_local + OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop'); +SELECT citus_add_local_table_to_metadata('foreign_table_name_drop_fails'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +-- table_name option is already added +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (ADD table_name 'table_name_drop'); +ERROR: option "table_name" provided more than once +-- throw error if user tries to drop table_name option from a foreign table inside metadata +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP table_name); +ERROR: alter foreign table alter options (drop table_name) command is not allowed for Citus tables +-- case sensitive option name +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP Table_Name); +ERROR: alter foreign table alter options (drop table_name) command is not allowed for Citus tables +-- other options are allowed to drop +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP schema_name); +CREATE FOREIGN TABLE foreign_table_name_drop ( + id INT +) + SERVER foreign_server_local + OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop'); +-- user can drop table_option if foreign table is not in metadata +ALTER FOREIGN TABLE foreign_table_name_drop OPTIONS (DROP table_name); +-- we should not intercept data wrappers other than postgres_fdw +CREATE EXTENSION file_fdw; +-- remove validator method to add table_name option; otherwise, table_name option is not allowed +SELECT result FROM run_command_on_all_nodes('ALTER FOREIGN DATA WRAPPER file_fdw NO VALIDATOR'); + result +--------------------------------------------------------------------- + ALTER FOREIGN DATA WRAPPER + ALTER FOREIGN DATA WRAPPER + ALTER FOREIGN DATA WRAPPER +(3 rows) + +CREATE SERVER citustest FOREIGN DATA WRAPPER file_fdw; +\copy (select i from generate_series(0,100)i) to '/tmp/test_file_fdw.data'; +CREATE FOREIGN TABLE citustest_filefdw ( + data text +) + SERVER citustest + OPTIONS ( filename '/tmp/test_file_fdw.data'); +-- add non-postgres_fdw table into metadata even if it does not have table_name option +SELECT citus_add_local_table_to_metadata('citustest_filefdw'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +ALTER FOREIGN TABLE citustest_filefdw OPTIONS (ADD table_name 'unused_table_name_option'); +-- drop table_name option of non-postgres_fdw table even if it is inside metadata +ALTER FOREIGN TABLE citustest_filefdw OPTIONS (DROP table_name); -- cleanup at exit set client_min_messages to error; DROP SCHEMA foreign_tables_schema_mx CASCADE; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 7ba049c6c..092ec9e5c 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -773,6 +773,14 @@ SELECT 1 FROM columnar_table; -- seq scan ERROR: loaded Citus library version differs from installed extension version CREATE TABLE new_columnar_table (a int) USING columnar; ERROR: loaded Citus library version differs from installed extension version +-- disable version checks for other sessions too +ALTER SYSTEM SET citus.enable_version_checks TO OFF; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + -- do cleanup for the rest of the tests SET citus.enable_version_checks TO OFF; SET columnar.enable_version_checks TO OFF; @@ -1303,12 +1311,28 @@ SELECT * FROM multi_extension.print_extension_changes(); | type cluster_clock (38 rows) +-- Test downgrade to 11.2-1 from 11.3-1 +ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.2-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 11.3-1 +ALTER EXTENSION citus UPDATE TO '11.3-1'; +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version SHOW citus.version; citus.version --------------------------------------------------------------------- - 11.2devel + 11.3devel (1 row) -- ensure no unexpected objects were created outside pg_catalog @@ -1329,11 +1353,19 @@ ORDER BY 1, 2; -- see incompatible version errors out RESET citus.enable_version_checks; RESET columnar.enable_version_checks; +-- reset version check config for other sessions too +ALTER SYSTEM RESET citus.enable_version_checks; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + DROP EXTENSION citus; DROP EXTENSION citus_columnar; CREATE EXTENSION citus VERSION '8.0-1'; ERROR: specified version incompatible with loaded Citus library -DETAIL: Loaded library requires 11.2, but 8.0-1 was specified. +DETAIL: Loaded library requires 11.3, but 8.0-1 was specified. HINT: If a newer library is present, restart the database and try the command again. -- Test non-distributed queries work even in version mismatch SET citus.enable_version_checks TO 'false'; @@ -1378,7 +1410,7 @@ ORDER BY 1; -- We should not distribute table in version mistmatch SELECT create_distributed_table('version_mismatch_table', 'column1'); ERROR: loaded Citus library version differs from installed extension version -DETAIL: Loaded library requires 11.2, but the installed extension version is 8.1-1. +DETAIL: Loaded library requires 11.3, but the installed extension version is 8.1-1. HINT: Run ALTER EXTENSION citus UPDATE and try again. -- This function will cause fail in next ALTER EXTENSION CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass) diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 447fb1ea8..1d5ce0798 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -121,3 +121,4 @@ test: ensure_no_shared_connection_leak test: check_mx test: generated_identity +test: drop_database diff --git a/src/test/regress/sql/background_rebalance.sql b/src/test/regress/sql/background_rebalance.sql index 58cdcb123..4d105655b 100644 --- a/src/test/regress/sql/background_rebalance.sql +++ b/src/test/regress/sql/background_rebalance.sql @@ -61,7 +61,6 @@ SELECT citus_rebalance_wait(); DROP TABLE t1; - -- make sure a non-super user can stop rebalancing CREATE USER non_super_user_rebalance WITH LOGIN; GRANT ALL ON SCHEMA background_rebalance TO non_super_user_rebalance; @@ -77,6 +76,37 @@ SELECT citus_rebalance_stop(); RESET ROLE; +CREATE TABLE ref_no_pk(a int); +SELECT create_reference_table('ref_no_pk'); +CREATE TABLE ref_with_pk(a int primary key); +SELECT create_reference_table('ref_with_pk'); +-- Add coordinator so there's a node which doesn't have the reference tables +SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0); + +-- fails +BEGIN; +SELECT 1 FROM citus_rebalance_start(); +ROLLBACK; +-- success +BEGIN; +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical'); +ROLLBACK; +-- success +BEGIN; +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes'); +ROLLBACK; + +-- fails +SELECT 1 FROM citus_rebalance_start(); +-- succeeds +SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical'); +-- wait for success +SELECT citus_rebalance_wait(); +SELECT state, details from citus_rebalance_status(); + +-- Remove coordinator again to allow rerunning of this test +SELECT 1 FROM citus_remove_node('localhost', :master_port); +SELECT public.wait_until_metadata_sync(30000); SET client_min_messages TO WARNING; DROP SCHEMA background_rebalance CASCADE; diff --git a/src/test/regress/sql/drop_database.sql b/src/test/regress/sql/drop_database.sql new file mode 100644 index 000000000..29d9d427a --- /dev/null +++ b/src/test/regress/sql/drop_database.sql @@ -0,0 +1,45 @@ +-- coordinator +CREATE SCHEMA drop_database; +SET search_path TO drop_database; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 35137400; + +CREATE DATABASE citus_created; + +\c citus_created +CREATE EXTENSION citus; + +CREATE DATABASE citus_not_created; + +\c citus_not_created +DROP DATABASE citus_created; + +\c regression +DROP DATABASE citus_not_created; + +-- worker1 +\c - - - :worker_1_port + +SET search_path TO drop_database; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 35137400; + +CREATE DATABASE citus_created; + +\c citus_created +CREATE EXTENSION citus; + +CREATE DATABASE citus_not_created; + +\c citus_not_created +DROP DATABASE citus_created; + +\c regression +DROP DATABASE citus_not_created; + +\c - - - :master_port + +SET client_min_messages TO WARNING; +DROP SCHEMA drop_database CASCADE; diff --git a/src/test/regress/sql/foreign_tables_mx.sql b/src/test/regress/sql/foreign_tables_mx.sql index 7b6784aab..eec5b7316 100644 --- a/src/test/regress/sql/foreign_tables_mx.sql +++ b/src/test/regress/sql/foreign_tables_mx.sql @@ -247,6 +247,62 @@ SELECT undistribute_table('foreign_table_local_fails'); DROP FOREIGN TABLE foreign_table_local; +-- disallow dropping table_name when foreign table is in metadata +CREATE TABLE table_name_drop(id int); +CREATE FOREIGN TABLE foreign_table_name_drop_fails ( + id INT +) + SERVER foreign_server_local + OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop'); + +SELECT citus_add_local_table_to_metadata('foreign_table_name_drop_fails'); + +-- table_name option is already added +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (ADD table_name 'table_name_drop'); + +-- throw error if user tries to drop table_name option from a foreign table inside metadata +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP table_name); + +-- case sensitive option name +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP Table_Name); + +-- other options are allowed to drop +ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP schema_name); + +CREATE FOREIGN TABLE foreign_table_name_drop ( + id INT +) + SERVER foreign_server_local + OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop'); + +-- user can drop table_option if foreign table is not in metadata +ALTER FOREIGN TABLE foreign_table_name_drop OPTIONS (DROP table_name); + +-- we should not intercept data wrappers other than postgres_fdw +CREATE EXTENSION file_fdw; + +-- remove validator method to add table_name option; otherwise, table_name option is not allowed +SELECT result FROM run_command_on_all_nodes('ALTER FOREIGN DATA WRAPPER file_fdw NO VALIDATOR'); + +CREATE SERVER citustest FOREIGN DATA WRAPPER file_fdw; + +\copy (select i from generate_series(0,100)i) to '/tmp/test_file_fdw.data'; +CREATE FOREIGN TABLE citustest_filefdw ( + data text +) + SERVER citustest + OPTIONS ( filename '/tmp/test_file_fdw.data'); + + +-- add non-postgres_fdw table into metadata even if it does not have table_name option +SELECT citus_add_local_table_to_metadata('citustest_filefdw'); + +ALTER FOREIGN TABLE citustest_filefdw OPTIONS (ADD table_name 'unused_table_name_option'); + +-- drop table_name option of non-postgres_fdw table even if it is inside metadata +ALTER FOREIGN TABLE citustest_filefdw OPTIONS (DROP table_name); + + -- cleanup at exit set client_min_messages to error; DROP SCHEMA foreign_tables_schema_mx CASCADE; diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index a44282521..8c8ade9d8 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -331,6 +331,10 @@ SELECT 1 FROM columnar_table; -- seq scan CREATE TABLE new_columnar_table (a int) USING columnar; +-- disable version checks for other sessions too +ALTER SYSTEM SET citus.enable_version_checks TO OFF; +SELECT pg_reload_conf(); + -- do cleanup for the rest of the tests SET citus.enable_version_checks TO OFF; SET columnar.enable_version_checks TO OFF; @@ -563,6 +567,16 @@ RESET client_min_messages; SELECT * FROM multi_extension.print_extension_changes(); +-- Test downgrade to 11.2-1 from 11.3-1 +ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.2-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); + +-- Snapshot of state at 11.3-1 +ALTER EXTENSION citus UPDATE TO '11.3-1'; +SELECT * FROM multi_extension.print_extension_changes(); + DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version @@ -582,6 +596,11 @@ ORDER BY 1, 2; -- see incompatible version errors out RESET citus.enable_version_checks; RESET columnar.enable_version_checks; + +-- reset version check config for other sessions too +ALTER SYSTEM RESET citus.enable_version_checks; +SELECT pg_reload_conf(); + DROP EXTENSION citus; DROP EXTENSION citus_columnar; CREATE EXTENSION citus VERSION '8.0-1';