From 2c190d068918d1c457894adf97f550e5b3739184 Mon Sep 17 00:00:00 2001 From: aykut-bozkurt <51649454+aykut-bozkurt@users.noreply.github.com> Date: Wed, 20 Sep 2023 16:45:04 +0300 Subject: [PATCH 1/8] Fix the changelog entry for citus_pause_node_within_txn() UDF (#7215) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2b2ce99..02fc91d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,8 @@ * Add `citus_schema_move()` function which moves tables within a distributed schema to another node (#7180) -* Adds `citus_pause_node()` UDF that allows pausing the node with given id - (#7089) +* Adds `citus_pause_node_within_txn()` UDF that allows pausing the node with + given id (#7089) * Makes sure to enforce shard level colocation with the GUC `citus.enable_non_colocated_router_query_pushdown` (#7076) From 0f28a69f12418d211ffba5f7ddd222fd0c47daeb Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Fri, 22 Sep 2023 16:24:18 +0200 Subject: [PATCH 2/8] Use the $(DLSUFFIX) instead of hard coded extensions for cdc (#7221) When cdc got added the makefiles hardcoded the `.so` extension instead of using the platform specifc `$(DLSUFFIX)` variable used by `pgxs.mk`. Also don't remove installed cdc artifacts on `make clean`. --- src/backend/distributed/cdc/Makefile.decoder | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/backend/distributed/cdc/Makefile.decoder b/src/backend/distributed/cdc/Makefile.decoder index 963f8bf15..8548cef47 100644 --- a/src/backend/distributed/cdc/Makefile.decoder +++ b/src/backend/distributed/cdc/Makefile.decoder @@ -14,11 +14,6 @@ override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include install: install-cdc -clean: clean-cdc - install-cdc: mkdir -p '$(citus_decoders_dir)' - $(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so' - -clean-cdc: - rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so' + $(INSTALL_SHLIB) citus_$(DECODER)$(DLSUFFIX) '$(citus_decoders_dir)/$(DECODER)$(DLSUFFIX)' From ab8a3fab74a3cfc438a1f6b5cc0159c829cdacf5 Mon Sep 17 00:00:00 2001 From: Teresa Giacomini <37844849+tgcmn1@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:24:06 -0700 Subject: [PATCH 3/8] Update README.md Update README.md to remove old customers --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ee07765c..53fada0e6 100644 --- a/README.md +++ b/README.md @@ -438,21 +438,21 @@ Citus is uniquely capable of scaling both analytical and transactional workloads The advanced parallel, distributed query engine in Citus combined with PostgreSQL features such as [array types](https://www.postgresql.org/docs/current/arrays.html), [JSONB](https://www.postgresql.org/docs/current/datatype-json.html), [lateral joins](https://heap.io/blog/engineering/postgresqls-powerful-new-join-type-lateral), and extensions like [HyperLogLog](https://github.com/citusdata/postgresql-hll) and [TopN](https://github.com/citusdata/postgresql-topn) allow you to build responsive analytics dashboards no matter how many customers or how much data you have. - Example real-time analytics users: [Algolia](https://www.citusdata.com/customers/algolia), [Heap](https://www.citusdata.com/customers/heap) + Example real-time analytics users: [Algolia](https://www.citusdata.com/customers/algolia) - **[Time series data](http://docs.citusdata.com/en/stable/use_cases/timeseries.html)**: Citus enables you to process and analyze very large amounts of time series data. The biggest Citus clusters store well over a petabyte of time series data and ingest terabytes per day. Citus integrates seamlessly with [Postgres table partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) and has [built-in functions for partitioning by time](https://www.citusdata.com/blog/2021/10/22/how-to-scale-postgres-for-time-series-data-with-citus/), which can speed up queries and writes on time series tables. You can take advantage of Citus’s parallel, distributed query engine for fast analytical queries, and use the built-in *columnar storage* to compress old partitions. - Example users: [MixRank](https://www.citusdata.com/customers/mixrank), [Windows team](https://techcommunity.microsoft.com/t5/azure-database-for-postgresql/architecting-petabyte-scale-analytics-by-scaling-out-postgres-on/ba-p/969685) + Example users: [MixRank](https://www.citusdata.com/customers/mixrank) - **[Software-as-a-service (SaaS) applications](http://docs.citusdata.com/en/stable/use_cases/multi_tenant.html)**: SaaS and other multi-tenant applications need to be able to scale their database as the number of tenants/customers grows. Citus enables you to transparently shard a complex data model by the tenant dimension, so your database can grow along with your business. By distributing tables along a tenant ID column and co-locating data for the same tenant, Citus can horizontally scale complex (tenant-scoped) queries, transactions, and foreign key graphs. Reference tables and distributed DDL commands make database management a breeze compared to manual sharding. On top of that, you have a built-in distributed query engine for doing cross-tenant analytics inside the database. - Example multi-tenant SaaS users: [Copper](https://www.citusdata.com/customers/copper), [Salesloft](https://fivetran.com/case-studies/replicating-sharded-databases-a-case-study-of-salesloft-citus-data-and-fivetran), [ConvertFlow](https://www.citusdata.com/customers/convertflow) + Example multi-tenant SaaS users: [Salesloft](https://fivetran.com/case-studies/replicating-sharded-databases-a-case-study-of-salesloft-citus-data-and-fivetran), [ConvertFlow](https://www.citusdata.com/customers/convertflow) - **[Microservices](https://docs.citusdata.com/en/stable/get_started/tutorial_microservices.html)**: Citus supports schema based sharding, which allows distributing regular database schemas across many machines. This sharding methodology fits nicely with typical Microservices architecture, where storage is fully owned by the service hence can’t share the same schema definition with other tenants. Citus allows distributing horizontally scalable state across services, solving one of the [main problems](https://stackoverflow.blog/2020/11/23/the-macro-problem-with-microservices/) of microservices. From 5926ec8bbb6c6ff4f94d26a163f32dc59fbc176b Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Tue, 19 Sep 2023 22:32:45 +0300 Subject: [PATCH 4/8] Fix broken blog link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53fada0e6..678b6c46a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -| **
The Citus database is 100% open source.

Learn what's new in the [Citus 12.0 release blog](https://www.citusdata.com/blog/2023/07/18/citus-12-schema-based-sharding-comes-to-postgres/) and the [Citus Updates page](https://www.citusdata.com/updates/).

**| +| **
The Citus database is 100% open source.

Learn what's new in the [Citus 12.0 release blog](https://www.citusdata.com/blog/2023/07/18/citus-12-schema-based-sharding-for-postgres/) and the [Citus Updates page](https://www.citusdata.com/updates/).

**| |---|
From f17d31fd94972a5099716f8b2de33732a7ac6b67 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Tue, 19 Sep 2023 22:33:13 +0300 Subject: [PATCH 5/8] Update PG and Citus versions in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 678b6c46a..6e1f62666 100644 --- a/README.md +++ b/README.md @@ -95,14 +95,14 @@ Install packages on Ubuntu / Debian: ```bash curl https://install.citusdata.com/community/deb.sh > add-citus-repo.sh sudo bash add-citus-repo.sh -sudo apt-get -y install postgresql-15-citus-12.0 +sudo apt-get -y install postgresql-16-citus-12.1 ``` Install packages on CentOS / Red Hat: ```bash curl https://install.citusdata.com/community/rpm.sh > add-citus-repo.sh sudo bash add-citus-repo.sh -sudo yum install -y citus120_15 +sudo yum install -y citus121_16 ``` To add Citus to your local PostgreSQL database, add the following to `postgresql.conf`: From 01e3c2479397593492453cf3b56ac96b09b4d254 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Fri, 22 Sep 2023 15:12:20 +0300 Subject: [PATCH 6/8] Update url for release blog --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e1f62666..fd4564189 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -| **
The Citus database is 100% open source.

Learn what's new in the [Citus 12.0 release blog](https://www.citusdata.com/blog/2023/07/18/citus-12-schema-based-sharding-for-postgres/) and the [Citus Updates page](https://www.citusdata.com/updates/).

**| +| **
The Citus database is 100% open source.

Learn what's new in the [Citus 12.1 release blog](https://www.citusdata.com/blog/2023/09/22/adding-postgres-16-support-to-citus-12-1/) and the [Citus Updates page](https://www.citusdata.com/updates/).

**| |---|
From 111b4c19bc0a429abc5be56dbb6ded9f080fa110 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 25 Sep 2023 11:14:35 +0300 Subject: [PATCH 7/8] Make sure to disallow creating a replicated distributed table concurrently (#7219) See explanation in https://github.com/citusdata/citus/issues/7216. Fixes https://github.com/citusdata/citus/issues/7216. DESCRIPTION: Makes sure to disallow creating a replicated distributed table concurrently --- .../commands/create_distributed_table.c | 13 ++++++ .../create_distributed_table_concurrently.out | 40 ++++++++----------- .../create_distributed_table_concurrently.sql | 9 +++++ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index dc06692b3..1e89c6b93 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -445,6 +445,19 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName, if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone( colocateWithTableName)) { + if (replicationModel != REPLICATION_MODEL_STREAMING) + { + ereport(ERROR, (errmsg("cannot create distributed table " + "concurrently because Citus allows " + "concurrent table distribution only when " + "citus.shard_replication_factor = 1"), + errhint("table %s is requested to be colocated " + "with %s which has " + "citus.shard_replication_factor > 1", + get_rel_name(relationId), + colocateWithTableName))); + } + EnsureColocateWithTableIsValid(relationId, distributionMethod, distributionColumnName, colocateWithTableName); diff --git a/src/test/regress/expected/create_distributed_table_concurrently.out b/src/test/regress/expected/create_distributed_table_concurrently.out index 025629efa..1bf366fb3 100644 --- a/src/test/regress/expected/create_distributed_table_concurrently.out +++ b/src/test/regress/expected/create_distributed_table_concurrently.out @@ -36,6 +36,19 @@ set citus.shard_replication_factor to 2; select create_distributed_table_concurrently('test','key', 'hash'); ERROR: cannot distribute a table concurrently when citus.shard_replication_factor > 1 set citus.shard_replication_factor to 1; +set citus.shard_replication_factor to 2; +create table dist_1(a int); +select create_distributed_table('dist_1', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +set citus.shard_replication_factor to 1; +create table dist_2(a int); +select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1'); +ERROR: cannot create distributed table concurrently because Citus allows concurrent table distribution only when citus.shard_replication_factor = 1 +HINT: table dist_2 is requested to be colocated with dist_1 which has citus.shard_replication_factor > 1 begin; select create_distributed_table_concurrently('test','key'); ERROR: create_distributed_table_concurrently cannot run inside a transaction block @@ -138,27 +151,8 @@ select count(*) from test; rollback; -- verify that we can undistribute the table begin; +set local client_min_messages to warning; select undistribute_table('test', cascade_via_foreign_keys := true); -NOTICE: converting the partitions of create_distributed_table_concurrently.test -NOTICE: creating a new table for create_distributed_table_concurrently.test -NOTICE: dropping the old create_distributed_table_concurrently.test -NOTICE: renaming the new table to create_distributed_table_concurrently.test -NOTICE: creating a new table for create_distributed_table_concurrently.ref -NOTICE: moving the data of create_distributed_table_concurrently.ref -NOTICE: dropping the old create_distributed_table_concurrently.ref -NOTICE: drop cascades to constraint test_id_fkey_1190041 on table create_distributed_table_concurrently.test_1190041 -CONTEXT: SQL statement "SELECT citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false)" -PL/pgSQL function citus_drop_trigger() line XX at PERFORM -SQL statement "DROP TABLE create_distributed_table_concurrently.ref CASCADE" -NOTICE: renaming the new table to create_distributed_table_concurrently.ref -NOTICE: creating a new table for create_distributed_table_concurrently.test_1 -NOTICE: moving the data of create_distributed_table_concurrently.test_1 -NOTICE: dropping the old create_distributed_table_concurrently.test_1 -NOTICE: renaming the new table to create_distributed_table_concurrently.test_1 -NOTICE: creating a new table for create_distributed_table_concurrently.test_2 -NOTICE: moving the data of create_distributed_table_concurrently.test_2 -NOTICE: dropping the old create_distributed_table_concurrently.test_2 -NOTICE: renaming the new table to create_distributed_table_concurrently.test_2 undistribute_table --------------------------------------------------------------------- @@ -245,7 +239,7 @@ insert into dist_table4 select s from generate_series(1,100) s; select count(*) as total from dist_table4; total --------------------------------------------------------------------- - 100 + 100 (1 row) -- verify we do not allow foreign keys from distributed table to citus local table concurrently @@ -295,13 +289,13 @@ select count(*) from test_columnar; select id from test_columnar where id = 1; id --------------------------------------------------------------------- - 1 + 1 (1 row) select id from test_columnar where id = 51; id --------------------------------------------------------------------- - 51 + 51 (1 row) select count(*) from test_columnar_1; diff --git a/src/test/regress/sql/create_distributed_table_concurrently.sql b/src/test/regress/sql/create_distributed_table_concurrently.sql index 9632eba6e..6820d782c 100644 --- a/src/test/regress/sql/create_distributed_table_concurrently.sql +++ b/src/test/regress/sql/create_distributed_table_concurrently.sql @@ -28,6 +28,14 @@ set citus.shard_replication_factor to 2; select create_distributed_table_concurrently('test','key', 'hash'); set citus.shard_replication_factor to 1; +set citus.shard_replication_factor to 2; +create table dist_1(a int); +select create_distributed_table('dist_1', 'a'); +set citus.shard_replication_factor to 1; + +create table dist_2(a int); +select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1'); + begin; select create_distributed_table_concurrently('test','key'); rollback; @@ -63,6 +71,7 @@ rollback; -- verify that we can undistribute the table begin; +set local client_min_messages to warning; select undistribute_table('test', cascade_via_foreign_keys := true); rollback; From a9d28ca96ff4dca2c63617873e2a49428c49944a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 25 Sep 2023 12:42:23 +0300 Subject: [PATCH 8/8] Adds make clean to installation steps (#7052) If you make a fresh install make clean is not required. However, if you install before, without a make install, one can get errors --------- Co-authored-by: aykut-bozkurt <51649454+aykut-bozkurt@users.noreply.github.com> --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66d026b8b..ac1f600ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,8 @@ that are missing in earlier minor versions. cd citus ./configure + # If you have already installed the project, you need to clean it first + make clean make make install # Optionally, you might instead want to use `make install-all` @@ -79,6 +81,8 @@ that are missing in earlier minor versions. git clone https://github.com/citusdata/citus.git cd citus ./configure + # If you have already installed the project previously, you need to clean it first + make clean make sudo make install # Optionally, you might instead want to use `sudo make install-all` @@ -129,6 +133,8 @@ that are missing in earlier minor versions. git clone https://github.com/citusdata/citus.git cd citus PG_CONFIG=/usr/pgsql-14/bin/pg_config ./configure + # If you have already installed the project previously, you need to clean it first + make clean make sudo make install # Optionally, you might instead want to use `sudo make install-all`