Merge branch 'main' into circleci-gha-migration

CHANGELOG.md

@@ -5,8 +5,8 @@
 * Add `citus_schema_move()` function which moves tables within a
   distributed schema to another node (#7180)
 
-* Adds `citus_pause_node()` UDF that allows pausing the node with given id
-  (#7089)
+* Adds `citus_pause_node_within_txn()` UDF that allows pausing the node with
+  given id (#7089)
 
 * Makes sure to enforce shard level colocation with the GUC
   `citus.enable_non_colocated_router_query_pushdown` (#7076)

CONTRIBUTING.md

@@ -41,6 +41,8 @@ that are missing in earlier minor versions.
 
   cd citus
   ./configure
+  # If you have already installed the project, you need to clean it first
+  make clean
   make
   make install
   # Optionally, you might instead want to use `make install-all`
@@ -79,6 +81,8 @@ that are missing in earlier minor versions.
   git clone https://github.com/citusdata/citus.git
   cd citus
   ./configure
+  # If you have already installed the project previously, you need to clean it first
+  make clean
   make
   sudo make install
   # Optionally, you might instead want to use `sudo make install-all`
@@ -129,6 +133,8 @@ that are missing in earlier minor versions.
   git clone https://github.com/citusdata/citus.git
   cd citus
   PG_CONFIG=/usr/pgsql-14/bin/pg_config ./configure
+  # If you have already installed the project previously, you need to clean it first
+  make clean
   make
   sudo make install
   # Optionally, you might instead want to use `sudo make install-all`

README.md

@@ -1,4 +1,4 @@
-| **<br/>The Citus database is 100% open source.<br/><img width=1000/><br/>Learn what's new in the [Citus 12.0 release blog](https://www.citusdata.com/blog/2023/07/18/citus-12-schema-based-sharding-comes-to-postgres/) and the [Citus Updates page](https://www.citusdata.com/updates/).<br/><br/>**|
+| **<br/>The Citus database is 100% open source.<br/><img width=1000/><br/>Learn what's new in the [Citus 12.1 release blog](https://www.citusdata.com/blog/2023/09/22/adding-postgres-16-support-to-citus-12-1/) and the [Citus Updates page](https://www.citusdata.com/updates/).<br/><br/>**|
 |---|
 <br/>
 
@@ -95,14 +95,14 @@ Install packages on Ubuntu / Debian:
 ```bash
 curl https://install.citusdata.com/community/deb.sh > add-citus-repo.sh
 sudo bash add-citus-repo.sh
-sudo apt-get -y install postgresql-15-citus-12.0
+sudo apt-get -y install postgresql-16-citus-12.1
 ```
 
 Install packages on CentOS / Red Hat:
 ```bash
 curl https://install.citusdata.com/community/rpm.sh > add-citus-repo.sh
 sudo bash add-citus-repo.sh
-sudo yum install -y citus120_15
+sudo yum install -y citus121_16
 ```
 
 To add Citus to your local PostgreSQL database, add the following to `postgresql.conf`:
@@ -438,21 +438,21 @@ Citus is uniquely capable of scaling both analytical and transactional workloads
 
   The advanced parallel, distributed query engine in Citus combined with PostgreSQL features such as [array types](https://www.postgresql.org/docs/current/arrays.html), [JSONB](https://www.postgresql.org/docs/current/datatype-json.html), [lateral joins](https://heap.io/blog/engineering/postgresqls-powerful-new-join-type-lateral), and extensions like [HyperLogLog](https://github.com/citusdata/postgresql-hll) and [TopN](https://github.com/citusdata/postgresql-topn) allow you to build responsive analytics dashboards no matter how many customers or how much data you have.
 
-  Example real-time analytics users: [Algolia](https://www.citusdata.com/customers/algolia), [Heap](https://www.citusdata.com/customers/heap)
+  Example real-time analytics users: [Algolia](https://www.citusdata.com/customers/algolia)
 
 - **[Time series data](http://docs.citusdata.com/en/stable/use_cases/timeseries.html)**:
   Citus enables you to process and analyze very large amounts of time series data. The biggest Citus clusters store well over a petabyte of time series data and ingest terabytes per day.
 
   Citus integrates seamlessly with [Postgres table partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) and has [built-in functions for partitioning by time](https://www.citusdata.com/blog/2021/10/22/how-to-scale-postgres-for-time-series-data-with-citus/), which can speed up queries and writes on time series tables. You can take advantage of Citus’s parallel, distributed query engine for fast analytical queries, and use the built-in *columnar storage* to compress old partitions.
 
-  Example users: [MixRank](https://www.citusdata.com/customers/mixrank), [Windows team](https://techcommunity.microsoft.com/t5/azure-database-for-postgresql/architecting-petabyte-scale-analytics-by-scaling-out-postgres-on/ba-p/969685)
+  Example users: [MixRank](https://www.citusdata.com/customers/mixrank)
 
 - **[Software-as-a-service (SaaS) applications](http://docs.citusdata.com/en/stable/use_cases/multi_tenant.html)**:
   SaaS and other multi-tenant applications need to be able to scale their database as the number of tenants/customers grows. Citus enables you to transparently shard a complex data model by the tenant dimension, so your database can grow along with your business.
 
   By distributing tables along a tenant ID column and co-locating data for the same tenant, Citus can horizontally scale complex (tenant-scoped) queries, transactions, and foreign key graphs. Reference tables and distributed DDL commands make database management a breeze compared to manual sharding. On top of that, you have a built-in distributed query engine for doing cross-tenant analytics inside the database.
 
-  Example multi-tenant SaaS users: [Copper](https://www.citusdata.com/customers/copper), [Salesloft](https://fivetran.com/case-studies/replicating-sharded-databases-a-case-study-of-salesloft-citus-data-and-fivetran), [ConvertFlow](https://www.citusdata.com/customers/convertflow)
+  Example multi-tenant SaaS users: [Salesloft](https://fivetran.com/case-studies/replicating-sharded-databases-a-case-study-of-salesloft-citus-data-and-fivetran), [ConvertFlow](https://www.citusdata.com/customers/convertflow)
 
 - **[Microservices](https://docs.citusdata.com/en/stable/get_started/tutorial_microservices.html)**: Citus supports schema based sharding, which allows distributing regular database schemas across many machines. This sharding methodology fits nicely with typical Microservices architecture, where storage is fully owned by the service hence can’t share the same schema definition with other tenants. Citus allows distributing horizontally scalable state across services, solving one of the [main problems](https://stackoverflow.blog/2020/11/23/the-macro-problem-with-microservices/) of microservices.
 

src/backend/distributed/cdc/Makefile.decoder

@@ -14,11 +14,6 @@ override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include
 
 install: install-cdc
 
-clean: clean-cdc
-
 install-cdc:
 	mkdir -p '$(citus_decoders_dir)'
-	$(INSTALL_SHLIB) citus_$(DECODER).so '$(citus_decoders_dir)/$(DECODER).so'
-
-clean-cdc:
-	rm -f '$(DESTDIR)$(datadir)/$(datamoduledir)/citus_decoders/$(DECODER).so'
+	$(INSTALL_SHLIB) citus_$(DECODER)$(DLSUFFIX) '$(citus_decoders_dir)/$(DECODER)$(DLSUFFIX)'

src/backend/distributed/commands/create_distributed_table.c

@@ -445,6 +445,19 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 	if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone(
 			colocateWithTableName))
 	{
+		if (replicationModel != REPLICATION_MODEL_STREAMING)
+		{
+			ereport(ERROR, (errmsg("cannot create distributed table "
+								   "concurrently because Citus allows "
+								   "concurrent table distribution only when "
+								   "citus.shard_replication_factor = 1"),
+							errhint("table %s is requested to be colocated "
+									"with %s which has "
+									"citus.shard_replication_factor > 1",
+									get_rel_name(relationId),
+									colocateWithTableName)));
+		}
+
 		EnsureColocateWithTableIsValid(relationId, distributionMethod,
 									   distributionColumnName,
 									   colocateWithTableName);

src/test/regress/expected/create_distributed_table_concurrently.out

@@ -36,6 +36,19 @@ set citus.shard_replication_factor to 2;
 select create_distributed_table_concurrently('test','key', 'hash');
 ERROR:  cannot distribute a table concurrently when citus.shard_replication_factor > 1
 set citus.shard_replication_factor to 1;
+set citus.shard_replication_factor to 2;
+create table dist_1(a int);
+select create_distributed_table('dist_1', 'a');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+set citus.shard_replication_factor to 1;
+create table dist_2(a int);
+select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1');
+ERROR:  cannot create distributed table concurrently because Citus allows concurrent table distribution only when citus.shard_replication_factor = 1
+HINT:  table dist_2 is requested to be colocated with dist_1 which has citus.shard_replication_factor > 1
 begin;
 select create_distributed_table_concurrently('test','key');
 ERROR:  create_distributed_table_concurrently cannot run inside a transaction block
@@ -138,27 +151,8 @@ select count(*) from test;
 rollback;
 -- verify that we can undistribute the table
 begin;
+set local client_min_messages to warning;
 select undistribute_table('test', cascade_via_foreign_keys := true);
-NOTICE:  converting the partitions of create_distributed_table_concurrently.test
-NOTICE:  creating a new table for create_distributed_table_concurrently.test
-NOTICE:  dropping the old create_distributed_table_concurrently.test
-NOTICE:  renaming the new table to create_distributed_table_concurrently.test
-NOTICE:  creating a new table for create_distributed_table_concurrently.ref
-NOTICE:  moving the data of create_distributed_table_concurrently.ref
-NOTICE:  dropping the old create_distributed_table_concurrently.ref
-NOTICE:  drop cascades to constraint test_id_fkey_1190041 on table create_distributed_table_concurrently.test_1190041
-CONTEXT:  SQL statement "SELECT citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false)"
-PL/pgSQL function citus_drop_trigger() line XX at PERFORM
-SQL statement "DROP TABLE create_distributed_table_concurrently.ref CASCADE"
-NOTICE:  renaming the new table to create_distributed_table_concurrently.ref
-NOTICE:  creating a new table for create_distributed_table_concurrently.test_1
-NOTICE:  moving the data of create_distributed_table_concurrently.test_1
-NOTICE:  dropping the old create_distributed_table_concurrently.test_1
-NOTICE:  renaming the new table to create_distributed_table_concurrently.test_1
-NOTICE:  creating a new table for create_distributed_table_concurrently.test_2
-NOTICE:  moving the data of create_distributed_table_concurrently.test_2
-NOTICE:  dropping the old create_distributed_table_concurrently.test_2
-NOTICE:  renaming the new table to create_distributed_table_concurrently.test_2
  undistribute_table
 ---------------------------------------------------------------------
 
@@ -245,7 +239,7 @@ insert into dist_table4 select s from generate_series(1,100) s;
 select count(*) as total from dist_table4;
  total
 ---------------------------------------------------------------------
-       100
+   100
 (1 row)
 
 -- verify we do not allow foreign keys from distributed table to citus local table concurrently
@@ -295,13 +289,13 @@ select count(*) from test_columnar;
 select id from test_columnar where id = 1;
  id
 ---------------------------------------------------------------------
-   1
+  1
 (1 row)
 
 select id from test_columnar where id = 51;
  id
 ---------------------------------------------------------------------
-   51
+ 51
 (1 row)
 
 select count(*) from test_columnar_1;

src/test/regress/sql/create_distributed_table_concurrently.sql

@@ -28,6 +28,14 @@ set citus.shard_replication_factor to 2;
 select create_distributed_table_concurrently('test','key', 'hash');
 set citus.shard_replication_factor to 1;
 
+set citus.shard_replication_factor to 2;
+create table dist_1(a int);
+select create_distributed_table('dist_1', 'a');
+set citus.shard_replication_factor to 1;
+
+create table dist_2(a int);
+select create_distributed_table_concurrently('dist_2', 'a', colocate_with=>'dist_1');
+
 begin;
 select create_distributed_table_concurrently('test','key');
 rollback;
@@ -63,6 +71,7 @@ rollback;
 
 -- verify that we can undistribute the table
 begin;
+set local client_min_messages to warning;
 select undistribute_table('test', cascade_via_foreign_keys := true);
 rollback;