external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle new option colliculocale in CREATE COLLATION logic 

In PG15, there is an added option to use ICU as global locale provider.
pg_collation has three locale-related fields: collcollate and collctype,
which are libc-related fields, and a new one colliculocale, which is the
ICU-related field. Only the libc-related fields or the ICU-related field
is set, never both.

Relevant PG commits:
f2553d43060edb210b36c63187d52a632448e1d2
54637508f87bd5f07fb9406bac6b08240283be3b
multi_shard_json_table
naisila 2022-07-27 12:07:04 +03:00
parent fbdcdc63c3
commit 12640e51de
4 changed files with 432 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
src/backend/distributed/commands/collation.c
View File

@ -63,15 +63,53 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati
const char *collname = NameStr(collationForm->collname); const char *collname = NameStr(collationForm->collname);
bool collisdeterministic = collationForm->collisdeterministic; bool collisdeterministic = collationForm->collisdeterministic;
char *collcollate;
char *collctype;
#if PG_VERSION_NUM >= PG_VERSION_15 #if PG_VERSION_NUM >= PG_VERSION_15
/*
* In PG15, there is an added option to use ICU as global locale provider.
* pg_collation has three locale-related fields: collcollate and collctype,
* which are libc-related fields, and a new one colliculocale, which is the
* ICU-related field. Only the libc-related fields or the ICU-related field
* is set, never both.
*/
char *colliculocale;
bool isnull; bool isnull;
Datum datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collcollate, Datum datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collcollate,
&isnull); &isnull);
Assert(!isnull); if (!isnull)
char *collcollate = TextDatumGetCString(datum); {
collcollate = TextDatumGetCString(datum);
}
else
{
collcollate = NULL;
}
datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collctype, &isnull); datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collctype, &isnull);
Assert(!isnull); if (!isnull)
char *collctype = TextDatumGetCString(datum); {
collctype = TextDatumGetCString(datum);
}
else
{
collctype = NULL;
}
datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_colliculocale, &isnull);
if (!isnull)
{
colliculocale = TextDatumGetCString(datum);
}
else
{
colliculocale = NULL;
}
AssertArg((collcollate && collctype) || colliculocale);
#else #else
/* /*
@ -79,8 +117,8 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati
* pstrdup() to match the interface of 15 so that we consistently free the * pstrdup() to match the interface of 15 so that we consistently free the
* result later. * result later.
*/ */
char *collcollate = pstrdup(NameStr(collationForm->collcollate)); collcollate = pstrdup(NameStr(collationForm->collcollate));
char *collctype = pstrdup(NameStr(collationForm->collctype)); collctype = pstrdup(NameStr(collationForm->collctype));
#endif #endif
if (collowner != NULL) if (collowner != NULL)
@ -106,6 +144,33 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati
"CREATE COLLATION %s (provider = '%s'", "CREATE COLLATION %s (provider = '%s'",
*quotedCollationName, providerString); *quotedCollationName, providerString);
#if PG_VERSION_NUM >= PG_VERSION_15
if (colliculocale)
{
appendStringInfo(&collationNameDef,
", locale = %s",
quote_literal_cstr(colliculocale));
pfree(colliculocale);
}
else
{
if (strcmp(collcollate, collctype) == 0)
{
appendStringInfo(&collationNameDef,
", locale = %s",
quote_literal_cstr(collcollate));
}
else
{
appendStringInfo(&collationNameDef,
", lc_collate = %s, lc_ctype = %s",
quote_literal_cstr(collcollate),
quote_literal_cstr(collctype));
}
pfree(collcollate);
pfree(collctype);
}
#else
if (strcmp(collcollate, collctype) == 0) if (strcmp(collcollate, collctype) == 0)
{ {
appendStringInfo(&collationNameDef, appendStringInfo(&collationNameDef,
@ -122,6 +187,7 @@ CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollati
pfree(collcollate); pfree(collcollate);
pfree(collctype); pfree(collctype);
#endif
if (!collisdeterministic) if (!collisdeterministic)
{ {

34
src/test/regress/expected/distributed_collations.out
View File

@ -1,8 +1,42 @@
SET citus.next_shard_id TO 20050000; SET citus.next_shard_id TO 20050000;
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 14 AS server_version_above_fourteen
\gset
CREATE USER collationuser; CREATE USER collationuser;
CREATE SCHEMA collation_tests AUTHORIZATION collationuser; CREATE SCHEMA collation_tests AUTHORIZATION collationuser;
CREATE SCHEMA collation_tests2 AUTHORIZATION collationuser; CREATE SCHEMA collation_tests2 AUTHORIZATION collationuser;
SET search_path to collation_tests; SET search_path to collation_tests;
\if :server_version_above_fourteen
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk');
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk', lc_ctype = 'de-u-co-phonebk');
-- works
CREATE COLLATION german_phonebook_test (provider = icu, locale = 'de-u-co-phonebk');
-- with icu provider, colliculocale will be set, collcollate and collctype will be null
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
DROP COLLATION german_phonebook_test;
-- with non-icu provider, colliculocale will be null, collcollate and collctype will be set
CREATE COLLATION default_provider (provider = libc, lc_collate = "POSIX", lc_ctype = "POSIX");
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''default_provider'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''default_provider'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider'';
');
DROP COLLATION default_provider;
\endif
CREATE COLLATION german_phonebook (provider = icu, locale = 'de-u-co-phonebk'); CREATE COLLATION german_phonebook (provider = icu, locale = 'de-u-co-phonebk');
SET citus.enable_ddl_propagation TO off; SET citus.enable_ddl_propagation TO off;
CREATE COLLATION german_phonebook_unpropagated (provider = icu, locale = 'de-u-co-phonebk'); CREATE COLLATION german_phonebook_unpropagated (provider = icu, locale = 'de-u-co-phonebk');

281
src/test/regress/expected/distributed_collations_0.out Normal file
View File

@ -0,0 +1,281 @@
SET citus.next_shard_id TO 20050000;
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 14 AS server_version_above_fourteen
\gset
CREATE USER collationuser;
CREATE SCHEMA collation_tests AUTHORIZATION collationuser;
CREATE SCHEMA collation_tests2 AUTHORIZATION collationuser;
SET search_path to collation_tests;
\if :server_version_above_fourteen
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk');
ERROR: parameter "locale" must be specified
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk', lc_ctype = 'de-u-co-phonebk');
ERROR: parameter "locale" must be specified
-- works
CREATE COLLATION german_phonebook_test (provider = icu, locale = 'de-u-co-phonebk');
-- with icu provider, colliculocale will be set, collcollate and collctype will be null
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
result
---------------------------------------------------------------------
(3 rows)
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
result
---------------------------------------------------------------------
(3 rows)
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
result
---------------------------------------------------------------------
de-u-co-phonebk
de-u-co-phonebk
de-u-co-phonebk
(3 rows)
DROP COLLATION german_phonebook_test;
-- with non-icu provider, colliculocale will be null, collcollate and collctype will be set
CREATE COLLATION default_provider (provider = libc, lc_collate = "POSIX", lc_ctype = "POSIX");
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''default_provider'';
');
result
---------------------------------------------------------------------
POSIX
POSIX
POSIX
(3 rows)
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''default_provider'';
');
result
---------------------------------------------------------------------
POSIX
POSIX
POSIX
(3 rows)
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider'';
');
result
---------------------------------------------------------------------
(3 rows)
DROP COLLATION default_provider;
\endif
CREATE COLLATION german_phonebook (provider = icu, locale = 'de-u-co-phonebk');
SET citus.enable_ddl_propagation TO off;
CREATE COLLATION german_phonebook_unpropagated (provider = icu, locale = 'de-u-co-phonebk');
SET citus.enable_ddl_propagation TO on;
\c - - - :worker_1_port
SELECT c.collname, nsp.nspname, a.rolname
FROM pg_collation c
JOIN pg_namespace nsp ON nsp.oid = c.collnamespace
JOIN pg_authid a ON a.oid = c.collowner
WHERE collname like 'german_phonebook%'
ORDER BY 1,2,3;
collname | nspname | rolname
---------------------------------------------------------------------
german_phonebook | collation_tests | postgres
(1 row)
\c - - - :master_port
SET search_path to collation_tests;
CREATE TABLE test_propagate(id int, t1 text COLLATE german_phonebook,
t2 text COLLATE german_phonebook_unpropagated);
INSERT INTO test_propagate VALUES (1, 'aesop', U&'\00E4sop'), (2, U&'Vo\1E9Er', 'Vossr');
SELECT create_distributed_table('test_propagate', 'id');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$collation_tests.test_propagate$$)
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- Test COLLATE is pushed down
SELECT * FROM collation_tests.test_propagate WHERE t2 < 'b';
id | t1 | t2
---------------------------------------------------------------------
1 | aesop | äsop
(1 row)
SELECT * FROM collation_tests.test_propagate WHERE t2 < 'b' COLLATE "C";
id | t1 | t2
---------------------------------------------------------------------
2 | Voẞr | Vossr
(1 row)
SELECT * FROM collation_tests.test_propagate WHERE t2 COLLATE "C" < 'b';
id | t1 | t2
---------------------------------------------------------------------
2 | Voẞr | Vossr
(1 row)
-- Test COLLATE is pushed down with aggregate function
SET citus.next_shard_id TO 20060000;
CREATE TABLE test_collate_pushed_down_aggregate (a int, b int);
SELECT create_distributed_table('test_collate_pushed_down_aggregate', 'a');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT alter_distributed_table('test_collate_pushed_down_aggregate', shard_count := 2, cascade_to_colocated:=false);
NOTICE: creating a new table for collation_tests.test_collate_pushed_down_aggregate
NOTICE: moving the data of collation_tests.test_collate_pushed_down_aggregate
NOTICE: dropping the old collation_tests.test_collate_pushed_down_aggregate
NOTICE: renaming the new table to collation_tests.test_collate_pushed_down_aggregate
alter_distributed_table
---------------------------------------------------------------------
(1 row)
SET citus.log_remote_commands TO true;
SELECT ALL MIN((lower(CAST(test_collate_pushed_down_aggregate.a AS VARCHAR)) COLLATE "C"))
FROM ONLY test_collate_pushed_down_aggregate;
NOTICE: issuing SELECT min((lower(((a)::character varying COLLATE "default")) COLLATE "C")) AS min FROM ONLY collation_tests.test_collate_pushed_down_aggregate_20060004 test_collate_pushed_down_aggregate WHERE true
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
NOTICE: issuing SELECT min((lower(((a)::character varying COLLATE "default")) COLLATE "C")) AS min FROM ONLY collation_tests.test_collate_pushed_down_aggregate_20060005 test_collate_pushed_down_aggregate WHERE true
DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx
min
---------------------------------------------------------------------
(1 row)
RESET citus.log_remote_commands;
-- Test range table with collated distribution column
CREATE TABLE test_range(key text COLLATE german_phonebook, val int);
SELECT create_distributed_table('test_range', 'key', 'range');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT master_create_empty_shard('test_range') AS new_shard_id
\gset
UPDATE pg_dist_shard SET shardminvalue = 'a', shardmaxvalue = 'f'
WHERE shardid = :new_shard_id;
SELECT master_create_empty_shard('test_range') AS new_shard_id
\gset
UPDATE pg_dist_shard SET shardminvalue = 'G', shardmaxvalue = 'Z'
WHERE shardid = :new_shard_id;
-- without german_phonebook collation, this would fail
INSERT INTO test_range VALUES (U&'\00E4sop', 1), (U&'Vo\1E9Er', 2);
-- without german_phonebook collation, this would not be router executable
SET client_min_messages TO debug;
SELECT * FROM test_range WHERE key > 'Ab' AND key < U&'\00E4z';
DEBUG: Creating router plan
key | val
---------------------------------------------------------------------
äsop | 1
(1 row)
\c - - - :worker_1_port
SELECT c.collname, nsp.nspname, a.rolname
FROM pg_collation c
JOIN pg_namespace nsp ON nsp.oid = c.collnamespace
JOIN pg_authid a ON a.oid = c.collowner
WHERE collname like 'german_phonebook%'
ORDER BY 1,2,3;
collname | nspname | rolname
---------------------------------------------------------------------
german_phonebook | collation_tests | postgres
german_phonebook_unpropagated | collation_tests | postgres
(2 rows)
\c - - - :master_port
ALTER COLLATION collation_tests.german_phonebook RENAME TO german_phonebook2;
ALTER COLLATION collation_tests.german_phonebook2 SET SCHEMA collation_tests2;
ALTER COLLATION collation_tests2.german_phonebook2 OWNER TO collationuser;
\c - - - :worker_1_port
SELECT c.collname, nsp.nspname, a.rolname
FROM pg_collation c
JOIN pg_namespace nsp ON nsp.oid = c.collnamespace
JOIN pg_authid a ON a.oid = c.collowner
WHERE collname like 'german_phonebook%'
ORDER BY 1,2,3;
collname | nspname | rolname
---------------------------------------------------------------------
german_phonebook2 | collation_tests2 | collationuser
german_phonebook_unpropagated | collation_tests | postgres
(2 rows)
\c - - - :master_port
SET client_min_messages TO error; -- suppress cascading objects dropping
DROP SCHEMA collation_tests CASCADE;
DROP SCHEMA collation_tests2 CASCADE;
DROP USER collationuser;
\c - - - :worker_1_port
-- test creating a collation on a worker
CREATE COLLATION another_german_phonebook (provider = icu, locale = 'de-u-co-phonebk');
ERROR: operation is not allowed on this node
HINT: Connect to the coordinator and run it again.
-- test if creating a collation on a worker on a local
-- schema raises the right error
SET citus.enable_ddl_propagation TO off;
CREATE SCHEMA collation_creation_on_worker;
SET citus.enable_ddl_propagation TO on;
CREATE COLLATION collation_creation_on_worker.another_german_phonebook (provider = icu, locale = 'de-u-co-phonebk');
ERROR: operation is not allowed on this node
HINT: Connect to the coordinator and run it again.
SET citus.enable_ddl_propagation TO off;
DROP SCHEMA collation_creation_on_worker;
SET citus.enable_ddl_propagation TO on;
\c - - - :master_port
-- will skip trying to propagate the collation due to temp schema
CREATE COLLATION pg_temp.temp_collation (provider = icu, locale = 'de-u-co-phonebk');
WARNING: "collation pg_temp_xxx.temp_collation" has dependency on unsupported object "schema pg_temp_xxx"
DETAIL: "collation pg_temp_xxx.temp_collation" will be created only locally
SET client_min_messages TO ERROR;
CREATE USER alter_collation_user;
SELECT 1 FROM run_command_on_workers('CREATE USER alter_collation_user');
?column?
---------------------------------------------------------------------
1
1
(2 rows)
RESET client_min_messages;
CREATE COLLATION alter_collation FROM "C";
ALTER COLLATION alter_collation OWNER TO alter_collation_user;
SELECT result FROM run_command_on_all_nodes('
SELECT collowner::regrole FROM pg_collation WHERE collname = ''alter_collation'';
');
result
---------------------------------------------------------------------
alter_collation_user
alter_collation_user
alter_collation_user
(3 rows)
DROP COLLATION alter_collation;
SET client_min_messages TO ERROR;
DROP USER alter_collation_user;
SELECT 1 FROM run_command_on_workers('DROP USER alter_collation_user');
?column?
---------------------------------------------------------------------
1
1
(2 rows)
RESET client_min_messages;

45
src/test/regress/sql/distributed_collations.sql
View File

@ -1,5 +1,9 @@
SET citus.next_shard_id TO 20050000; SET citus.next_shard_id TO 20050000;
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 14 AS server_version_above_fourteen
\gset
CREATE USER collationuser; CREATE USER collationuser;
CREATE SCHEMA collation_tests AUTHORIZATION collationuser; CREATE SCHEMA collation_tests AUTHORIZATION collationuser;
@ -7,6 +11,47 @@ CREATE SCHEMA collation_tests2 AUTHORIZATION collationuser;
SET search_path to collation_tests; SET search_path to collation_tests;
\if :server_version_above_fourteen
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk');
-- fail, needs "locale"
CREATE COLLATION german_phonebook_test (provider = icu, lc_collate = 'de-u-co-phonebk', lc_ctype = 'de-u-co-phonebk');
-- works
CREATE COLLATION german_phonebook_test (provider = icu, locale = 'de-u-co-phonebk');
-- with icu provider, colliculocale will be set, collcollate and collctype will be null
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''german_phonebook_test'';
');
DROP COLLATION german_phonebook_test;
-- with non-icu provider, colliculocale will be null, collcollate and collctype will be set
CREATE COLLATION default_provider (provider = libc, lc_collate = "POSIX", lc_ctype = "POSIX");
SELECT result FROM run_command_on_all_nodes('
SELECT collcollate FROM pg_collation WHERE collname = ''default_provider'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT collctype FROM pg_collation WHERE collname = ''default_provider'';
');
SELECT result FROM run_command_on_all_nodes('
SELECT colliculocale FROM pg_collation WHERE collname = ''default_provider'';
');
DROP COLLATION default_provider;
\endif
CREATE COLLATION german_phonebook (provider = icu, locale = 'de-u-co-phonebk'); CREATE COLLATION german_phonebook (provider = icu, locale = 'de-u-co-phonebk');
SET citus.enable_ddl_propagation TO off; SET citus.enable_ddl_propagation TO off;