external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert citus.hide_shards_from_app_name_prefixes to citus.show_shards_for_app_name_prefixes

release-11.0-onder_6_may_2
Marco Slot 2022-05-02 22:41:19 +02:00
parent b4a65b9c45
commit e8b41d1e5b
15 changed files with 146 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/distributed/citus.control
View File

@ -1,6 +1,6 @@
# Citus extension
comment = 'Citus distributed database'
default_version = '11.0-1'
default_version = '11.0-2'
module_pathname = '$libdir/citus'
relocatable = false
schema = pg_catalog

55
src/backend/distributed/shared_library_init.c
View File

@ -127,9 +127,9 @@ static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra,
static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source);
static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source);
static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source);
static bool HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra,
static bool ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra,
GucSource source);
static void HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra);
static void ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra);
static void ApplicationNameAssignHook(const char *newval, void *extra);
static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source);
static void NodeConninfoGucAssignHook(const char *newval, void *extra);
@ -1121,24 +1121,6 @@ RegisterCitusConfigVariables(void)
GUC_NO_SHOW_ALL,
NULL, NULL, NULL);
DefineCustomStringVariable(
"citus.hide_shards_from_app_name_prefixes",
gettext_noop("If application_name starts with one of these values, hide shards"),
gettext_noop("Citus places distributed tables and shards in the same schema. "
"That can cause confusion when inspecting the list of tables on "
"a node with shards. This GUC can be used to hide the shards from "
"pg_class for certain applications based on the application_name "
"of the connection. The default is *, which hides shards from all "
"applications. This behaviour can be overridden using the "
"citus.override_table_visibility setting"),
&HideShardsFromAppNamePrefixes,
"*",
PGC_USERSET,
GUC_STANDARD,
HideShardsFromAppNamePrefixesCheckHook,
HideShardsFromAppNamePrefixesAssignHook,
NULL);
DefineCustomIntVariable(
"citus.isolation_test_session_process_id",
NULL,
@ -1677,6 +1659,25 @@ RegisterCitusConfigVariables(void)
GUC_STANDARD,
NULL, NULL, NULL);
DefineCustomStringVariable(
"citus.show_shards_for_app_name_prefixes",
gettext_noop("If application_name starts with one of these values, show shards"),
gettext_noop("Citus places distributed tables and shards in the same schema. "
"That can cause confusion when inspecting the list of tables on "
"a node with shards. By default the shards are hidden from "
"pg_class. This GUC can be used to show the shards to certain "
"applications based on the application_name of the connection. "
"The default is empty string, which hides shards from all "
"applications. This behaviour can be overridden using the "
"citus.override_table_visibility setting"),
&ShowShardsForAppNamePrefixes,
"",
PGC_USERSET,
GUC_STANDARD,
ShowShardsForAppNamePrefixesCheckHook,
ShowShardsForAppNamePrefixesAssignHook,
NULL);
DefineCustomBoolVariable(
"citus.sort_returning",
gettext_noop("Sorts the RETURNING clause to get consistent test output"),
@ -1946,12 +1947,12 @@ WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source)
/*
* HideShardsFromAppNamePrefixesCheckHook ensures that the
* citus.hide_shards_from_app_name_prefixes holds a valid list of application_name
* ShowShardsForAppNamePrefixesCheckHook ensures that the
* citus.show_shards_for_app_name_prefixes holds a valid list of application_name
* values.
*/
static bool
HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source)
ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source)
{
List *prefixList = NIL;
@ -1981,7 +1982,7 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so
if (strcmp(prefixAscii, appNamePrefix) != 0)
{
GUC_check_errdetail("prefix %s in citus.hide_shards_from_app_name_prefixes "
GUC_check_errdetail("prefix %s in citus.show_shards_for_app_name_prefixes "
"contains non-ascii characters", appNamePrefix);
return false;
}
@ -1992,12 +1993,12 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so
/*
* HideShardsFromAppNamePrefixesAssignHook ensures changes to
* citus.hide_shards_from_app_name_prefixes are reflected in the decision
* ShowShardsForAppNamePrefixesAssignHook ensures changes to
* citus.show_shards_for_app_name_prefixes are reflected in the decision
* whether or not to show shards.
*/
static void
HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra)
ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra)
{
ResetHideShardsDecision();
}

2
src/backend/distributed/sql/citus--11.0-1--11.0-2.sql Normal file
View File

@ -0,0 +1,2 @@
#include "udfs/citus_shards_on_worker/11.0-2.sql"
#include "udfs/citus_shard_indexes_on_worker/11.0-2.sql"

2
src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql Normal file
View File

@ -0,0 +1,2 @@
#include "../udfs/citus_shards_on_worker/11.0-1.sql"
#include "../udfs/citus_shard_indexes_on_worker/11.0-1.sql"

39
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql generated Normal file
View File

@ -0,0 +1,39 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT schema_name name,
OUT index_name name,
OUT table_type text,
OUT owner_name name,
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
c2.relname as "Table"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
WHERE c.relkind IN ('i','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS
SELECT schema_name as "Schema",
index_name as "Name",
table_type as "Type",
owner_name as "Owner",
shard_name as "Table"
FROM pg_catalog.citus_shard_indexes_on_worker() s;

2
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql
View File

@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible

34
src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql generated Normal file
View File

@ -0,0 +1,34 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT schema_name name,
OUT shard_name name,
OUT table_type text,
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','p','v','m','S','f','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS
SELECT schema_name as "Schema",
shard_name as "Name",
table_type as "Type",
owner_name as "Owner"
FROM pg_catalog.citus_shards_on_worker() s;

2
src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql
View File

@ -5,7 +5,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible

23
src/backend/distributed/worker/worker_shard_visibility.c
View File

@ -40,8 +40,8 @@ typedef enum HideShardsMode
bool OverrideTableVisibility = true;
bool EnableManualChangesToShards = false;
/* hide shards when the application_name starts with one of: */
char *HideShardsFromAppNamePrefixes = "*";
/* show shards when the application_name starts with one of: */
char *ShowShardsForAppNamePrefixes = "";
/* cache of whether or not to hide shards */
static HideShardsMode HideShards = CHECK_APPLICATION_NAME;
@ -271,8 +271,8 @@ RelationIsAKnownShard(Oid shardRelationId)
/*
* HideShardsFromSomeApplications transforms queries to pg_class to
* filter out known shards if the application_name matches any of
* the prefixes in citus.hide_shards_from_app_name_prefixes.
* filter out known shards if the application_name does not match any of
* the prefixes in citus.show_shards_for_app_name_prefix.
*/
void
HideShardsFromSomeApplications(Query *query)
@ -294,7 +294,7 @@ HideShardsFromSomeApplications(Query *query)
* ShouldHideShards returns whether we should hide shards in the current
* session. It only checks the application_name once and then uses a
* cached response unless either the application_name or
* citus.hide_shards_from_app_name_prefixes changes.
* citus.show_shards_for_app_name_prefix changes.
*/
static bool
ShouldHideShards(void)
@ -367,32 +367,33 @@ ShouldHideShardsInternal(void)
List *prefixList = NIL;
/* SplitGUCList scribbles on the input */
char *splitCopy = pstrdup(HideShardsFromAppNamePrefixes);
char *splitCopy = pstrdup(ShowShardsForAppNamePrefixes);
if (!SplitGUCList(splitCopy, ',', &prefixList))
{
/* invalid GUC value, ignore */
return false;
return true;
}
char *appNamePrefix = NULL;
foreach_ptr(appNamePrefix, prefixList)
{
/* always hide shards when one of the prefixes is * */
/* never hide shards when one of the prefixes is * */
if (strcmp(appNamePrefix, "*") == 0)
{
return true;
return false;
}
/* compare only the first first <prefixLength> characters */
int prefixLength = strlen(appNamePrefix);
if (strncmp(application_name, appNamePrefix, prefixLength) == 0)
{
return true;
return false;
}
}
return false;
/* default behaviour: hide shards */
return true;
}

2
src/include/distributed/worker_shard_visibility.h
View File

@ -15,7 +15,7 @@
extern bool OverrideTableVisibility;
extern bool EnableManualChangesToShards;
extern char *HideShardsFromAppNamePrefixes;
extern char *ShowShardsForAppNamePrefixes;
extern void HideShardsFromSomeApplications(Query *query);

7
src/test/regress/expected/multi_extension.out
View File

@ -1031,6 +1031,13 @@ SELECT * FROM multi_extension.print_extension_changes();
| view citus_stat_activity
(41 rows)
-- Snapshot of state at 11.0-2
ALTER EXTENSION citus UPDATE TO '11.0-2';
SELECT * FROM multi_extension.print_extension_changes();
previous_object | current_object
---------------------------------------------------------------------
(0 rows)
DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
-- show running version
SHOW citus.version;

12
src/test/regress/expected/multi_mx_hide_shard_names.out
View File

@ -114,7 +114,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
(2 rows)
-- changing application_name reveals the shards
SET application_name TO '';
SET application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
@ -137,7 +137,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SET LOCAL application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
@ -160,7 +160,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
SET application_name TO 'pg_regress';
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
@ -173,7 +173,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
SET application_name TO 'pg_regress';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
@ -196,9 +196,9 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
(2 rows)
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
-- changing citus.show_shards_for_app_name_prefix reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------

2
src/test/regress/pg_regress_multi.pl
View File

@ -466,7 +466,7 @@ push(@pgOptions, "citus.explain_analyze_sort_method='taskId'");
push(@pgOptions, "citus.enable_manual_changes_to_shards=on");
# Some tests look at shards in pg_class, make sure we can usually see them:
push(@pgOptions, "citus.hide_shards_from_app_name_prefixes='psql,pg_dump'");
push(@pgOptions, "citus.show_shards_for_app_name_prefixes='pg_regress'");
# we disable slow start by default to encourage parallelism within tests
push(@pgOptions, "citus.executor_slow_start_interval=0ms");

4
src/test/regress/sql/multi_extension.sql
View File

@ -459,6 +459,10 @@ SELECT * FROM multi_extension.print_extension_changes();
ALTER EXTENSION citus UPDATE TO '11.0-1';
SELECT * FROM multi_extension.print_extension_changes();
-- Snapshot of state at 11.0-2
ALTER EXTENSION citus UPDATE TO '11.0-2';
SELECT * FROM multi_extension.print_extension_changes();
DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
-- show running version

12
src/test/regress/sql/multi_mx_hide_shard_names.sql
View File

@ -67,7 +67,7 @@ SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- changing application_name reveals the shards
SET application_name TO '';
SET application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
RESET application_name;
@ -76,7 +76,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SET LOCAL application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
@ -85,7 +85,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
SET application_name TO 'pg_regress';
ROLLBACK;
-- shards are hidden again after GUCs are reset
@ -94,7 +94,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
SET application_name TO 'pg_regress';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK TO SAVEPOINT s1;
@ -102,9 +102,9 @@ ROLLBACK TO SAVEPOINT s1;
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
-- changing citus.show_shards_for_app_name_prefix reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;