external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5920 from citusdata/marcocitus/show-shards-guc

pull/5925/head
Marco Slot 2022-05-03 15:00:09 +02:00 committed by GitHub
parent 3e1180de78 ceb593c9da
commit 0e1e2275f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 134 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
src/backend/distributed/shared_library_init.c
View File

@ -159,9 +159,9 @@ static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra,
static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source);
static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source);
static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source);
static bool HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra,
GucSource source);
static void HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra);
static bool ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra,
GucSource source);
static void ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra);
static void ApplicationNameAssignHook(const char *newval, void *extra);
static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source);
static void NodeConninfoGucAssignHook(const char *newval, void *extra);
@ -1174,24 +1174,6 @@ RegisterCitusConfigVariables(void)
GUC_NO_SHOW_ALL,
NULL, NULL, NULL);
DefineCustomStringVariable(
"citus.hide_shards_from_app_name_prefixes",
gettext_noop("If application_name starts with one of these values, hide shards"),
gettext_noop("Citus places distributed tables and shards in the same schema. "
"That can cause confusion when inspecting the list of tables on "
"a node with shards. This GUC can be used to hide the shards from "
"pg_class for certain applications based on the application_name "
"of the connection. The default is *, which hides shards from all "
"applications. This behaviour can be overridden using the "
"citus.override_table_visibility setting"),
&HideShardsFromAppNamePrefixes,
"*",
PGC_USERSET,
GUC_STANDARD,
HideShardsFromAppNamePrefixesCheckHook,
HideShardsFromAppNamePrefixesAssignHook,
NULL);
DefineCustomIntVariable(
"citus.isolation_test_session_process_id",
NULL,
@ -1716,6 +1698,25 @@ RegisterCitusConfigVariables(void)
GUC_STANDARD,
NULL, NULL, NULL);
DefineCustomStringVariable(
"citus.show_shards_for_app_name_prefixes",
gettext_noop("If application_name starts with one of these values, show shards"),
gettext_noop("Citus places distributed tables and shards in the same schema. "
"That can cause confusion when inspecting the list of tables on "
"a node with shards. By default the shards are hidden from "
"pg_class. This GUC can be used to show the shards to certain "
"applications based on the application_name of the connection. "
"The default is empty string, which hides shards from all "
"applications. This behaviour can be overridden using the "
"citus.override_table_visibility setting"),
&ShowShardsForAppNamePrefixes,
"",
PGC_USERSET,
GUC_STANDARD,
ShowShardsForAppNamePrefixesCheckHook,
ShowShardsForAppNamePrefixesAssignHook,
NULL);
DefineCustomBoolVariable(
"citus.sort_returning",
gettext_noop("Sorts the RETURNING clause to get consistent test output"),
@ -1985,12 +1986,12 @@ WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source)
/*
* HideShardsFromAppNamePrefixesCheckHook ensures that the
* citus.hide_shards_from_app_name_prefixes holds a valid list of application_name
* ShowShardsForAppNamePrefixesCheckHook ensures that the
* citus.show_shards_for_app_name_prefixes holds a valid list of application_name
* values.
*/
static bool
HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source)
ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source)
{
List *prefixList = NIL;
@ -2020,7 +2021,7 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so
if (strcmp(prefixAscii, appNamePrefix) != 0)
{
GUC_check_errdetail("prefix %s in citus.hide_shards_from_app_name_prefixes "
GUC_check_errdetail("prefix %s in citus.show_shards_for_app_name_prefixes "
"contains non-ascii characters", appNamePrefix);
return false;
}
@ -2031,12 +2032,12 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so
/*
* HideShardsFromAppNamePrefixesAssignHook ensures changes to
* citus.hide_shards_from_app_name_prefixes are reflected in the decision
* ShowShardsForAppNamePrefixesAssignHook ensures changes to
* citus.show_shards_for_app_name_prefixes are reflected in the decision
* whether or not to show shards.
*/
static void
HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra)
ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra)
{
ResetHideShardsDecision();
}

3
src/backend/distributed/sql/citus--11.0-1--11.0-2.sql
View File

@ -1 +1,2 @@
-- bump version to 11.0-2
#include "udfs/citus_shards_on_worker/11.0-2.sql"
#include "udfs/citus_shard_indexes_on_worker/11.0-2.sql"

3
src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql
View File

@ -1 +1,2 @@
-- bump down version to 11.0-1
#include "../udfs/citus_shards_on_worker/11.0-1.sql"
#include "../udfs/citus_shard_indexes_on_worker/11.0-1.sql"

39
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql generated Normal file
View File

@ -0,0 +1,39 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT schema_name name,
OUT index_name name,
OUT table_type text,
OUT owner_name name,
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner",
c2.relname as "Table"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
WHERE c.relkind IN ('i','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS
SELECT schema_name as "Schema",
index_name as "Name",
table_type as "Type",
owner_name as "Owner",
shard_name as "Table"
FROM pg_catalog.citus_shard_indexes_on_worker() s;

2
src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql
View File

@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker(
OUT shard_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \di produces, except pg_table_is_visible

34
src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql generated Normal file
View File

@ -0,0 +1,34 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT schema_name name,
OUT shard_name name,
OUT table_type text,
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible
-- is replaced with pg_catalog.relation_is_a_known_shard(c.oid)
RETURN QUERY
SELECT n.nspname as "Schema",
c.relname as "Name",
CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type",
pg_catalog.pg_get_userbyid(c.relowner) as "Owner"
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind IN ('r','p','v','m','S','f','')
AND n.nspname <> 'pg_catalog'
AND n.nspname <> 'information_schema'
AND n.nspname !~ '^pg_toast'
AND pg_catalog.relation_is_a_known_shard(c.oid)
ORDER BY 1,2;
END;
$$;
CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS
SELECT schema_name as "Schema",
shard_name as "Name",
table_type as "Type",
owner_name as "Owner"
FROM pg_catalog.citus_shards_on_worker() s;

2
src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql
View File

@ -5,7 +5,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker(
OUT owner_name name)
RETURNS SETOF record
LANGUAGE plpgsql
SET citus.hide_shards_from_app_name_prefixes = ''
SET citus.show_shards_for_app_name_prefixes = '*'
AS $$
BEGIN
-- this is the query that \d produces, except pg_table_is_visible

23
src/backend/distributed/worker/worker_shard_visibility.c
View File

@ -40,8 +40,8 @@ typedef enum HideShardsMode
bool OverrideTableVisibility = true;
bool EnableManualChangesToShards = false;
/* hide shards when the application_name starts with one of: */
char *HideShardsFromAppNamePrefixes = "*";
/* show shards when the application_name starts with one of: */
char *ShowShardsForAppNamePrefixes = "";
/* cache of whether or not to hide shards */
static HideShardsMode HideShards = CHECK_APPLICATION_NAME;
@ -271,8 +271,8 @@ RelationIsAKnownShard(Oid shardRelationId)
/*
* HideShardsFromSomeApplications transforms queries to pg_class to
* filter out known shards if the application_name matches any of
* the prefixes in citus.hide_shards_from_app_name_prefixes.
* filter out known shards if the application_name does not match any of
* the prefixes in citus.show_shards_for_app_name_prefix.
*/
void
HideShardsFromSomeApplications(Query *query)
@ -294,7 +294,7 @@ HideShardsFromSomeApplications(Query *query)
* ShouldHideShards returns whether we should hide shards in the current
* session. It only checks the application_name once and then uses a
* cached response unless either the application_name or
* citus.hide_shards_from_app_name_prefixes changes.
* citus.show_shards_for_app_name_prefix changes.
*/
static bool
ShouldHideShards(void)
@ -367,32 +367,33 @@ ShouldHideShardsInternal(void)
List *prefixList = NIL;
/* SplitGUCList scribbles on the input */
char *splitCopy = pstrdup(HideShardsFromAppNamePrefixes);
char *splitCopy = pstrdup(ShowShardsForAppNamePrefixes);
if (!SplitGUCList(splitCopy, ',', &prefixList))
{
/* invalid GUC value, ignore */
return false;
return true;
}
char *appNamePrefix = NULL;
foreach_ptr(appNamePrefix, prefixList)
{
/* always hide shards when one of the prefixes is * */
/* never hide shards when one of the prefixes is * */
if (strcmp(appNamePrefix, "*") == 0)
{
return true;
return false;
}
/* compare only the first first <prefixLength> characters */
int prefixLength = strlen(appNamePrefix);
if (strncmp(application_name, appNamePrefix, prefixLength) == 0)
{
return true;
return false;
}
}
return false;
/* default behaviour: hide shards */
return true;
}

2
src/include/distributed/worker_shard_visibility.h
View File

@ -15,7 +15,7 @@
extern bool OverrideTableVisibility;
extern bool EnableManualChangesToShards;
extern char *HideShardsFromAppNamePrefixes;
extern char *ShowShardsForAppNamePrefixes;
extern void HideShardsFromSomeApplications(Query *query);

12
src/test/regress/expected/multi_mx_hide_shard_names.out
View File

@ -114,7 +114,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
(2 rows)
-- changing application_name reveals the shards
SET application_name TO '';
SET application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
@ -137,7 +137,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SET LOCAL application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------
@ -160,7 +160,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
SET application_name TO 'pg_regress';
ROLLBACK;
-- shards are hidden again after GUCs are reset
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
@ -173,7 +173,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
SET application_name TO 'pg_regress';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
@ -196,9 +196,9 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
(2 rows)
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
-- changing citus.show_shards_for_app_name_prefix reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
relname
---------------------------------------------------------------------

2
src/test/regress/pg_regress_multi.pl
View File

@ -466,7 +466,7 @@ push(@pgOptions, "citus.explain_analyze_sort_method='taskId'");
push(@pgOptions, "citus.enable_manual_changes_to_shards=on");
# Some tests look at shards in pg_class, make sure we can usually see them:
push(@pgOptions, "citus.hide_shards_from_app_name_prefixes='psql,pg_dump'");
push(@pgOptions, "citus.show_shards_for_app_name_prefixes='pg_regress'");
# we disable slow start by default to encourage parallelism within tests
push(@pgOptions, "citus.executor_slow_start_interval=0ms");

12
src/test/regress/sql/multi_mx_hide_shard_names.sql
View File

@ -67,7 +67,7 @@ SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_name
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
-- changing application_name reveals the shards
SET application_name TO '';
SET application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
RESET application_name;
@ -76,7 +76,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- changing application_name in transaction reveals the shards
BEGIN;
SET LOCAL application_name TO '';
SET LOCAL application_name TO 'pg_regress';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
@ -85,7 +85,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- now with session-level GUC, but ROLLBACK;
BEGIN;
SET application_name TO '';
SET application_name TO 'pg_regress';
ROLLBACK;
-- shards are hidden again after GUCs are reset
@ -94,7 +94,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name
-- we should hide correctly based on application_name with savepoints
BEGIN;
SAVEPOINT s1;
SET application_name TO '';
SET application_name TO 'pg_regress';
-- changing application_name reveals the shards
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK TO SAVEPOINT s1;
@ -102,9 +102,9 @@ ROLLBACK TO SAVEPOINT s1;
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;
-- changing citus.hide_shards_from_app_name_prefixes reveals the shards
-- changing citus.show_shards_for_app_name_prefix reveals the shards
BEGIN;
SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql';
SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql';
SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname;
ROLLBACK;