From ceb593c9da336a5b61a8288a1f2920914e1d669e Mon Sep 17 00:00:00 2001 From: Marco Slot Date: Mon, 2 May 2022 22:41:19 +0200 Subject: [PATCH] Convert citus.hide_shards_from_app_name_prefixes to citus.show_shards_for_app_name_prefixes --- src/backend/distributed/shared_library_init.c | 57 ++++++++++--------- .../distributed/sql/citus--11.0-1--11.0-2.sql | 3 +- .../sql/downgrades/citus--11.0-2--11.0-1.sql | 3 +- .../citus_shard_indexes_on_worker/11.0-2.sql | 39 +++++++++++++ .../citus_shard_indexes_on_worker/latest.sql | 2 +- .../udfs/citus_shards_on_worker/11.0-2.sql | 34 +++++++++++ .../udfs/citus_shards_on_worker/latest.sql | 2 +- .../worker/worker_shard_visibility.c | 23 ++++---- .../distributed/worker_shard_visibility.h | 2 +- .../expected/multi_mx_hide_shard_names.out | 12 ++-- src/test/regress/pg_regress_multi.pl | 2 +- .../regress/sql/multi_mx_hide_shard_names.sql | 12 ++-- 12 files changed, 134 insertions(+), 57 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql create mode 100644 src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 00b541968..fddd1b49d 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -159,9 +159,9 @@ static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra, static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source); static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source); static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source); -static bool HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, - GucSource source); -static void HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra); +static bool ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, + GucSource source); +static void ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra); static void ApplicationNameAssignHook(const char *newval, void *extra); static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source); static void NodeConninfoGucAssignHook(const char *newval, void *extra); @@ -1174,24 +1174,6 @@ RegisterCitusConfigVariables(void) GUC_NO_SHOW_ALL, NULL, NULL, NULL); - DefineCustomStringVariable( - "citus.hide_shards_from_app_name_prefixes", - gettext_noop("If application_name starts with one of these values, hide shards"), - gettext_noop("Citus places distributed tables and shards in the same schema. " - "That can cause confusion when inspecting the list of tables on " - "a node with shards. This GUC can be used to hide the shards from " - "pg_class for certain applications based on the application_name " - "of the connection. The default is *, which hides shards from all " - "applications. This behaviour can be overridden using the " - "citus.override_table_visibility setting"), - &HideShardsFromAppNamePrefixes, - "*", - PGC_USERSET, - GUC_STANDARD, - HideShardsFromAppNamePrefixesCheckHook, - HideShardsFromAppNamePrefixesAssignHook, - NULL); - DefineCustomIntVariable( "citus.isolation_test_session_process_id", NULL, @@ -1716,6 +1698,25 @@ RegisterCitusConfigVariables(void) GUC_STANDARD, NULL, NULL, NULL); + DefineCustomStringVariable( + "citus.show_shards_for_app_name_prefixes", + gettext_noop("If application_name starts with one of these values, show shards"), + gettext_noop("Citus places distributed tables and shards in the same schema. " + "That can cause confusion when inspecting the list of tables on " + "a node with shards. By default the shards are hidden from " + "pg_class. This GUC can be used to show the shards to certain " + "applications based on the application_name of the connection. " + "The default is empty string, which hides shards from all " + "applications. This behaviour can be overridden using the " + "citus.override_table_visibility setting"), + &ShowShardsForAppNamePrefixes, + "", + PGC_USERSET, + GUC_STANDARD, + ShowShardsForAppNamePrefixesCheckHook, + ShowShardsForAppNamePrefixesAssignHook, + NULL); + DefineCustomBoolVariable( "citus.sort_returning", gettext_noop("Sorts the RETURNING clause to get consistent test output"), @@ -1985,12 +1986,12 @@ WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source) /* - * HideShardsFromAppNamePrefixesCheckHook ensures that the - * citus.hide_shards_from_app_name_prefixes holds a valid list of application_name + * ShowShardsForAppNamePrefixesCheckHook ensures that the + * citus.show_shards_for_app_name_prefixes holds a valid list of application_name * values. */ static bool -HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source) +ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source) { List *prefixList = NIL; @@ -2020,7 +2021,7 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so if (strcmp(prefixAscii, appNamePrefix) != 0) { - GUC_check_errdetail("prefix %s in citus.hide_shards_from_app_name_prefixes " + GUC_check_errdetail("prefix %s in citus.show_shards_for_app_name_prefixes " "contains non-ascii characters", appNamePrefix); return false; } @@ -2031,12 +2032,12 @@ HideShardsFromAppNamePrefixesCheckHook(char **newval, void **extra, GucSource so /* - * HideShardsFromAppNamePrefixesAssignHook ensures changes to - * citus.hide_shards_from_app_name_prefixes are reflected in the decision + * ShowShardsForAppNamePrefixesAssignHook ensures changes to + * citus.show_shards_for_app_name_prefixes are reflected in the decision * whether or not to show shards. */ static void -HideShardsFromAppNamePrefixesAssignHook(const char *newval, void *extra) +ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra) { ResetHideShardsDecision(); } diff --git a/src/backend/distributed/sql/citus--11.0-1--11.0-2.sql b/src/backend/distributed/sql/citus--11.0-1--11.0-2.sql index 7f39b5980..54fdd2f44 100644 --- a/src/backend/distributed/sql/citus--11.0-1--11.0-2.sql +++ b/src/backend/distributed/sql/citus--11.0-1--11.0-2.sql @@ -1 +1,2 @@ --- bump version to 11.0-2 +#include "udfs/citus_shards_on_worker/11.0-2.sql" +#include "udfs/citus_shard_indexes_on_worker/11.0-2.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql b/src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql index 163dca315..fca765ba7 100644 --- a/src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql @@ -1 +1,2 @@ --- bump down version to 11.0-1 +#include "../udfs/citus_shards_on_worker/11.0-1.sql" +#include "../udfs/citus_shard_indexes_on_worker/11.0-1.sql" diff --git a/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql new file mode 100644 index 000000000..fd4684b18 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql @@ -0,0 +1,39 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( + OUT schema_name name, + OUT index_name name, + OUT table_type text, + OUT owner_name name, + OUT shard_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.show_shards_for_app_name_prefixes = '*' + AS $$ +BEGIN + -- this is the query that \di produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner", + c2.relname as "Table" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid + LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid + WHERE c.relkind IN ('i','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS + SELECT schema_name as "Schema", + index_name as "Name", + table_type as "Type", + owner_name as "Owner", + shard_name as "Table" + FROM pg_catalog.citus_shard_indexes_on_worker() s; diff --git a/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql index d98cdafe5..fd4684b18 100644 --- a/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql @@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( OUT shard_name name) RETURNS SETOF record LANGUAGE plpgsql - SET citus.hide_shards_from_app_name_prefixes = '' + SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \di produces, except pg_table_is_visible diff --git a/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql b/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql new file mode 100644 index 000000000..dbb7498e8 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql @@ -0,0 +1,34 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( + OUT schema_name name, + OUT shard_name name, + OUT table_type text, + OUT owner_name name) + RETURNS SETOF record + LANGUAGE plpgsql + SET citus.show_shards_for_app_name_prefixes = '*' + AS $$ +BEGIN + -- this is the query that \d produces, except pg_table_is_visible + -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) + RETURN QUERY + SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','p','v','m','S','f','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.relation_is_a_known_shard(c.oid) + ORDER BY 1,2; +END; +$$; + +CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS + SELECT schema_name as "Schema", + shard_name as "Name", + table_type as "Type", + owner_name as "Owner" + FROM pg_catalog.citus_shards_on_worker() s; diff --git a/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql b/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql index 895c92ae8..dbb7498e8 100644 --- a/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql @@ -5,7 +5,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( OUT owner_name name) RETURNS SETOF record LANGUAGE plpgsql - SET citus.hide_shards_from_app_name_prefixes = '' + SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \d produces, except pg_table_is_visible diff --git a/src/backend/distributed/worker/worker_shard_visibility.c b/src/backend/distributed/worker/worker_shard_visibility.c index da9c87a22..e482a955c 100644 --- a/src/backend/distributed/worker/worker_shard_visibility.c +++ b/src/backend/distributed/worker/worker_shard_visibility.c @@ -40,8 +40,8 @@ typedef enum HideShardsMode bool OverrideTableVisibility = true; bool EnableManualChangesToShards = false; -/* hide shards when the application_name starts with one of: */ -char *HideShardsFromAppNamePrefixes = "*"; +/* show shards when the application_name starts with one of: */ +char *ShowShardsForAppNamePrefixes = ""; /* cache of whether or not to hide shards */ static HideShardsMode HideShards = CHECK_APPLICATION_NAME; @@ -271,8 +271,8 @@ RelationIsAKnownShard(Oid shardRelationId) /* * HideShardsFromSomeApplications transforms queries to pg_class to - * filter out known shards if the application_name matches any of - * the prefixes in citus.hide_shards_from_app_name_prefixes. + * filter out known shards if the application_name does not match any of + * the prefixes in citus.show_shards_for_app_name_prefix. */ void HideShardsFromSomeApplications(Query *query) @@ -294,7 +294,7 @@ HideShardsFromSomeApplications(Query *query) * ShouldHideShards returns whether we should hide shards in the current * session. It only checks the application_name once and then uses a * cached response unless either the application_name or - * citus.hide_shards_from_app_name_prefixes changes. + * citus.show_shards_for_app_name_prefix changes. */ static bool ShouldHideShards(void) @@ -367,32 +367,33 @@ ShouldHideShardsInternal(void) List *prefixList = NIL; /* SplitGUCList scribbles on the input */ - char *splitCopy = pstrdup(HideShardsFromAppNamePrefixes); + char *splitCopy = pstrdup(ShowShardsForAppNamePrefixes); if (!SplitGUCList(splitCopy, ',', &prefixList)) { /* invalid GUC value, ignore */ - return false; + return true; } char *appNamePrefix = NULL; foreach_ptr(appNamePrefix, prefixList) { - /* always hide shards when one of the prefixes is * */ + /* never hide shards when one of the prefixes is * */ if (strcmp(appNamePrefix, "*") == 0) { - return true; + return false; } /* compare only the first first characters */ int prefixLength = strlen(appNamePrefix); if (strncmp(application_name, appNamePrefix, prefixLength) == 0) { - return true; + return false; } } - return false; + /* default behaviour: hide shards */ + return true; } diff --git a/src/include/distributed/worker_shard_visibility.h b/src/include/distributed/worker_shard_visibility.h index 957992fed..7eea5fbf7 100644 --- a/src/include/distributed/worker_shard_visibility.h +++ b/src/include/distributed/worker_shard_visibility.h @@ -15,7 +15,7 @@ extern bool OverrideTableVisibility; extern bool EnableManualChangesToShards; -extern char *HideShardsFromAppNamePrefixes; +extern char *ShowShardsForAppNamePrefixes; extern void HideShardsFromSomeApplications(Query *query); diff --git a/src/test/regress/expected/multi_mx_hide_shard_names.out b/src/test/regress/expected/multi_mx_hide_shard_names.out index c0265b282..60003caa2 100644 --- a/src/test/regress/expected/multi_mx_hide_shard_names.out +++ b/src/test/regress/expected/multi_mx_hide_shard_names.out @@ -114,7 +114,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name (2 rows) -- changing application_name reveals the shards -SET application_name TO ''; +SET application_name TO 'pg_regress'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; relname --------------------------------------------------------------------- @@ -137,7 +137,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- changing application_name in transaction reveals the shards BEGIN; -SET LOCAL application_name TO ''; +SET LOCAL application_name TO 'pg_regress'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; relname --------------------------------------------------------------------- @@ -160,7 +160,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- now with session-level GUC, but ROLLBACK; BEGIN; -SET application_name TO ''; +SET application_name TO 'pg_regress'; ROLLBACK; -- shards are hidden again after GUCs are reset SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; @@ -173,7 +173,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- we should hide correctly based on application_name with savepoints BEGIN; SAVEPOINT s1; -SET application_name TO ''; +SET application_name TO 'pg_regress'; -- changing application_name reveals the shards SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; relname @@ -196,9 +196,9 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name (2 rows) ROLLBACK; --- changing citus.hide_shards_from_app_name_prefixes reveals the shards +-- changing citus.show_shards_for_app_name_prefix reveals the shards BEGIN; -SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql'; +SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; relname --------------------------------------------------------------------- diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 9b03b88d8..b3147b06b 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -466,7 +466,7 @@ push(@pgOptions, "citus.explain_analyze_sort_method='taskId'"); push(@pgOptions, "citus.enable_manual_changes_to_shards=on"); # Some tests look at shards in pg_class, make sure we can usually see them: -push(@pgOptions, "citus.hide_shards_from_app_name_prefixes='psql,pg_dump'"); +push(@pgOptions, "citus.show_shards_for_app_name_prefixes='pg_regress'"); # we disable slow start by default to encourage parallelism within tests push(@pgOptions, "citus.executor_slow_start_interval=0ms"); diff --git a/src/test/regress/sql/multi_mx_hide_shard_names.sql b/src/test/regress/sql/multi_mx_hide_shard_names.sql index b56329150..558a699a6 100644 --- a/src/test/regress/sql/multi_mx_hide_shard_names.sql +++ b/src/test/regress/sql/multi_mx_hide_shard_names.sql @@ -67,7 +67,7 @@ SELECT * FROM citus_shard_indexes_on_worker WHERE "Schema" = 'mx_hide_shard_name SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; -- changing application_name reveals the shards -SET application_name TO ''; +SET application_name TO 'pg_regress'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; RESET application_name; @@ -76,7 +76,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- changing application_name in transaction reveals the shards BEGIN; -SET LOCAL application_name TO ''; +SET LOCAL application_name TO 'pg_regress'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; ROLLBACK; @@ -85,7 +85,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- now with session-level GUC, but ROLLBACK; BEGIN; -SET application_name TO ''; +SET application_name TO 'pg_regress'; ROLLBACK; -- shards are hidden again after GUCs are reset @@ -94,7 +94,7 @@ SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_name -- we should hide correctly based on application_name with savepoints BEGIN; SAVEPOINT s1; -SET application_name TO ''; +SET application_name TO 'pg_regress'; -- changing application_name reveals the shards SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; ROLLBACK TO SAVEPOINT s1; @@ -102,9 +102,9 @@ ROLLBACK TO SAVEPOINT s1; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; ROLLBACK; --- changing citus.hide_shards_from_app_name_prefixes reveals the shards +-- changing citus.show_shards_for_app_name_prefix reveals the shards BEGIN; -SET LOCAL citus.hide_shards_from_app_name_prefixes TO 'notpsql'; +SET LOCAL citus.show_shards_for_app_name_prefixes TO 'psql'; SELECT relname FROM pg_catalog.pg_class WHERE relnamespace = 'mx_hide_shard_names'::regnamespace ORDER BY relname; ROLLBACK;